Compare commits
64
Commits
e231185572
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b77a298a7e | ||
|
|
8db72edd3c | ||
|
|
d5d5449835 | ||
|
|
874a91a3c6 | ||
|
|
4669a8d1ab | ||
|
|
34197afa01 | ||
|
|
0125ccd7d2 | ||
|
|
fe5472bab4 | ||
|
|
48f5a7d3a9 | ||
|
|
4864a43413 | ||
|
|
cd88dbd336 | ||
|
|
65d570f114 | ||
|
|
1586bd1134 | ||
|
|
c43772fd94 | ||
|
|
7ee2b42230 | ||
|
|
e42becb479 | ||
|
|
43b59c5dff | ||
|
|
81715273d3 | ||
|
|
085b661936 | ||
|
|
21414307c8 | ||
|
|
b8c810a349 | ||
|
|
0f21fe7888 | ||
|
|
bd257afbd5 | ||
|
|
357e08a4f6 | ||
|
|
a83b921af0 | ||
|
|
3d09d5235f | ||
|
|
f883830975 | ||
|
|
96325c295c | ||
|
|
9ca2c2bfb3 | ||
|
|
20c8d9454a | ||
|
|
3d192a4857 | ||
|
|
8af77c2c75 | ||
|
|
fd522f4322 | ||
|
|
7d80155d16 | ||
|
|
bafc31bf85 | ||
|
|
fbf5a3f909 | ||
|
|
20e14872ef | ||
|
|
5702187695 | ||
|
|
a79182e53b | ||
|
|
ff16938096 | ||
|
|
e3f22c737f | ||
|
|
2b99d455fc | ||
|
|
9e99f31c23 | ||
|
|
df39a499ef | ||
|
|
11725908ec | ||
|
|
379a52c2e8 | ||
|
|
b0d489a86c | ||
|
|
6355176fda | ||
|
|
300d3c1bf4 | ||
|
|
baf1501d81 | ||
|
|
a5ba47e23c | ||
|
|
6a62a7e552 | ||
|
|
5890eb104d | ||
|
|
262b39ae09 | ||
|
|
2f2a8bb57e | ||
|
|
1cdf12df00 | ||
|
|
f614eeb445 | ||
|
|
53a3c91f53 | ||
|
|
df5f01bfa1 | ||
|
|
32014ac6de | ||
|
|
c7cb9f0a2c | ||
|
|
87410d79ed | ||
|
|
1c3c3edc7d | ||
|
|
b2e2bf0638 |
@@ -95,7 +95,7 @@ PROXY_IMAGE ?= mesh-route-proxy:$(VERSION)
|
||||
PROXY_DEV_TAG ?= mesh-route-proxy:development
|
||||
|
||||
proxy-image:
|
||||
docker build --build-arg GO_BASE=$(GO_BASE) -f examples/route-proxy/Dockerfile -t $(PROXY_IMAGE) -t $(PROXY_DEV_TAG) .
|
||||
docker build --build-arg GO_BASE=$(GO_BASE) -f modules/route-proxy/Dockerfile -t $(PROXY_IMAGE) -t $(PROXY_DEV_TAG) .
|
||||
@echo
|
||||
@docker image inspect $(PROXY_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||
|
||||
@@ -110,14 +110,18 @@ proxy-image:
|
||||
# after formatting never ran through it. The script is the one list of steps; this target only gives it
|
||||
# a database (MESH_TEST_POSTGRES, exported above).
|
||||
#
|
||||
# **Two levels** (issue 895): the unit level, `go test ./...` untagged, timed and held to a bound; and the integration
|
||||
# level, `-tags integration`, the tests that start a bus, a store or another process. Each is said with its time.
|
||||
#
|
||||
# **Packages in parallel, under the race detector, each test on a bus of its own** (internal/testbus),
|
||||
# as the script runs them: a bus per test, of the release the mesh runs, makes the suite the same in any
|
||||
# order, and the timeout bounds a hang to a failure with a stack, never a stalled gate.
|
||||
check: postgres
|
||||
@sh merge-check.sh ; status=$$? ; docker rm -f $(PG_CONTAINER) >/dev/null 2>&1 || true ; exit $$status
|
||||
|
||||
# Without a database the store's tests skip rather than fail, so this is the honest subset and not
|
||||
# the gate.
|
||||
# The unit level alone (untagged: no bus, no store, no other process), which a developer runs continuously. The tests
|
||||
# that need a bus or the store carry `//go:build integration` and run in `check`, not here; without a database the
|
||||
# store's tests would skip rather than fail anyway.
|
||||
test:
|
||||
go test -timeout 15m ./...
|
||||
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
#!/bin/sh
|
||||
# The unit tests of every Go module of its own below this repository's root, run from the root, for merge-check.sh
|
||||
# and for the test that tries it on a tree (merge_check_nested_test.go). Its own file so that the test needs no
|
||||
# particular merge-check.sh: the check that judges a change to merge-check.sh is main's.
|
||||
set -eu
|
||||
cgo="${cgo:-0}"
|
||||
race="${race:-}"
|
||||
|
||||
# nested_modules vets and tests every Go module of its own below the root, `modules/<name>/go.mod` (novox/hq ADR 0420
|
||||
# §5, issue 1025): `./...` stops at a nested go.mod, so a module that lives whole in this repository would have its
|
||||
# unit tests run by nobody. None is a pass. Each runs in its own directory, outside the vendor directory this
|
||||
# repository's own packages are built from; a nested module vendors what the forge holds (its own `vendor/`, so the
|
||||
# check runs offline), as this repository's root module does (issue 1019). A failing module fails the check; the others are still run.
|
||||
nested_modules() {
|
||||
nested=0
|
||||
for gomod in modules/*/go.mod; do
|
||||
[ -e "$gomod" ] || continue
|
||||
dir=$(dirname "$gomod")
|
||||
echo "nested module $dir:"
|
||||
if ! (cd "$dir" && GOFLAGS= CGO_ENABLED=0 go vet ./... &&
|
||||
GOFLAGS= CGO_ENABLED="${cgo:-0}" go test ${race:-} -count=1 -timeout 30m ./...); then
|
||||
echo "nested module $dir FAILED"
|
||||
nested=1
|
||||
fi
|
||||
done
|
||||
return $nested
|
||||
}
|
||||
|
||||
nested_modules
|
||||
@@ -27,6 +27,7 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -290,6 +291,26 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
if err := work.Began(ctx); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "cannot say a build started: %v\n", err)
|
||||
}
|
||||
// **A running check watches its pull request's key** (novox/hq ADR 0406 §4, issue 974): when the head or the trunk
|
||||
// commit in it moves, the check is for a state nobody wants any more, and ends itself — nothing is asked of the forge
|
||||
// (issue 726). A group's check, a try and a build watch nothing.
|
||||
var superseded atomic.Bool
|
||||
if w, ok := work.(interface {
|
||||
WatchPull(context.Context, func(link.PullKey)) (func(), error)
|
||||
}); ok && request.Check != nil {
|
||||
stopWatching, err := w.WatchPull(building, func(k link.PullKey) {
|
||||
superseded.Store(true)
|
||||
say("superseded", fmt.Sprintf("the pull request's key moved to %.12s on %.12s while this checked %.12s on %.12s: stopped",
|
||||
k.Head, k.Trunk, request.Ref, request.Check.Trunk))
|
||||
cancel()
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "the key of %s/%s#%d could not be watched, so its check runs on if the key moves: %v\n",
|
||||
request.Check.Owner, request.Check.Repo, request.Check.Number, err)
|
||||
} else {
|
||||
defer stopWatching()
|
||||
}
|
||||
}
|
||||
|
||||
result := link.BuildResult{
|
||||
ID: request.ID, Repository: request.Repository, Path: request.Path,
|
||||
@@ -382,7 +403,11 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
if mine != nil {
|
||||
killed = h.returned(mine) && err != nil
|
||||
}
|
||||
if killed {
|
||||
if superseded.Load() && err != nil {
|
||||
// Ended by its own key moving on (ADR 0406 §4): said as the check's failure, as a kill is.
|
||||
result.Failed = link.SupersededByKey
|
||||
say("failed", link.SupersededByKey)
|
||||
} else if killed {
|
||||
// **Killed by hand is the outcome, whatever the build was doing** (novox/hq ADR 0219): the
|
||||
// error it ended with is the kill's consequence, not a fault of the source.
|
||||
result.Failed = link.KilledByHand
|
||||
|
||||
+139
-72
@@ -27,9 +27,10 @@ import (
|
||||
// commit in it, with at most one commit per repository, the latest of its branch, which contains the earlier
|
||||
// ones. The walk names every merge it answers.
|
||||
//
|
||||
// - One walk at a time. Merges that come while a walk runs join the next batch. A walk that started is never
|
||||
// superseded; one still waiting for its delivery's word is folded into the next batch's walk, which
|
||||
// answers every merge it answered — the only supersession left.
|
||||
// - One walk at a time beside a related one (ADR 0407 decision 4, issue 973). Merges that come while a walk
|
||||
// runs join the next batch; a closed batch that shares nothing with an open walk is cut beside it. A walk
|
||||
// that started is never superseded; one still waiting for its delivery's word is folded into the walk of a
|
||||
// batch related to it, which answers every merge it answered — the only supersession left.
|
||||
// - A merge heard after a later merge of its branch was put in a walk is answered by that walk: a batch
|
||||
// still assembling, an open walk that builds everything it moves, or a walk done that did, which then
|
||||
// names it at once. Otherwise it joins the next batch.
|
||||
@@ -39,8 +40,9 @@ import (
|
||||
//
|
||||
// **A merge waits only for a batch it shares something with** (novox/hq ADR 0304 decision 1, ADR 0306 decisions
|
||||
// 2 and 3, issue 525): it joins a batch it is related to, and one that is related to none and whose delivery
|
||||
// group has no member still to merge is a batch whose window is closed at once (batchToJoin). Walks stay one at
|
||||
// a time: such a batch is cut when no walk is open, and queued behind the one that is otherwise.
|
||||
// group has no member still to merge is a batch whose window is closed at once (batchToJoin). Such a batch is cut
|
||||
// at once, beside any open walk it shares nothing with, and queued behind the first one it is related to
|
||||
// (cutUnrelated).
|
||||
//
|
||||
// A merge on the controller's own path (a module whose walk waits for nobody's word) never shares a batch with
|
||||
// one that waits for mesh-delivery's: each kind has a batch of its own, an own-path batch is cut first and folds
|
||||
@@ -61,13 +63,14 @@ var (
|
||||
// mergeWindowFileVar names the file the controller's window settings are written to.
|
||||
const mergeWindowFileVar = "MESH_MERGE_WINDOW_FILE"
|
||||
|
||||
// batchCutEvery is how often the controller looks whether a batch's window closed.
|
||||
const batchCutEvery = 5 * time.Second
|
||||
|
||||
// batchLateAfter is how long past its maximum a batch may still be assembling, with no walk open, before
|
||||
// the controller is said to have failed to cut it (ADR 0276 decision 9).
|
||||
const batchLateAfter = time.Minute
|
||||
|
||||
// mergeGrace is how long before a batch's first merge a merge may have been made and still be told apart from one
|
||||
// made before the controller began keeping merges (owedLate).
|
||||
const mergeGrace = 10 * time.Minute
|
||||
|
||||
// edgeGroupOrder is a delivery group's order made an edge of a walk's tiers (ADR 0276 decision 5): the
|
||||
// modules of the member after depend on the modules of the member before, and are gated on them.
|
||||
const edgeGroupOrder = "group-order"
|
||||
@@ -125,8 +128,7 @@ func durationSetting(v any) (time.Duration, bool) {
|
||||
// hearMerge puts a merge the forge announced into the batch that answers it, at the moment it was heard,
|
||||
// and cuts what is due.
|
||||
func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.Time) error {
|
||||
// One merge acted on at a time, whoever hands it over: the bus, or the catch-up that reads back
|
||||
// what the bus did not hand over (novox/hq issue 266).
|
||||
// One merge acted on at a time, whoever hands it over (novox/hq issue 266).
|
||||
actingOnMerges.Lock()
|
||||
defer actingOnMerges.Unlock()
|
||||
|
||||
@@ -245,8 +247,8 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
|
||||
}
|
||||
|
||||
// owedLate says a merge that reads as history is still the record's to answer (review of this change): a cut
|
||||
// marks every module it moved seen at that moment, so a merge of that branch the bus lost and the catch-up
|
||||
// hands over later — made before the cut, heard after it — reads as history and was dropped. One that touches
|
||||
// marks every module it moved seen at that moment, so a merge of that branch
|
||||
// handed over late — made before the cut, heard after it — reads as history and was dropped. One that touches
|
||||
// a module the mesh holds, of a branch whose merges the controller keeps, made since it began keeping them,
|
||||
// is kept; one made before it began is old news, as before.
|
||||
func owedLate(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, merged time.Time,
|
||||
@@ -340,8 +342,8 @@ func ownPath(touches []inventory.Entry) bool {
|
||||
// names why): a new batch, which waits for the window, beside the ones it shares nothing with; its words say
|
||||
// why.
|
||||
// - none, and no member of its group is still to merge: a new batch whose window is closed at once. Cut at
|
||||
// once means only that: walks stay one at a time (ADR 0276 decision 4), so with a walk open the batch is
|
||||
// queued behind it as any closed batch is.
|
||||
// once means only that: with a walk open that it is related to the batch is queued behind it as any closed
|
||||
// batch is (ADR 0276 decision 4, ADR 0407 decision 4); beside one it shares nothing with it is cut.
|
||||
func batchToJoin(ctx context.Context, inv *inventory.Inventory, heard inventory.BatchedMerge, entries []inventory.Entry,
|
||||
read map[string][]inventory.ReadRepository, now time.Time, own bool, waits string) (inventory.Plan, string, error) {
|
||||
edges, err := inv.Dependencies(ctx)
|
||||
@@ -406,7 +408,9 @@ func relatedBatches(ctx context.Context, inv *inventory.Inventory, merges []inve
|
||||
}
|
||||
carry := newFootprint()
|
||||
if !own {
|
||||
// An own-path batch folds no waiting walk (cutBatchesHeld); a catalogue batch may fold every one.
|
||||
// An own-path batch folds no waiting walk (cutBatchesHeld); a catalogue batch folds those it is related to
|
||||
// (cutUnrelated). Every one is counted here, which only makes a merge join a batch more readily: it waits
|
||||
// for the window, never walks beside what it shares something with (issue 973).
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
@@ -544,6 +548,11 @@ func newBatch(ctx context.Context, inv *inventory.Inventory, first inventory.Bat
|
||||
// keepBatch writes a batch as its merges and the clock say it now: the commits it holds, the merges they
|
||||
// answer, its window, and assembling or queued behind a walk. Saved only when that changed.
|
||||
func keepBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan, now time.Time, behind string) error {
|
||||
return keepBatchBecause(ctx, inv, b, now, behind, "")
|
||||
}
|
||||
|
||||
// keepBatchBecause is keepBatch for a batch queued behind a walk it is related to, which its words say (issue 973).
|
||||
func keepBatchBecause(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan, now time.Time, behind, why string) error {
|
||||
merges, err := inv.MergesOf(ctx, b.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -574,19 +583,25 @@ func keepBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan,
|
||||
cutAtOnce = was.CutAtOnce // the merge's, until another joins it (ADR 0304 decision 1)
|
||||
waits = was.Waits // its group's, as said when it was made (ADR 0306 decision 2)
|
||||
}
|
||||
b.Delivery.Batch = &inventory.PlanBatch{ClosesAt: closes, AtMost: latest, Behind: behind, Own: b.OwnPath(),
|
||||
b.Delivery.Batch = &inventory.PlanBatch{ClosesAt: closes, AtMost: latest, Behind: behind, BehindWhy: why, Own: b.OwnPath(),
|
||||
Closed: closed, Withheld: withheld, CutAtOnce: cutAtOnce, Waits: waits}
|
||||
b.State = inventory.PlanAssembling
|
||||
if behind != "" && windowClosed(*b, now) {
|
||||
b.State = inventory.PlanQueued
|
||||
} else {
|
||||
b.Delivery.Batch.Behind = ""
|
||||
b.Delivery.Batch.Behind, b.Delivery.Batch.BehindWhy = "", ""
|
||||
}
|
||||
b.Note = batchWords(*b, now)
|
||||
if unNoted(*b) == before {
|
||||
return nil
|
||||
}
|
||||
return inv.SavePlan(ctx, b)
|
||||
if err := inv.SavePlan(ctx, b); err != nil {
|
||||
return err
|
||||
}
|
||||
// The window's clock follows the record, only when the record moved: a merge made the quiet key again, a
|
||||
// batch that opened made its most (ADR 0407 decision 1).
|
||||
armWindow(ctx, *b, now)
|
||||
return nil
|
||||
}
|
||||
|
||||
// windowOf is when a batch's window closes, unless another merge comes, and when at the latest: the last
|
||||
@@ -739,31 +754,10 @@ func nameMerges(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan
|
||||
return inv.SavePlan(ctx, p)
|
||||
}
|
||||
|
||||
// batchCutter cuts what is due on a timer, until the context ends: a window closes with no merge to say so.
|
||||
func batchCutter(ctx context.Context, open *stores) {
|
||||
tick := time.NewTicker(batchCutEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
release, err := open.inventory.HoldPlans(ctx, false)
|
||||
if err != nil {
|
||||
continue // whoever holds the plans moves them, and cuts at the end
|
||||
}
|
||||
if err := cutBatchesHeld(ctx, open, time.Now().UTC()); err != nil {
|
||||
fmt.Printf("batches: %v\n", err)
|
||||
}
|
||||
release()
|
||||
}
|
||||
}
|
||||
|
||||
// cutBatchesHeld takes the step that is due, for a caller holding the plans: after a failed walk, the search
|
||||
// for the merge that brought the failure; with no walk open, the next merge of that search walked alone, or a
|
||||
// closed batch cut into one walk with the waiting walks folded in; with a walk open, a closed batch said
|
||||
// queued behind it.
|
||||
// for the merge that brought the failure; with no walk open, the next merge of that search walked alone; else
|
||||
// each closed batch cut into a walk with the waiting walks it is related to folded in, unless it is related to a
|
||||
// walk open, behind which it is said queued (cutUnrelated). While the search runs everything waits.
|
||||
func cutBatchesHeld(ctx context.Context, open *stores, now time.Time) error {
|
||||
inv := open.inventory
|
||||
if err := searchAfterFailures(ctx, inv); err != nil {
|
||||
@@ -780,8 +774,7 @@ func cutBatchesHeld(ctx context.Context, open *stores, now time.Time) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var started *inventory.Plan
|
||||
var waiting []inventory.Plan
|
||||
var started, waiting []inventory.Plan
|
||||
for i := range plans {
|
||||
p := plans[i]
|
||||
if p.Release != nil {
|
||||
@@ -789,8 +782,8 @@ func cutBatchesHeld(ctx context.Context, open *stores, now time.Time) error {
|
||||
}
|
||||
if p.Waiting() {
|
||||
waiting = append(waiting, p)
|
||||
} else if started == nil {
|
||||
started = &plans[i]
|
||||
} else {
|
||||
started = append(started, p)
|
||||
}
|
||||
}
|
||||
batches, err := inv.Batches(ctx)
|
||||
@@ -805,46 +798,116 @@ func cutBatchesHeld(ctx context.Context, open *stores, now time.Time) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if started != nil {
|
||||
// **One walk at a time** (ADR 0276 decision 4): every batch waits behind it, whatever its window says.
|
||||
return keepAll(started.ID)
|
||||
}
|
||||
if len(alone) > 0 {
|
||||
// A failed walk's search walks its merges alone, one at a time: everything waits for it (kept as it was,
|
||||
// issue 973).
|
||||
if len(started) > 0 {
|
||||
return keepAll(started[0].ID)
|
||||
}
|
||||
if len(waiting) > 0 {
|
||||
// The waiting walk is the one open; the search walks after it, and the batches wait behind both.
|
||||
return keepAll(waiting[0].ID)
|
||||
}
|
||||
return walkAlone(ctx, open, alone[0], now)
|
||||
}
|
||||
if err := keepAll(""); err != nil {
|
||||
return cutUnrelated(ctx, open, batches, started, waiting, now)
|
||||
}
|
||||
|
||||
// cutUnrelated cuts the closed batches that share nothing with an open walk, each into a walk beside the open
|
||||
// ones, and says the others queued behind the first walk they are related to (novox/hq ADR 0407 decision 4,
|
||||
// issue 973). A batch cut in this look counts as open for the batches after it.
|
||||
//
|
||||
// - An own-path batch goes first and folds no waiting walk; a catalogue batch folds the waiting walks it is
|
||||
// related to, and those it shares nothing with keep waiting.
|
||||
// - **The controller's own path is related to everything** (decided for issue 973): a walk or batch that
|
||||
// replaces the controller, the node-engine, the tool runner, the bus or mesh-delivery never runs beside
|
||||
// another, and nothing runs beside it.
|
||||
// - A batch whose window is still open is assembling; one related to an open walk is queued behind it, and
|
||||
// ADR 0276 decision 4 stands for it: nothing supersedes a walk that started.
|
||||
func cutUnrelated(ctx context.Context, open *stores, batches, started, waiting []inventory.Plan, now time.Time) error {
|
||||
inv := open.inventory
|
||||
sort.SliceStable(batches, func(i, j int) bool { return batches[i].OwnPath() && !batches[j].OwnPath() })
|
||||
var entries []inventory.Entry
|
||||
var read map[string][]inventory.ReadRepository
|
||||
var edges []inventory.Edge
|
||||
judged := false
|
||||
openWalks, err := openWalksOf(ctx, inv, started)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The oldest batch whose window closed is cut, an own-path one before a catalogue one (it goes first, and
|
||||
// the catalogue batch queues behind it rather than being cut and overtaken). Several of a kind may be open
|
||||
// since a merge that shares nothing is cut at once (ADR 0304 decision 1): walks stay one at a time, so one
|
||||
// is cut per look, and once its walk started the others are queued behind it.
|
||||
sort.SliceStable(batches, func(i, j int) bool { return batches[i].OwnPath() && !batches[j].OwnPath() })
|
||||
for i := range batches {
|
||||
b := &batches[i]
|
||||
if !windowClosed(*b, now) {
|
||||
continue
|
||||
}
|
||||
if b.OwnPath() {
|
||||
// A walk on the controller's own path folds no walk waiting for its word (decided during the build,
|
||||
// 2026-10-10): that walk keeps waiting beside it, and starts once this one ended (advanceOnce).
|
||||
waiting = nil
|
||||
}
|
||||
if err := cutBatch(ctx, open, b, waiting, now); err != nil {
|
||||
return err
|
||||
}
|
||||
// The batches left wait behind the walk just cut when it started; the next look does the same.
|
||||
if b.Open() && !b.Waiting() {
|
||||
if batches, err = inv.Batches(ctx); err != nil {
|
||||
if err := keepBatch(ctx, inv, b, now, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
return keepAll(b.ID)
|
||||
continue
|
||||
}
|
||||
if !judged {
|
||||
var err error
|
||||
if entries, err = inv.Catalogued(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if read, err = readForPlanning(ctx, inv); err != nil {
|
||||
return err
|
||||
}
|
||||
if edges, err = inv.Dependencies(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
judged = true
|
||||
}
|
||||
held, err := inv.MergesOf(ctx, b.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fp := footprintAsCut(held, entries, read, edges)
|
||||
var fold, keep []inventory.Plan
|
||||
if !b.OwnPath() {
|
||||
for _, w := range waiting {
|
||||
ww, err := openWalkOf(ctx, inv, w)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ok, _ := related(fp, ww.fp, edges); ok {
|
||||
fold = append(fold, w)
|
||||
fp = unionFootprint(fp, ww.fp)
|
||||
} else {
|
||||
keep = append(keep, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
if behind, why := relatedOpen(fp, b.OwnPath(), openWalks, edges); behind != "" {
|
||||
if err := keepBatchBecause(ctx, inv, b, now, behind, why); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := keepBatch(ctx, inv, b, now, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := cutBatch(ctx, open, b, fold, now); err != nil {
|
||||
return err
|
||||
}
|
||||
waiting = keep
|
||||
if c := windowClockOf(); c != nil {
|
||||
c.Forget(ctx, b.ID) // a batch cut has no window left to close
|
||||
}
|
||||
cut, err := inv.PlanByID(ctx, b.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !cut.Open() {
|
||||
continue
|
||||
}
|
||||
w, err := openWalkOf(ctx, inv, cut)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cut.Waiting() {
|
||||
waiting = append(waiting, cut)
|
||||
} else {
|
||||
openWalks = append(openWalks, w)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1396,7 +1459,11 @@ func batchWords(b inventory.Plan, now time.Time) string {
|
||||
grouped += " (on the controller's own path)"
|
||||
}
|
||||
if b.State == inventory.PlanQueued {
|
||||
return fmt.Sprintf("queued behind %s; grouped: %s; plan not yet calculated", w.Behind, grouped)
|
||||
because := ""
|
||||
if w.BehindWhy != "" {
|
||||
because = " (" + w.BehindWhy + ")"
|
||||
}
|
||||
return fmt.Sprintf("queued behind %s%s; grouped: %s; plan not yet calculated", w.Behind, because, grouped)
|
||||
}
|
||||
if w.CutAtOnce {
|
||||
return fmt.Sprintf("cut at once, sharing nothing with a batch assembling; grouped: %s; plan not yet calculated",
|
||||
|
||||
@@ -334,8 +334,8 @@ func captured(t *testing.T, run func() error) string {
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// One walk at a time: a merge heard while a walk runs joins the next batch, which is cut when the walk ends;
|
||||
// the walk that started is never superseded.
|
||||
// A merge heard during a walk it is related to (the same repository, issue 973) joins the next batch, which is
|
||||
// cut when the walk ends; the walk that started is never superseded (ADR 0276 decision 4, ADR 0407 decision 4).
|
||||
func TestAMergeDuringAWalkJoinsTheNextBatch(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
@@ -346,7 +346,7 @@ func TestAMergeDuringAWalkJoinsTheNextBatch(t *testing.T) {
|
||||
t.Fatalf("the first batch was not cut: %+v", ws)
|
||||
}
|
||||
first := ws[0]
|
||||
hear(t, open, repoMerge("two", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
|
||||
hear(t, open, repoMerge("one", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
|
||||
cutAt(t, open, t0.Add(5*time.Minute))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != first.ID {
|
||||
@@ -391,7 +391,7 @@ func TestAWaitingWalkIsFoldedIntoTheNextBatch(t *testing.T) {
|
||||
t.Fatalf("the walk does not wait for its word: %+v", ws)
|
||||
}
|
||||
waiting := ws[0]
|
||||
hear(t, open, repoMerge("two", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
|
||||
hear(t, open, repoMerge("one", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
|
||||
cutAt(t, open, t0.Add(2*time.Minute+90*time.Second))
|
||||
ws, bs := walks(t, open)
|
||||
if len(bs) != 0 || len(ws) != 2 {
|
||||
@@ -405,7 +405,7 @@ func TestAWaitingWalkIsFoldedIntoTheNextBatch(t *testing.T) {
|
||||
if folded.State != inventory.PlanSuperseded || folded.Delivery.TakenOverBy != newer.ID {
|
||||
t.Fatalf("the waiting walk is %s, taken over by %q", folded.State, folded.Delivery.TakenOverBy)
|
||||
}
|
||||
if len(newer.Delivery.Merges) != 2 || newer.CommitOf("novox/one") != "c1" || newer.CommitOf("novox/two") != "c2" {
|
||||
if len(newer.Delivery.Merges) != 2 || newer.CommitOf("novox/one") != "c2" {
|
||||
t.Fatalf("the newer walk answers %+v", newer.Delivery.Merges)
|
||||
}
|
||||
if _, in := newer.Modules["one"]; !in {
|
||||
@@ -662,8 +662,8 @@ func TestALateMergeOfAOneModuleRepositoryIsNamed(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// One walk at a time holds against the delivery's word too: a waiting walk let go beside a started one takes the
|
||||
// word and starts once that one ended, asking nothing before.
|
||||
// A related walk waits for the delivery's word too: a waiting walk let go beside a started one it is related to
|
||||
// (the same repository, issue 973) takes the word and starts once that one ended, asking nothing before.
|
||||
func TestAWalkLetGoBesideAStartedOneStartsOnceItEnded(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asked := asksWithPaths(t)
|
||||
@@ -671,8 +671,8 @@ func TestAWalkLetGoBesideAStartedOneStartsOnceItEnded(t *testing.T) {
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
started, _ := walks(t, open)
|
||||
waiting := inventory.Plan{ID: "plan-waiting", Repository: "novox/two", Branch: "main", Commit: "d1", Created: t0,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"two"}}, Modules: map[string]*inventory.PlanModule{"two": {}},
|
||||
waiting := inventory.Plan{ID: "plan-waiting", Repository: "novox/one", Branch: "main", Commit: "d1", Created: t0,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"one"}}, Modules: map[string]*inventory.PlanModule{"one": {}},
|
||||
Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}}
|
||||
if err := open.inventory.SavePlan(ctx, &waiting); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -729,49 +729,6 @@ func TestARemovedFileIsTheLastMergesWord(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The catch-up hands over what the bus lost after its branch's later merges were cut, and the record keeps it:
|
||||
// an earlier merge is named by the walk that carried it; a merge made before a cut and heard after it, which
|
||||
// no later merge carries, joins the next batch — neither reads as history and is dropped (review of this change).
|
||||
func TestTheCatchUpKeepsWhatACutMadeHistory(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
base := time.Now().UTC().Add(-time.Hour).Truncate(time.Second)
|
||||
hear(t, open, repoMerge("one", "c2", base.Add(10*time.Second)), base.Add(11*time.Second))
|
||||
cutAt(t, open, base.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
w := ws[0]
|
||||
w.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lost := []link.AnnouncedMerge{
|
||||
{SourceMoved: repoMerge("one", "c1", base), At: base.Add(time.Second)},
|
||||
{SourceMoved: repoMerge("one", "c3lost", base.Add(20*time.Second)), At: base.Add(21 * time.Second)},
|
||||
}
|
||||
catalogued := func(ctx context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
|
||||
entries, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
read, err := readForPlanning(ctx, open.inventory)
|
||||
return entries, read, err
|
||||
}
|
||||
if err := catchUpOnMerges(ctx, time.Now(), unheardMerges{announcedList(lost), open.inventory}, catalogued,
|
||||
(following{open: open}).SourceMoved, t.Logf); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
||||
if len(got.Delivery.Merges) != 2 || !sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/one",
|
||||
Commit: "c1", Carried: "c2"}) {
|
||||
t.Fatalf("the earlier merge the catch-up handed over is not named by the walk that carried it: %+v",
|
||||
got.Delivery.Merges)
|
||||
}
|
||||
if _, kept, err := open.inventory.MergeOf(ctx, "novox/one", "c3lost"); err != nil || !kept {
|
||||
t.Fatalf("the merge made before the cut and heard after it was dropped: %v %v", kept, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge on the controller's own path never shares a batch with one that waits for mesh-delivery's word
|
||||
// (decided during the build, 2026-10-10): the two are batches of their own, cut one after the other; a walk on
|
||||
// the own path folds no waiting catalogue walk but batches its merges again behind it; the catalogue's walk
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// An unrelated closed batch is cut into a walk beside an open one, and that walk asks its first tier at once
|
||||
// (issue 973, novox/hq ADR 0407 decision 4): a hotfix is never held by an unrelated delivery.
|
||||
func TestAnUnrelatedBatchWalksBesideAnOpenWalk(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
if ws, _ := walks(t, open); len(ws) != 1 || !ws[0].Open() {
|
||||
t.Fatalf("the first walk is not open: %+v", ws)
|
||||
}
|
||||
heard := t0.Add(100 * time.Second)
|
||||
hear(t, open, lone(repoMerge("two", "c2", heard)), heard)
|
||||
advanceHeld(ctx, open)
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 2 || len(bs) != 0 {
|
||||
t.Fatalf("%d walk(s) and %d batch(es); want the unrelated batch cut beside the open walk", len(ws), len(bs))
|
||||
}
|
||||
for _, w := range ws {
|
||||
if !w.Open() || w.Waiting() || deferred(w) || !askedAny(w) {
|
||||
t.Errorf("%s (%s) is not walking: open %v, deferred %v, asked %v, note %q", w.ID, w.Named(), w.Open(),
|
||||
deferred(w), askedAny(w), w.Note)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A closed batch related to an open walk is queued behind it with the reason, while one that shares nothing is
|
||||
// cut beside it; the walk that started is never superseded (ADR 0276 decision 4, issue 973).
|
||||
func TestARelatedBatchWaitsBehindTheOpenWalk(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
first := func() inventory.Plan { ws, _ := walks(t, open); return ws[len(ws)-1] }()
|
||||
heard := t0.Add(100 * time.Second)
|
||||
hear(t, open, lone(repoMerge("one", "c2", heard)), heard)
|
||||
hear(t, open, lone(repoMerge("three", "c3", heard.Add(time.Second))), heard.Add(time.Second))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 2 || len(bs) != 1 {
|
||||
t.Fatalf("%d walk(s) and %d batch(es); want the unrelated merge walking and the related one queued", len(ws), len(bs))
|
||||
}
|
||||
b := bs[0]
|
||||
words := batchWords(b, heard)
|
||||
if b.State != inventory.PlanQueued || b.Delivery.Batch.Behind != first.ID || b.CommitOf("novox/one") != "c2" ||
|
||||
!strings.Contains(words, "queued behind "+first.ID) || !strings.Contains(words, "both change novox/one") {
|
||||
t.Fatalf("the related batch is %s, behind %q, reading %q", b.State, b.Delivery.Batch.Behind, words)
|
||||
}
|
||||
if got, _ := open.inventory.PlanByID(ctx, first.ID); !got.Open() {
|
||||
t.Fatalf("the started walk was %s by the related merge", got.State)
|
||||
}
|
||||
first.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, heard.Add(2*time.Second))
|
||||
if _, bs := walks(t, open); len(bs) != 0 {
|
||||
t.Fatalf("the queued batch was not cut once the walk it waited for ended: %+v", bs)
|
||||
}
|
||||
}
|
||||
|
||||
// A batch on the controller's own path never walks beside another walk, and nothing walks beside it, whatever
|
||||
// the two share (issue 973): it replaces the controller, so it goes alone.
|
||||
func TestAnOwnPathBatchNeverWalksBeside(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
ws, _ := walks(t, open)
|
||||
first := ws[0]
|
||||
heard := t0.Add(100 * time.Second)
|
||||
hear(t, open, lone(repoMerge("mesh-controller", "k1", heard)), heard)
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 1 || !bs[0].OwnPath() || bs[0].State != inventory.PlanQueued ||
|
||||
bs[0].Delivery.Batch.Behind != first.ID || !strings.Contains(batchWords(bs[0], heard), "own path") {
|
||||
t.Fatalf("an own-path batch beside an open walk: %d walk(s), batches %+v", len(ws), bs)
|
||||
}
|
||||
first.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, heard.Add(time.Second))
|
||||
ws, _ = walks(t, open)
|
||||
if len(ws) != 2 || !ws[0].Open() || ws[0].CommitOf("novox/mesh-controller") != "k1" {
|
||||
t.Fatalf("the own-path batch was not cut once the walk ended: %+v", ws)
|
||||
}
|
||||
own := ws[0]
|
||||
// Unrelated to the own-path walk by every footprint, a merge still waits for it.
|
||||
later := heard.Add(time.Minute)
|
||||
hear(t, open, lone(repoMerge("two", "c2", later)), later)
|
||||
ws, bs = walks(t, open)
|
||||
if len(ws) != 2 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != own.ID {
|
||||
t.Fatalf("a merge beside the own-path walk: %d walk(s), batches %+v", len(ws), bs)
|
||||
}
|
||||
}
|
||||
@@ -435,6 +435,20 @@ func mayRegisterFrom(ctx context.Context, inv *inventory.Inventory, module strin
|
||||
module, sourceWords(built.Repository, built.Seat))
|
||||
}
|
||||
}
|
||||
// **Or the move the operator approved** (novox/hq ADR 0420 §7, issue 1025): a known module built from another
|
||||
// repository by a merge, whose pull request mesh-delivery merged on the operator's Approve of that head.
|
||||
// A verb alone, or a merge with no such warrant, is refused as before.
|
||||
movedWhy := ""
|
||||
if why != "" && !terminal && !isNew {
|
||||
accepted, err := approvedMove(ctx, inv, forms, buildID, built)
|
||||
switch {
|
||||
case err == nil:
|
||||
movedWhy, why = accepted, ""
|
||||
case !errors.Is(err, errNotAMerge):
|
||||
fmt.Printf("%s: %s is built from another repository, and no approved move covers this build: %v\n",
|
||||
buildID, module, err)
|
||||
}
|
||||
}
|
||||
if why != "" && !terminal {
|
||||
return refuse(why)
|
||||
}
|
||||
@@ -471,7 +485,8 @@ func mayRegisterFrom(ctx context.Context, inv *inventory.Inventory, module strin
|
||||
}
|
||||
// The same repository by the forge's id, not only its name: one deleted and made again is another.
|
||||
recorded := map[string]int64{}
|
||||
if !isNew {
|
||||
// A moved module is registered from the repository the operator approved, not the one it was registered from.
|
||||
if !isNew && movedWhy == "" {
|
||||
id, err := inv.SourceIdentity(ctx, module)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
@@ -492,6 +507,10 @@ func mayRegisterFrom(ctx context.Context, inv *inventory.Inventory, module strin
|
||||
owner, name, m, facts.ID, m, id))
|
||||
}
|
||||
}
|
||||
if movedWhy != "" {
|
||||
fmt.Printf("%s: %s moved from %s to %s/%s — registered at the operator's Approve: %s\n", buildID, module,
|
||||
sourceWords(was.Repository, was.Seat), owner, name, movedWhy)
|
||||
}
|
||||
return facts.ID, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -14,6 +14,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
@@ -382,3 +384,75 @@ func TestAnUnknownIdentityKeepsTheRecordedOne(t *testing.T) {
|
||||
t.Fatalf("the recorded id became %d", id)
|
||||
}
|
||||
}
|
||||
|
||||
// **The operator's Approve of a reviewed group registers a moved module from its new repository** (novox/hq ADR 0420 §7,
|
||||
// issue 1025). sudo is built from the catalogue; a merge into the repository of node-engine's module, announced by
|
||||
// mesh-delivery with the ask it merged on, brings its build from there. Accepted with the operator's Approve of the
|
||||
// merged head; refused with no kept merge, a warrant for other heads, a Decline, a person who is not the operator, and
|
||||
// through a verb.
|
||||
func TestAModuleMovesOnTheOperatorsApproveOfTheMergeThatMovedIt(t *testing.T) {
|
||||
const commit = "c0ffee0123456789"
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
change func(*anApprovedMerge)
|
||||
unkept bool
|
||||
request string // who asked: "merge", or "build" through a verb
|
||||
moved bool
|
||||
}{
|
||||
{what: "the operator's Approve of the merged head", request: "merge", moved: true},
|
||||
{what: "no merge mesh-delivery announced", unkept: true, request: "merge"},
|
||||
{what: "a warrant for other heads", request: "merge", change: func(a *anApprovedMerge) {
|
||||
a.merge.Heads = map[string]string{"novox/mesh-host@aaaaaaaaaaaa": "e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5"}
|
||||
}},
|
||||
{what: "a Decline", request: "merge", change: func(a *anApprovedMerge) {
|
||||
a.warrant.Option, a.warrant.Label, a.merge.Option = "decline", "Decline", "decline"
|
||||
}},
|
||||
{what: "a person who is not the operator", request: "merge", change: func(a *anApprovedMerge) { a.warrant.By.Who = "agent" }},
|
||||
{what: "a build asked through a verb", request: "build"},
|
||||
} {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
a := approvedMerge(t)
|
||||
a.merge.Owner, a.merge.Repo, a.merge.Number, a.merge.HeadSHA = "novox", "mesh-host", 7, "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1"
|
||||
if c.change != nil {
|
||||
c.change(&a)
|
||||
}
|
||||
wasKept, wasAsk := keptDeliveryMerge, routerAskOf
|
||||
keptDeliveryMerge = func(_ context.Context, owner, repo string, number int, head string) (deliveryMerge, bool, error) {
|
||||
return a.merge, !c.unkept && owner == "novox" && repo == "mesh-host" && number == 7 && head == a.merge.HeadSHA, nil
|
||||
}
|
||||
routerAskOf = func(_ context.Context, _ *inventory.Inventory, id string) (*asks.Ask, *asks.Warrant, error) {
|
||||
if id != a.merge.AskID {
|
||||
t.Errorf("%s: the router was asked for %q", c.what, id)
|
||||
}
|
||||
return &a.ask, &a.warrant, nil
|
||||
}
|
||||
t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk })
|
||||
event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-host", "base": "main", "number": 7,
|
||||
"head_sha": a.merge.HeadSHA, "merge_commit_sha": commit})
|
||||
if _, err := inv.AddMerge(ctx, inventory.BatchedMerge{Repository: "novox/mesh-host", Branch: "main", Commit: commit,
|
||||
Merged: time.Now(), Heard: time.Now(), Event: event}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id := fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano())
|
||||
if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: id, Repository: "novox/mesh-host", Seat: "git",
|
||||
Path: "modules/sudo", Ref: "main", Commit: commit, For: c.request}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
moved := onTrunk(id, "novox/mesh-host", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "2"})
|
||||
moved.Commit = commit
|
||||
_, _, err := takeIn(ctx, inv, moved)
|
||||
src, _ := inv.SourceOf(ctx, "sudo")
|
||||
switch {
|
||||
case c.moved && err != nil:
|
||||
t.Errorf("%s: the move was refused: %v", c.what, err)
|
||||
case c.moved && src.Repository != "novox/mesh-host":
|
||||
t.Errorf("%s: sudo is built from %q", c.what, src.Repository)
|
||||
case !c.moved && !errors.Is(err, errNotItsSource):
|
||||
t.Errorf("%s: the move was not refused as ADR 0266 has it: %v", c.what, err)
|
||||
case !c.moved && src.Repository != "novox/mesh-catalog":
|
||||
t.Errorf("%s: a refused move changed sudo's source to %q", c.what, src.Repository)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// moduleCheck judges manifests where they are written, with no mesh (novox/hq ADR 0037, issue 148).
|
||||
@@ -114,6 +115,20 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
failed += len(wrong)
|
||||
faulted[m.Module] = true
|
||||
}
|
||||
// A module lives whole in one repository (novox/hq ADR 0420 §1, issue 1019): when the check is told the
|
||||
// repository it runs for (MESH_CHECK_REPOSITORY, set by the build's check), an artifact building from
|
||||
// another is refused, as the merge gate refuses it. A run by hand names no repository and judges none.
|
||||
if repo := os.Getenv("MESH_CHECK_REPOSITORY"); repo != "" && m.Build != nil {
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Context != nil && a.Context.Repository != "" && !sameRepository(a.Context.Repository,
|
||||
link.SourceMoved{Owner: ownerOf(repo), Repo: repoOf(repo)}) {
|
||||
fmt.Fprintf(out, "%s: %s builds artifact %s from %s: a module lives whole in one repository "+
|
||||
"(novox/hq ADR 0420 §1)\n", path, m.Module, a.Name, a.Context.Repository)
|
||||
failed++
|
||||
faulted[m.Module] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if named := catalogue.InstallationProblems(m); len(named) > 0 {
|
||||
for _, p := range named {
|
||||
fmt.Fprintf(out, "%s: %s\n", path, p)
|
||||
|
||||
@@ -122,3 +122,40 @@ func TestASeatAnotherModuleDeclaresIsANoteOverSomeManifests(t *testing.T) {
|
||||
t.Fatalf("the note was not said:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// **A module lives whole in one repository** (novox/hq ADR 0420 §1, issue 1019): `module check`, told the
|
||||
// repository it runs for, refuses an artifact whose build context names another; run by hand, it does not know.
|
||||
func TestModuleCheckRefusesAnArtifactContextNamingAnotherRepository(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
write := func(name, context string) string {
|
||||
path := filepath.Join(dir, name+".json")
|
||||
raw := `{"module":"` + name + `","version":"1","build":{"artifacts":[{"name":"server","kind":"image","from":"Dockerfile"` + context + `}]}}`
|
||||
if err := os.WriteFile(path, []byte(raw), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
other := write("other", `,"context":{"seat":"git","repository":"novox/mesh-controller","ref":"main"}`)
|
||||
own := write("own", `,"context":{"seat":"git","repository":"novox/mesh-catalog","ref":"main"}`)
|
||||
none := write("none", ``)
|
||||
|
||||
t.Setenv("MESH_CHECK_REPOSITORY", "novox/mesh-catalog")
|
||||
var out bytes.Buffer
|
||||
if err := moduleCheck([]string{other}, &out); err == nil {
|
||||
t.Fatalf("a context naming another repository passed:\n%s", out.String())
|
||||
}
|
||||
if !strings.Contains(out.String(), "other builds artifact server from novox/mesh-controller: a module lives whole in one repository (novox/hq ADR 0420 §1)") {
|
||||
t.Errorf("the refusal is not in the words of the gate:\n%s", out.String())
|
||||
}
|
||||
for _, path := range []string{own, none} {
|
||||
out.Reset()
|
||||
if err := moduleCheck([]string{path}, &out); err != nil {
|
||||
t.Errorf("%s was refused: %v\n%s", path, err, out.String())
|
||||
}
|
||||
}
|
||||
t.Setenv("MESH_CHECK_REPOSITORY", "")
|
||||
out.Reset()
|
||||
if err := moduleCheck([]string{other}, &out); err != nil {
|
||||
t.Errorf("a local run, with no repository named, refused it: %v\n%s", err, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -191,8 +191,9 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||
if err := ask.Ask(ctx, request); err != nil {
|
||||
return err
|
||||
}
|
||||
// A new head ends the check of the old one (novox/hq issue 726).
|
||||
endingSupersededChecks(ctx, seat, p, request.ID, f.oneShot)
|
||||
// **A newer request replaces the one still waiting, and a newer key stops the one running** (novox/hq ADR 0406 §3, §4,
|
||||
// issue 974): the check queue keeps one request a pull request, and a running check watches its key. Nothing is asked
|
||||
// of the forge here (issue 726).
|
||||
what := "its own merge-check.sh alone: " + noModuleTouched
|
||||
if scope.gated() {
|
||||
what = fmt.Sprintf("the gate over %s (a merge would build %d module(s) in %d tier(s))",
|
||||
@@ -270,7 +271,7 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
|
||||
Held: heldBy(ctx),
|
||||
Seats: seatBases(ctx),
|
||||
Source: sourceOnSeat(source),
|
||||
Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base,
|
||||
Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base, Trunk: p.BaseSHA, Ready: p.Ready,
|
||||
Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents, New: scope.New,
|
||||
Manifests: scope.Manifests, Judge: scope.Judge, Builds: builds, Named: namedForTheCheck(ctx, p, builds),
|
||||
Release: release, Running: runningBy(ctx)},
|
||||
@@ -582,6 +583,13 @@ const maxCheckReport = 60 << 10
|
||||
// checked says a merge check's verdict as the controller's `checked`. Nothing is recorded or
|
||||
// registered: a check builds nothing (issue 240's rule for a dry run, kept for a check).
|
||||
func checked(ctx context.Context, result link.BuildResult) {
|
||||
// **A check stopped by its key says nothing** (novox/hq ADR 0406 §4, issue 974, review 2): it stopped because the head or
|
||||
// the trunk commit moved on, and the newer request will judge. Said as an error it would name the live head — red
|
||||
// statuses on the forge, a delivery refused — and written to the key it would stand in the newer request's way.
|
||||
if result.Failed == link.SupersededByKey {
|
||||
fmt.Printf("%s: stopped, its pull request's key moved on; the newer check will judge\n", result.ID)
|
||||
return
|
||||
}
|
||||
sayChecked(ctx, checkedOf(result))
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// Acceptance 5 of milestone 2 (novox/hq ADR 0406 §3, issue 974), live, with the store and a real bus: two merges into one
|
||||
// repository within a minute, with two open pull requests, leave each pull request with exactly one queued check, by the
|
||||
// stream's rule. Each merge moves the trunk, and mesh-delivery asks the controller to check both open pull requests again
|
||||
// against it (`delivery check`, which is the controller's own path for a head announced); nothing moves a request, nothing
|
||||
// asks the forge, and the queue is read back whole.
|
||||
func TestTwoMergesWithinAMinuteLeaveEachOpenPullRequestOneQueuedCheck(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
js, _ := aBuildQueue(t)
|
||||
servingBus.Store(js)
|
||||
t.Cleanup(func() { servingBus.Store(nil) })
|
||||
holdTheDeliverySeat(t, open)
|
||||
was := askDeliveryOwner
|
||||
t.Cleanup(func() { askDeliveryOwner = was })
|
||||
askDeliveryOwner = func(_ context.Context, _ *nats.Conn, verb string, _ map[string]any) (json.RawMessage, error) {
|
||||
if verb != "groups" {
|
||||
t.Fatalf("asked the owner %s", verb)
|
||||
}
|
||||
return json.RawMessage(`[]`), nil
|
||||
}
|
||||
// The repository the two pull requests are into, as a module the mesh builds from it.
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-controller", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
heads := map[int]string{7: "7777777777777777777777777777777777777777", 8: "8888888888888888888888888888888888888888"}
|
||||
check := func(number int, trunk string) {
|
||||
t.Helper()
|
||||
p := orderMember{ID: fmt.Sprintf("novox/mesh-controller@%.12s", heads[number]), Repository: "novox/mesh-controller",
|
||||
Base: "main", Head: heads[number], Number: number, Paths: []string{"main.go"}, ModuleDirsSaid: true,
|
||||
BaseSHA: trunk, Ready: number == 8}.pull()
|
||||
p.MergeCheck, p.MergeCheckSaid = true, true
|
||||
if err := (following{open: open, oneShot: true}).PullUpdated(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// Announced against the trunk as it stands, then asked again after each of two merges.
|
||||
for _, trunk := range []string{"1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222",
|
||||
"3333333333333333333333333333333333333333"} {
|
||||
check(7, trunk)
|
||||
check(8, trunk)
|
||||
}
|
||||
|
||||
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seen := map[int]string{}
|
||||
for _, a := range q.Asks {
|
||||
c := a.Request.Check
|
||||
if c == nil {
|
||||
t.Fatalf("%s is no check", a.ID)
|
||||
}
|
||||
if _, twice := seen[c.Number]; twice {
|
||||
t.Errorf("pull request %d has two queued checks", c.Number)
|
||||
}
|
||||
seen[c.Number] = c.Trunk
|
||||
if a.State != link.AskWaiting {
|
||||
t.Errorf("%s is %s", a.ID, a.State)
|
||||
}
|
||||
if want := "3333333333333333333333333333333333333333"; c.Trunk != want {
|
||||
t.Errorf("pull request %d is queued against %.8s; want the newest trunk %.8s", c.Number, c.Trunk, want)
|
||||
}
|
||||
}
|
||||
if len(seen) != 2 {
|
||||
t.Fatalf("queued checks for %v; want one for each of the two open pull requests", seen)
|
||||
}
|
||||
// By the streams' own count: the ready one on its stream, the other on its.
|
||||
for stream, want := range map[string]uint64{broker.ChecksReadyStream: 1, broker.ChecksOtherStream: 1} {
|
||||
info, err := js.Context().StreamInfo(stream)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.State.Msgs != want {
|
||||
t.Errorf("%s holds %d request(s); want %d", stream, info.State.Msgs, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,16 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
@@ -288,3 +295,47 @@ func TestARecheckedHeadsDirectVerdictNamesItsTrunk(t *testing.T) {
|
||||
t.Fatalf("a rechecked head's direct verdict names trunk %q", c.Trunk)
|
||||
}
|
||||
}
|
||||
|
||||
// A check stopped by its pull request's key moving on says no verdict (issue 974, review 2): an error said for it would name the
|
||||
// live head, and the newer request is the one that judges. A check that failed for any other reason still says its error.
|
||||
func TestACheckStoppedByItsKeyIsAnnouncedAsNoVerdict(t *testing.T) {
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
legacy, err := js.Conn().JetStream()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
was := checkEvents
|
||||
checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy}
|
||||
t.Cleanup(func() { checkEvents = was })
|
||||
heard := make(chan *nats.Msg, 4)
|
||||
sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyChecked), heard)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog", Number: 3}
|
||||
checked(t.Context(), link.BuildResult{ID: "stopped", Ref: "abc", Checked: asked, Failed: link.SupersededByKey})
|
||||
checked(t.Context(), link.BuildResult{ID: "broken", Ref: "abc", Checked: asked, Failed: "the facts snapshot cannot be read"})
|
||||
select {
|
||||
case m := <-heard:
|
||||
var c link.Checked
|
||||
_ = json.Unmarshal(m.Data, &c)
|
||||
if c.ID != "broken" || c.Verdict != "error" {
|
||||
t.Fatalf("the first verdict said is %+v; want only the broken check's error", c)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("nothing was said")
|
||||
}
|
||||
select {
|
||||
case m := <-heard:
|
||||
t.Fatalf("a second verdict was said: %s", m.Data)
|
||||
case <-time.After(300 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
|
||||
@@ -136,8 +136,9 @@ func letGo(ctx context.Context, inv *inventory.Inventory, id, by, why string) (i
|
||||
return p, fmt.Errorf("%s was let go by %s at %s already", p.ID, p.Delivery.By,
|
||||
p.Delivery.Go.Local().Format("15:04:05"))
|
||||
}
|
||||
// **One walk at a time** (novox/hq ADR 0276): the word is taken, and the walk starts once no other walk is
|
||||
// started (advanceOnce), so the delivery's owner says it once and is not refused.
|
||||
// **One walk at a time beside a related one** (novox/hq ADR 0276, ADR 0407 decision 4, issue 973): the word is
|
||||
// taken, and the walk starts once no started walk it is related to is open (advanceOnce), so the delivery's
|
||||
// owner says it once and is not refused.
|
||||
now := time.Now().UTC()
|
||||
p.Delivery.Go, p.Delivery.By, p.Delivery.Why = &now, by, why
|
||||
p.Note = "let go by " + by + "; its first tier is asked next"
|
||||
@@ -268,7 +269,7 @@ func stopWalkWithout(ctx context.Context, inv *inventory.Inventory, id, by, why
|
||||
append(moves, foldMoves...)); err != nil {
|
||||
return p, said, fmt.Errorf("%s is not stopped, and none of its merges moved: %w", p.ID, err)
|
||||
}
|
||||
// What the batch holds and names now: written again by the cutter's next look should this fail.
|
||||
// What the batch holds and names now: written again by the next look should this fail.
|
||||
if err := keepBatch(ctx, inv, &target, now, ""); err != nil {
|
||||
fmt.Printf(" %s holds its merges; its record is written again at the next look: %v\n", target.ID, err)
|
||||
}
|
||||
@@ -583,6 +584,9 @@ type orderMember struct {
|
||||
// BaseSHA is the base branch's head the forge last announced the head with (novox/hq ADR 0323): the trunk commit
|
||||
// a verdict no build judges names when the head is checked again through mesh-delivery.
|
||||
BaseSHA string `json:"base_sha,omitempty"`
|
||||
// Ready says the member's pull request is marked ready (novox/hq ADR 0406 §3, issue 974): its check is queued before
|
||||
// every other check. Said by mesh-delivery on a ready mark.
|
||||
Ready bool `json:"ready,omitempty"`
|
||||
}
|
||||
|
||||
// pull is the member as the forge announced it.
|
||||
@@ -594,7 +598,7 @@ func (m orderMember) pull() link.PullUpdated {
|
||||
}
|
||||
return link.PullUpdated{Owner: owner, Repo: repo, Number: m.Number, Base: base, Commit: m.Head, BaseSHA: m.BaseSHA,
|
||||
CloneURL: m.CloneURL, Paths: m.Paths, PathsTruncated: m.PathsTruncated, Removed: m.Removed,
|
||||
ModuleDirs: m.ModuleDirs, ModuleDirsSaid: m.ModuleDirsSaid}
|
||||
ModuleDirs: m.ModuleDirs, ModuleDirsSaid: m.ModuleDirsSaid, Ready: m.Ready}
|
||||
}
|
||||
|
||||
// orderPair is one "before" among a group's members, why, and the rules it won over.
|
||||
@@ -891,6 +895,8 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
removed := set.String("removed", "", "the files it deletes, comma-separated")
|
||||
membersJSON := set.String("members", "", "a group's members, as JSON")
|
||||
group := set.String("group", "", "a delivery group's id")
|
||||
ready := set.Bool("ready", false, "the head's pull request is marked ready: its check is queued before every other check "+
|
||||
"(novox/hq ADR 0406 §3)")
|
||||
by := set.String("by", catalogue.DeliverySeat, "who says it")
|
||||
why := set.String("why", "", "why")
|
||||
without := set.String("without", "", "merges a stop leaves out, <repository>@<commit>, comma-separated: the walk's "+
|
||||
@@ -906,6 +912,12 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
if err := json.Unmarshal([]byte(*membersJSON), &members); err != nil {
|
||||
return fmt.Errorf("the members are not readable: %w", err)
|
||||
}
|
||||
// Marked ready by the verb's word for every member it names, besides any member that says so itself.
|
||||
if *ready {
|
||||
for i := range members {
|
||||
members[i].Ready = true
|
||||
}
|
||||
}
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
@@ -976,7 +988,7 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s (%s at %s) is let go by %s; its first tier is asked at the next pass\n", p.ID, p.Repository,
|
||||
fmt.Printf("%s (%s at %s) is let go by %s; its first tier is asked when the controller next looks at the plans, which it does as this ends\n", p.ID, p.Repository,
|
||||
short(p.Commit), *by)
|
||||
return nil
|
||||
case "stop":
|
||||
@@ -1165,6 +1177,8 @@ func askGroupCheck(ctx context.Context, open *stores, group string, members []or
|
||||
return "", err
|
||||
}
|
||||
request.Check.Group = group
|
||||
// A group's check is queued as ready when any member's pull request is marked ready.
|
||||
request.Check.Ready = slices.ContainsFunc(members, func(m orderMember) bool { return m.Ready })
|
||||
world, err := theRestOfTheMesh(ctx, open.inventory, shelfOf(entries), "")
|
||||
if err != nil {
|
||||
return "", err
|
||||
|
||||
@@ -287,6 +287,12 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
m.Architecture, m.Kernel = reported.Architecture, reported.Kernel
|
||||
// Whether its node-engine takes handed files is its contract (issue 969).
|
||||
if stated, had, err := inv.HealthOf(ctx, n.Name); err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
} else if had {
|
||||
m.Contract = stated.Contract
|
||||
}
|
||||
outward, err := inv.OutwardLinksOf(ctx, n.Name)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
|
||||
@@ -227,8 +227,9 @@ func TestAMergeRelatedToTwoBatchesFoldsThem(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// With a walk open, a batch cut at once still waits for it, said queued behind it (walks stay one at a time, ADR
|
||||
// 0276 decision 4); it is cut once that walk ended, with no window to wait for.
|
||||
// With a walk open that it is related to (the same repository, issue 973), a batch cut at once still waits for
|
||||
// it, said queued behind it (ADR 0276 decision 4, ADR 0407 decision 4); it is cut once that walk ended, with no
|
||||
// window to wait for.
|
||||
func TestACutAtOnceBatchQueuesBehindAnOpenWalk(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "pr/c1")
|
||||
@@ -241,7 +242,7 @@ func TestACutAtOnceBatchQueuesBehindAnOpenWalk(t *testing.T) {
|
||||
}
|
||||
first := ws[0]
|
||||
heard := t0.Add(100 * time.Second)
|
||||
hear(t, open, lone(repoMerge("two", "c2", heard)), heard)
|
||||
hear(t, open, lone(repoMerge("one", "c2", heard)), heard)
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != first.ID ||
|
||||
!bs[0].Delivery.Batch.CutAtOnce {
|
||||
@@ -280,7 +281,8 @@ func TestAStopWalksTheOthersAgainInABatchOfTheirOwnWhenTheyShareNothing(t *testi
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
walk := ws[0]
|
||||
hear(t, open, repoMerge("two", "two1cccccccc", t0.Add(3*time.Minute)), t0.Add(3*time.Minute))
|
||||
// Related to the walk by its repository (queued behind it), and to nothing the stop walks again.
|
||||
hear(t, open, catalogueMerge("cat2cccccccc", "notes", t0.Add(3*time.Minute)), t0.Add(3*time.Minute))
|
||||
_, bs := walks(t, open)
|
||||
if len(bs) != 1 {
|
||||
t.Fatalf("the merge heard while the walk ran is in %d batches", len(bs))
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// **A repository's merge check runs every module's unit tests, a Go module of its own below the root included**
|
||||
// (novox/hq ADR 0420 §5, issue 1025): `./...` stops at a nested go.mod. This runs the script
|
||||
// check-nested-modules.sh in a tree with a `go` that only records where it was run and fails for a module named bad.
|
||||
func TestTheMergeCheckVetsAndTestsEveryNestedGoModule(t *testing.T) {
|
||||
script, err := filepath.Abs("../../check-nested-modules.sh")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bin, tree := t.TempDir(), t.TempDir()
|
||||
log := filepath.Join(t.TempDir(), "go.log")
|
||||
fake := "#!/bin/sh\necho \"$(basename \"$PWD\") $1 GOFLAGS=[$GOFLAGS]\" >>\"$LOG\"\n" +
|
||||
"[ \"$(basename \"$PWD\")\" = bad ] && [ \"$1\" = test ] && exit 1\nexit 0\n"
|
||||
if err := os.WriteFile(filepath.Join(bin, "go"), []byte(fake), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
run := func() (string, error) {
|
||||
cmd := exec.Command("sh", script)
|
||||
cmd.Dir = tree
|
||||
cmd.Env = append(os.Environ(), "PATH="+bin+":"+os.Getenv("PATH"), "LOG="+log, "GOFLAGS=-mod=vendor")
|
||||
out, err := cmd.CombinedOutput()
|
||||
return string(out), err
|
||||
}
|
||||
// No modules directory: a pass, nothing run.
|
||||
if out, err := run(); err != nil {
|
||||
t.Fatalf("a repository with no nested module failed: %v\n%s", err, out)
|
||||
}
|
||||
if _, err := os.Stat(log); err == nil {
|
||||
t.Fatal("go ran with no nested module")
|
||||
}
|
||||
for _, m := range []string{"good", "bad", "other"} {
|
||||
dir := filepath.Join(tree, "modules", m)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module x\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// A directory with no go.mod is no Go module of its own.
|
||||
if err := os.MkdirAll(filepath.Join(tree, "modules", "typescript"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := run()
|
||||
if err == nil {
|
||||
t.Fatalf("a nested module whose tests fail passed the check:\n%s", out)
|
||||
}
|
||||
raw, _ := os.ReadFile(log)
|
||||
ran := string(raw)
|
||||
for _, want := range []string{"good vet GOFLAGS=[]", "good test GOFLAGS=[]", "other test GOFLAGS=[]", "bad test GOFLAGS=[]"} {
|
||||
if !strings.Contains(ran, want) {
|
||||
t.Errorf("%q was not run, nor outside the vendor directory:\n%s", want, ran)
|
||||
}
|
||||
}
|
||||
if strings.Contains(ran, "typescript") {
|
||||
t.Errorf("a directory with no go.mod was run:\n%s", ran)
|
||||
}
|
||||
if !strings.Contains(out, "nested module modules/bad FAILED") {
|
||||
t.Errorf("the failing module is not named:\n%s", out)
|
||||
}
|
||||
}
|
||||
@@ -67,6 +67,8 @@ type mergeCheckInput struct {
|
||||
group []groupChange
|
||||
// admin is a PostgreSQL the gate may create and drop databases in.
|
||||
admin string
|
||||
// beside is the directory holding the core clones beside the check (MESH_CHECK_BESIDE), for finding moves.
|
||||
beside string
|
||||
// say is where progress goes; the verdict is returned.
|
||||
say io.Writer
|
||||
}
|
||||
@@ -197,7 +199,7 @@ func mergeGateCommand(ctx context.Context, args []string) error {
|
||||
os.Stdout = os.Stderr
|
||||
}
|
||||
v, err := judgeChange(ctx, mergeCheckInput{facts: f, repository: *repository, tree: *tree, changed: splitList(*changed),
|
||||
group: others, admin: *admin, say: out})
|
||||
group: others, admin: *admin, say: out, beside: os.Getenv("MESH_CHECK_BESIDE")})
|
||||
os.Stdout = stdout
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -271,6 +273,25 @@ func judgeChange(ctx context.Context, in mergeCheckInput) (mergeVerdict, error)
|
||||
// The change, then each other head of its delivery group laid over it in turn (novox/hq ADR 0239): one
|
||||
// future state of the mesh, judged as the group would leave it.
|
||||
heads := append([]groupChange{{Repository: in.repository, Tree: in.tree, Changed: in.changed}}, in.group...)
|
||||
// **A module one head removes and another defines is a move** (novox/hq ADR 0420 §6, issue 1025): neither
|
||||
// the removal of a module machines run (ADR 0236) nor a second definition of one module name (issue 239).
|
||||
// The change is judged with the module built from the repository that now defines it.
|
||||
moves, err := movesOfChange(in, sources)
|
||||
if err != nil {
|
||||
return v, err
|
||||
}
|
||||
changeSources := sources
|
||||
if len(moves) > 0 {
|
||||
changeSources = make(map[string]inventory.Source, len(sources))
|
||||
for k, o := range sources {
|
||||
changeSources[k] = o
|
||||
}
|
||||
for name, mv := range moves {
|
||||
o := changeSources[name]
|
||||
o.Repository, o.Path = mv.To, mv.Dir
|
||||
changeSources[name] = o
|
||||
}
|
||||
}
|
||||
for i, h := range heads {
|
||||
var r *mergeReach
|
||||
if h.Repository != "" && len(h.Changed) > 0 {
|
||||
@@ -289,13 +310,23 @@ func judgeChange(ctx context.Context, in mergeCheckInput) (mergeVerdict, error)
|
||||
continue
|
||||
}
|
||||
var failures []string
|
||||
change, failures, err = shelfWithChange(change, sources, in.facts, h.Repository, h.Tree, v.Modules,
|
||||
scopeOfReach(r, h.Repository))
|
||||
scope := scopeOfReach(r, h.Repository)
|
||||
if scope != nil {
|
||||
for _, mv := range moves {
|
||||
if mv.To == h.Repository {
|
||||
scope[mv.Dir] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
change, failures, err = shelfWithChange(change, changeSources, in.facts, h.Repository, h.Tree, v.Modules, scope)
|
||||
if err != nil {
|
||||
return v, err
|
||||
}
|
||||
v.Failures = append(v.Failures, failures...)
|
||||
}
|
||||
for name := range moves {
|
||||
v.Modules[name] = "moved"
|
||||
}
|
||||
if len(in.group) > 0 {
|
||||
say("a delivery group: %d head(s) composed together", len(heads))
|
||||
}
|
||||
@@ -332,7 +363,7 @@ func judgeChange(ctx context.Context, in mergeCheckInput) (mergeVerdict, error)
|
||||
}
|
||||
}
|
||||
sort.Strings(added)
|
||||
after, tried, notes, err := composeEveryMachine(ctx, in, change, sources, added)
|
||||
after, tried, notes, err := composeEveryMachine(ctx, in, change, changeSources, added)
|
||||
if err != nil {
|
||||
return v, fmt.Errorf("the mesh with the change could not be raised: %w", err)
|
||||
}
|
||||
@@ -531,6 +562,120 @@ func sourcesOfFacts(f snapshot.Facts) map[string]inventory.Source {
|
||||
return out
|
||||
}
|
||||
|
||||
// movesOfChange is the modules the change moves: with a delivery group, those the group's heads move; without
|
||||
// one, those it moves with the core clones beside the check (MESH_CHECK_BESIDE, mesh-catalog and
|
||||
// mesh-controller, the repository a module can move between), so a member judged alone sees the move its
|
||||
// group's composed check sees (issue 1043). The clones beside are used to find moves only, never as a
|
||||
// composition of what machines get.
|
||||
func movesOfChange(in mergeCheckInput, sources map[string]inventory.Source) (map[string]moveOfModule, error) {
|
||||
heads := append([]groupChange{{Repository: in.repository, Tree: in.tree, Changed: in.changed}}, in.group...)
|
||||
if len(in.group) == 0 && in.beside != "" && in.repository != "" && in.tree != "" {
|
||||
for _, name := range []string{"mesh-catalog", "mesh-controller"} {
|
||||
dir := filepath.Join(in.beside, name)
|
||||
if repoOf(in.repository) == name {
|
||||
continue
|
||||
}
|
||||
if st, err := os.Stat(dir); err != nil || !st.IsDir() {
|
||||
continue
|
||||
}
|
||||
heads = append(heads, groupChange{Repository: ownerOf(in.repository) + "/" + name, Tree: dir})
|
||||
}
|
||||
}
|
||||
return movesInGroup(heads, sources)
|
||||
}
|
||||
|
||||
// moveOfModule is a module a delivery group moves: the repository the mesh builds it from, the one whose
|
||||
// head now defines it, and its directory there.
|
||||
type moveOfModule struct{ From, To, Dir string }
|
||||
|
||||
// movesInGroup is the modules one head of a group removes and another defines (novox/hq ADR 0420 §6): a name
|
||||
// the mesh builds from a head's repository that the head's tree no longer defines, and some other head's tree
|
||||
// defines. A name removed by a head with no other head defining it, or defined by a head the mesh builds from
|
||||
// another repository with none removing it, is no move.
|
||||
func movesInGroup(heads []groupChange, sources map[string]inventory.Source) (map[string]moveOfModule, error) {
|
||||
if len(heads) < 2 {
|
||||
return nil, nil
|
||||
}
|
||||
defined := make([]map[string]string, len(heads))
|
||||
for i, h := range heads {
|
||||
if h.Tree == "" {
|
||||
continue
|
||||
}
|
||||
d, err := namesDefinedIn(h.Tree)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defined[i] = d
|
||||
}
|
||||
var moves map[string]moveOfModule
|
||||
for name, s := range sources {
|
||||
if s.Repository == "" {
|
||||
continue
|
||||
}
|
||||
for i, h := range heads {
|
||||
if h.Tree == "" || !sameRepository(s.Repository, link.SourceMoved{Owner: ownerOf(h.Repository), Repo: repoOf(h.Repository)}) {
|
||||
continue
|
||||
}
|
||||
if _, still := defined[i][name]; still {
|
||||
break
|
||||
}
|
||||
// Removed by head i: defined by another?
|
||||
for j, o := range heads {
|
||||
if dir, ok := defined[j][name]; ok && j != i && o.Repository != h.Repository {
|
||||
if moves == nil {
|
||||
moves = map[string]moveOfModule{}
|
||||
}
|
||||
moves[name] = moveOfModule{From: h.Repository, To: o.Repository, Dir: dir}
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
return moves, nil
|
||||
}
|
||||
|
||||
// namesDefinedIn is the module names a tree's module.json files define, each with its directory; a manifest
|
||||
// that does not read is left to shelfWithChange to refuse.
|
||||
func namesDefinedIn(tree string) (map[string]string, error) {
|
||||
found := map[string]string{}
|
||||
err := filepath.WalkDir(tree, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
if path != tree && (ignoredInTree[d.Name()] || strings.HasPrefix(d.Name(), ".")) {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if d.Name() != "module.json" {
|
||||
return nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
}
|
||||
if json.Unmarshal(raw, &m) != nil || m.Module == "" {
|
||||
return nil
|
||||
}
|
||||
dir, err := filepath.Rel(tree, filepath.Dir(path))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if dir == "." {
|
||||
dir = ""
|
||||
}
|
||||
if _, twice := found[m.Module]; !twice {
|
||||
found[m.Module] = dir
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return found, err
|
||||
}
|
||||
|
||||
// ignoredInTree are directories a module.json in is not a module of the repository.
|
||||
var ignoredInTree = map[string]bool{".git": true, "vendor": true, "node_modules": true, "testdata": true,
|
||||
"examples": true, "fixtures": true}
|
||||
@@ -621,6 +766,21 @@ func shelfWithChange(base map[string]catalogue.Manifest, sources map[string]inve
|
||||
"of one module name (issue 239)", orRoot(dir), m.Module, s.Repository))
|
||||
return nil
|
||||
}
|
||||
// A module lives whole in one repository (novox/hq ADR 0420 §1, issue 1019): no artifact builds from another.
|
||||
if m.Build != nil {
|
||||
own := link.SourceMoved{Owner: ownerOf(repository), Repo: repoOf(repository)}
|
||||
other := false
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Context != nil && a.Context.Repository != "" && !sameRepository(a.Context.Repository, own) {
|
||||
failures = append(failures, fmt.Sprintf("%s builds artifact %s from %s: a module lives whole in "+
|
||||
"one repository (novox/hq ADR 0420 §1)", m.Module, a.Name, a.Context.Repository))
|
||||
other = true
|
||||
}
|
||||
}
|
||||
if other {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
resolved, err := standInBuild(written, texts)
|
||||
if err == nil {
|
||||
// As registration reads a built manifest: strictly, again (takeIn).
|
||||
@@ -1017,6 +1177,13 @@ func raiseFromFacts(ctx context.Context, open *stores, f snapshot.Facts, shelf m
|
||||
"kernel": m.Kernel, "capabilities": caps}); err != nil {
|
||||
return notes, err
|
||||
}
|
||||
// What its node-engine takes, as the mesh composes it today (issue 969): without the statement the gate
|
||||
// composes a moved module the old way on a machine the mesh hands it to.
|
||||
if stated, ok := standInHealth(m, f.Taken); ok {
|
||||
if _, err := inv.RecordHealth(ctx, stated); err != nil {
|
||||
return notes, err
|
||||
}
|
||||
}
|
||||
for _, record := range []func(context.Context, string, string) error{inv.RecordSealingKey, inv.RecordOverlayKey} {
|
||||
key, err := standInKey()
|
||||
if err != nil {
|
||||
@@ -1260,6 +1427,8 @@ func graphOfFacts(f snapshot.Facts) ([]inventory.Entry, map[string][]inventory.R
|
||||
read[mod.Name] = append(read[mod.Name], inventory.ReadRepository{Own: true, Paths: s.Paths})
|
||||
}
|
||||
}
|
||||
// A context naming the module's own repository is its own build source (novox/hq ADR 0420 §4).
|
||||
read[mod.Name] = inventory.OwnRepositoryAsOwn(read[mod.Name], mod.Repository)
|
||||
}
|
||||
var edges []inventory.Edge
|
||||
for _, e := range f.Edges {
|
||||
@@ -1355,3 +1524,12 @@ func firstOr(items, otherwise []string) string {
|
||||
}
|
||||
return "it says nothing more"
|
||||
}
|
||||
|
||||
// standInHealth is the statement the gate gives a machine of the snapshot so that it composes as the mesh does:
|
||||
// only its contract, which says whether it takes handed files (issue 969). False for a snapshot that carries none.
|
||||
func standInHealth(m snapshot.Machine, taken time.Time) (inventory.NodeHealth, bool) {
|
||||
if m.Contract == 0 {
|
||||
return inventory.NodeHealth{}, false
|
||||
}
|
||||
return inventory.NodeHealth{Node: m.Name, Contract: m.Contract, SaidAt: taken, HeardAt: taken}, true
|
||||
}
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
snapshot "github.com/novox/mesh-controller/internal/facts"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// Issue 969: the gate composes a machine as the mesh does, so a moved module is handed where the machine takes
|
||||
// handed files and composed the old way where it does not.
|
||||
func TestIssue969TheGateGivesAMachineItsContractSoItTakesHandedFiles(t *testing.T) {
|
||||
taken := time.Date(2026, 10, 16, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
stated, ok := standInHealth(snapshot.Machine{Name: "f35", Contract: link.HandedContract}, taken)
|
||||
if !ok || !takesHanded(stated, ok) {
|
||||
t.Fatalf("a machine whose engine takes handed files must be given that in the gate: %+v, %v", stated, ok)
|
||||
}
|
||||
old, ok := standInHealth(snapshot.Machine{Name: "e12", Contract: link.HandedContract - 1}, taken)
|
||||
if !ok || takesHanded(old, ok) {
|
||||
t.Fatalf("a machine whose engine is older must not take handed files: %+v, %v", old, ok)
|
||||
}
|
||||
none, ok := standInHealth(snapshot.Machine{Name: "d01"}, taken)
|
||||
if ok || takesHanded(none, ok) {
|
||||
t.Fatalf("a snapshot that carries no contract states nothing: %+v, %v", none, ok)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 969, through the store: a machine whose engine stated the handed contract is in the snapshot with it, the
|
||||
// gate raises it as taking handed files, and one whose statement is below it is raised as not taking them. Run in
|
||||
// the merge check with the other tests that need a store; the composition itself, handed here and refused there, is
|
||||
// the catalogue's TestAHandedModuleNamingARunTimeBoundKeyComposesOnlyWhereItIsHanded.
|
||||
func TestIssue969TheGateRaisesEachMachineAsItsEngineStatedItself(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
said := time.Date(2026, 10, 16, 12, 0, 0, 0, time.UTC)
|
||||
for node, contract := range map[string]int{"anchor": link.HandedContract, "laptop": link.HandedContract - 1} {
|
||||
if _, err := open.inventory.RecordHealth(ctx, inventory.NodeHealth{Node: node, Contract: contract,
|
||||
SaidAt: said, HeardAt: said}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
f, err := gatherFacts(ctx, open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(f.Machines) != 2 {
|
||||
t.Fatalf("the snapshot holds %d machines", len(f.Machines))
|
||||
}
|
||||
want := map[string]bool{}
|
||||
for _, m := range f.Machines {
|
||||
if m.Contract == 0 {
|
||||
t.Fatalf("%s: the snapshot carries no contract", m.Name)
|
||||
}
|
||||
want[m.Name] = m.Contract >= link.HandedContract
|
||||
}
|
||||
handing := 0
|
||||
for _, hands := range want {
|
||||
if hands {
|
||||
handing++
|
||||
}
|
||||
}
|
||||
if handing != 1 {
|
||||
t.Fatalf("one machine takes handed files in the snapshot, got %v", want)
|
||||
}
|
||||
|
||||
raised, drop, err := throwawayStores(ctx, os.Getenv("MESH_TEST_POSTGRES"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer drop()
|
||||
if _, err := raiseFromFacts(ctx, raised, f, map[string]catalogue.Manifest{}, map[string]inventory.Source{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, hands := range want {
|
||||
got, err := engineTakesHanded(ctx, raised.inventory, name)
|
||||
if err != nil || got != hands {
|
||||
t.Errorf("%s: the gate raises it as taking handed files = %v (%v), the mesh says %v", name, got, err, hands)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -434,3 +434,236 @@ func sdkPinnedByGoMod(t *testing.T) string {
|
||||
t.Fatalf("go.mod requires no %s", sdkModule)
|
||||
return ""
|
||||
}
|
||||
|
||||
// **A module one head of a group removes and another defines is a move** (novox/hq ADR 0420 §6, issue 1025):
|
||||
// neither a second definition of one module name (issue 239) nor the removal of a module machines run
|
||||
// (ADR 0236). Alone, each head is still refused. This lays the heads over the shelf as judgeChange does,
|
||||
// which needs no database.
|
||||
func TestAGroupsMoveOfAModuleIsNeitherTwoDefinitionsNorARemoval(t *testing.T) {
|
||||
const catalogueRepo, controllerRepo = "novox/mesh-catalog", "novox/mesh-controller"
|
||||
built := func(repo, path string) inventory.Source {
|
||||
return inventory.Source{Repository: "http://forge.internal:20000/" + repo + ".git", Path: path, BuiltFrom: "c0ffee"}
|
||||
}
|
||||
sources := map[string]inventory.Source{"mover": built(catalogueRepo, "modules/mover")}
|
||||
raw := `{"module":"mover","version":"1"}`
|
||||
base := map[string]catalogue.Manifest{}
|
||||
m, err := catalogue.ParseManifest([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
base["mover"] = m
|
||||
facts := snapshot.Facts{Machines: []snapshot.Machine{{Name: "anchor", Assigned: []string{"mover"}}}}
|
||||
|
||||
catalogueWithout := aTree(t, map[string]string{"other": `{"module":"other","version":"1"}`})
|
||||
controllerWith := func() string {
|
||||
dir := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(dir, "modules", "mover"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "modules", "mover", "module.json"), []byte(raw), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return dir
|
||||
}()
|
||||
removes := groupChange{Repository: catalogueRepo, Tree: catalogueWithout, Changed: []string{"modules/mover/module.json"}}
|
||||
defines := groupChange{Repository: controllerRepo, Tree: controllerWith, Changed: []string{"modules/mover/module.json"}}
|
||||
|
||||
lay := func(heads ...groupChange) (map[string]catalogue.Manifest, []string, map[string]string) {
|
||||
t.Helper()
|
||||
moves, err := movesInGroup(heads, sources)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
at := sources
|
||||
if len(moves) > 0 {
|
||||
at = map[string]inventory.Source{}
|
||||
for k, v := range sources {
|
||||
at[k] = v
|
||||
}
|
||||
for name, mv := range moves {
|
||||
o := at[name]
|
||||
o.Repository, o.Path = mv.To, mv.Dir
|
||||
at[name] = o
|
||||
}
|
||||
}
|
||||
shelf, said := base, map[string]string{}
|
||||
var failures []string
|
||||
for _, h := range heads {
|
||||
var f []string
|
||||
if shelf, f, err = shelfWithChange(shelf, at, facts, h.Repository, h.Tree, said, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failures = append(failures, f...)
|
||||
}
|
||||
for name := range moves {
|
||||
said[name] = "moved"
|
||||
}
|
||||
return shelf, failures, said
|
||||
}
|
||||
for _, order := range [][]groupChange{{removes, defines}, {defines, removes}} {
|
||||
shelf, failures, said := lay(order...)
|
||||
if len(failures) > 0 {
|
||||
t.Fatalf("a move of a module machines run, laid %s first, was refused: %v", order[0].Repository, failures)
|
||||
}
|
||||
if said["mover"] != "moved" {
|
||||
t.Errorf("the move reads %v, not moved", said)
|
||||
}
|
||||
if _, held := shelf["mover"]; !held {
|
||||
t.Errorf("the moved module is not on the shelf after laying %s first", order[0].Repository)
|
||||
}
|
||||
}
|
||||
// Alone, a head is still refused: a second definition (issue 239), and a removal of what runs (ADR 0236).
|
||||
if _, failures, _ := lay(defines); len(failures) != 1 || !strings.Contains(failures[0], "issue 239") {
|
||||
t.Errorf("a definition of a module another repository builds, alone: %v", failures)
|
||||
}
|
||||
if _, failures, _ := lay(removes); len(failures) != 1 || !strings.Contains(failures[0], "ADR 0236") {
|
||||
t.Errorf("the removal of a module machines run, alone: %v", failures)
|
||||
}
|
||||
}
|
||||
|
||||
// A group that moves a module machines run passes the gate whole, in either order of its heads, and says the
|
||||
// module moved (novox/hq ADR 0420 §6, issue 1025); a head alone is still refused. Needs the test store.
|
||||
func TestADeliveryGroupMovingAModuleMachinesRunPassesTheGate(t *testing.T) {
|
||||
f, manifests := catalogueMeshWithAnApp(t)
|
||||
catTree := aTree(t, withEdit(manifests, "album", ""))
|
||||
app := t.TempDir()
|
||||
// The app module is built from this repository's root, and stays there: only album moves.
|
||||
if err := os.WriteFile(filepath.Join(app, "module.json"), []byte(`{"module":"app","version":"1"}`), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(app, "modules", "album"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(app, "modules", "album", "module.json"), []byte(manifests["album"]), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
removes := groupChange{Repository: "novox/mesh-catalog", Tree: catTree, Changed: []string{"modules/album/module.json"}}
|
||||
defines := groupChange{Repository: "novox/app", Tree: app, Changed: []string{"modules/album/module.json"}}
|
||||
judge := func(first groupChange, rest ...groupChange) mergeVerdict {
|
||||
v, err := judgeChange(t.Context(), mergeCheckInput{facts: f, admin: os.Getenv("MESH_TEST_POSTGRES"),
|
||||
repository: first.Repository, tree: first.Tree, changed: first.Changed, group: rest})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
for _, v := range []mergeVerdict{judge(removes, defines), judge(defines, removes)} {
|
||||
if v.Verdict == "fail" || v.Modules["album"] != "moved" {
|
||||
t.Fatalf("a group moving a module machines run: %v\n%s", v.Modules, v.Report())
|
||||
}
|
||||
}
|
||||
if v := judge(defines); v.Verdict != "fail" {
|
||||
t.Errorf("a definition of a module another repository builds passed alone:\n%s", v.Report())
|
||||
}
|
||||
if v := judge(removes); v.Verdict != "fail" {
|
||||
t.Errorf("the removal of a module machines run passed alone:\n%s", v.Report())
|
||||
}
|
||||
}
|
||||
|
||||
// **A module lives whole in one repository** (novox/hq ADR 0420 §1, issue 1019): an artifact whose build
|
||||
// context names another repository is refused by the gate, and one naming its own passes it.
|
||||
func TestTheGateRefusesAnArtifactContextNamingAnotherRepository(t *testing.T) {
|
||||
withContext := func(repo string) string {
|
||||
return `{"module":"proxy","version":"1","build":{"artifacts":[{"name":"server","kind":"image","from":"Dockerfile",` +
|
||||
`"context":{"seat":"git","repository":"` + repo + `","ref":"main"}}]}}`
|
||||
}
|
||||
judge := func(repo string) []string {
|
||||
t.Helper()
|
||||
tree := aTree(t, map[string]string{"proxy": withContext(repo)})
|
||||
_, failures, err := shelfWithChange(map[string]catalogue.Manifest{}, map[string]inventory.Source{},
|
||||
snapshot.Facts{}, "novox/mesh-controller", tree, map[string]string{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return failures
|
||||
}
|
||||
const want = "proxy builds artifact server from novox/mesh-catalog: a module lives whole in one repository (novox/hq ADR 0420 §1)"
|
||||
failures := judge("novox/mesh-catalog")
|
||||
if len(failures) != 1 || !strings.Contains(failures[0], want) {
|
||||
t.Errorf("a context naming another repository: %v, want it to say %q", failures, want)
|
||||
}
|
||||
for _, f := range judge("novox/mesh-controller") {
|
||||
if strings.Contains(f, "ADR 0420 §1") {
|
||||
t.Errorf("a context naming the module's own repository was refused: %s", f)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **A member of a delivery group judged alone sees a move** through the core clones beside its check
|
||||
// (novox/hq ADR 0420 §6, issue 1043): the controller's own gate, with the catalogue beside having removed the
|
||||
// module, is not refused for a second definition (issue 239); the catalogue's own gate, with the controller
|
||||
// beside defining it, is not refused for a removal (ADR 0236). A clone beside at main finds no move.
|
||||
func TestAMemberJudgedAloneSeesAMoveThroughTheClonesBeside(t *testing.T) {
|
||||
const catalogueRepo, controllerRepo = "novox/mesh-catalog", "novox/mesh-controller"
|
||||
raw := `{"module":"mover","version":"1"}`
|
||||
sources := map[string]inventory.Source{"mover": {Repository: "http://forge.internal:20000/" + catalogueRepo + ".git",
|
||||
Path: "modules/mover", BuiltFrom: "c0ffee"}}
|
||||
// A tree is written into dir: defining mover, or holding only another module.
|
||||
write := func(dir string, defines bool) string {
|
||||
name, body := "other", `{"module":"other","version":"1"}`
|
||||
if defines {
|
||||
name, body = "mover", raw
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(dir, "modules", name), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "modules", name, "module.json"), []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return dir
|
||||
}
|
||||
defining := func() string { return write(t.TempDir(), true) }
|
||||
without := func() string { return write(t.TempDir(), false) }
|
||||
// beside lays out a MESH_CHECK_BESIDE directory holding the named repositories' trees, each defining mover or not.
|
||||
beside := func(trees map[string]bool) string {
|
||||
root := t.TempDir()
|
||||
for name, defines := range trees {
|
||||
write(filepath.Join(root, name), defines)
|
||||
}
|
||||
return root
|
||||
}
|
||||
moves := func(repository, tree, besideRoot string) map[string]moveOfModule {
|
||||
t.Helper()
|
||||
got, err := movesOfChange(mergeCheckInput{repository: repository, tree: tree, beside: besideRoot}, sources)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
// The controller's member: defines mover; the catalogue beside has removed it.
|
||||
if got := moves(controllerRepo, defining(), beside(map[string]bool{"mesh-catalog": false})); got["mover"].To != controllerRepo {
|
||||
t.Errorf("the controller's change defining mover, the catalogue beside without it: moves %v", got)
|
||||
}
|
||||
// The sources the gate then judges with: built from the controller, the definition is not a second one (issue 239);
|
||||
// unmoved, it still is.
|
||||
judge := func(at map[string]inventory.Source) []string {
|
||||
t.Helper()
|
||||
_, failures, err := shelfWithChange(map[string]catalogue.Manifest{}, at, snapshot.Facts{}, controllerRepo, defining(),
|
||||
map[string]string{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return failures
|
||||
}
|
||||
mv := moves(controllerRepo, defining(), beside(map[string]bool{"mesh-catalog": false}))["mover"]
|
||||
moved := map[string]inventory.Source{"mover": {Repository: mv.To, Path: mv.Dir, BuiltFrom: "c0ffee"}}
|
||||
if f := judge(moved); len(f) != 0 {
|
||||
t.Errorf("a definition of a module the catalogue beside removed was refused: %v", f)
|
||||
}
|
||||
if f := judge(sources); len(f) != 1 || !strings.Contains(f[0], "issue 239") {
|
||||
t.Errorf("with the catalogue beside at main, the definition: %v, want the issue 239 refusal", f)
|
||||
}
|
||||
if got := moves(controllerRepo, defining(), beside(map[string]bool{"mesh-catalog": true})); len(got) != 0 {
|
||||
t.Errorf("the catalogue beside at main still defines mover, yet moves %v", got)
|
||||
}
|
||||
// The catalogue's member: removes mover; the controller beside defines it.
|
||||
if got := moves(catalogueRepo, without(), beside(map[string]bool{"mesh-controller": true})); got["mover"].To != controllerRepo {
|
||||
t.Errorf("the catalogue's change removing mover, the controller beside defining it: moves %v", got)
|
||||
}
|
||||
if got := moves(catalogueRepo, without(), beside(map[string]bool{"mesh-controller": false})); len(got) != 0 {
|
||||
t.Errorf("the controller beside at main does not define mover, yet moves %v", got)
|
||||
}
|
||||
if got := moves(catalogueRepo, without(), ""); len(got) != 0 {
|
||||
t.Errorf("nothing beside, yet moves %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// The merge path whole, against a real bus and the store (issue 973 and issue 921, novox/hq ADR 0407 decisions 1
|
||||
// and 3; ADR 0391's proof): the forge's merged event is published on the subject the controller consumes, the
|
||||
// controller's server hands it over and the window keys are on the bus. Store-backed: skips without one.
|
||||
|
||||
// noListener hears no node: these tests send merges only.
|
||||
type noListener struct{ link.Listener }
|
||||
|
||||
// theMergePathOn serves the controller's consume loop and its merge window on a test bus, and returns a way to
|
||||
// publish the forge's merged event.
|
||||
func theMergePathOn(t *testing.T, open *stores) func(link.SourceMoved) {
|
||||
t.Helper()
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
server := link.ConnectNats(js, nil, noListener{})
|
||||
if err := server.Follows(following{open: open}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() { defer close(done); _ = server.Serve(ctx) }()
|
||||
if err := startMergeWindow(ctx, open, js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
stop()
|
||||
<-done
|
||||
setWindowClock(nil)
|
||||
windowLook.Store(nil)
|
||||
})
|
||||
return func(m link.SourceMoved) {
|
||||
t.Helper()
|
||||
body, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.Context().Publish(broker.ControllerFollows[3], body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func becomes(t *testing.T, within time.Duration, what string, is func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(within)
|
||||
for time.Now().Before(deadline) {
|
||||
if is() {
|
||||
return
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("%s did not happen within %s", what, within)
|
||||
}
|
||||
|
||||
func mergeKept(open *stores, repo, commit string) bool {
|
||||
_, known, err := open.inventory.MergeOf(context.Background(), repo, commit)
|
||||
return err == nil && known
|
||||
}
|
||||
|
||||
// Two merges of one repository heard while another controller holds the plans, and a rollout of that repository
|
||||
// open, are both recorded once the plans are free (handed over again by the bus, never dropped), join one batch,
|
||||
// and the next rollout carries both: no person released anything (issue 921).
|
||||
func TestMergesHeardWhileAnotherHolderHasThePlansAreRecordedAndCarried(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
was, wasPoll := inventory.HoldWaitFor, inventory.HoldPoll
|
||||
inventory.HoldWaitFor, inventory.HoldPoll = time.Second, 20*time.Millisecond
|
||||
t.Cleanup(func() { inventory.HoldWaitFor, inventory.HoldPoll = was, wasPoll })
|
||||
// A window of a second: a merge joining a batch opens its window again (ADR 0407 decision 1), so the batch the
|
||||
// two held merges make is cut when that window has closed and the rollout before it has ended, not before.
|
||||
settings := t.TempDir() + "/merge-window.json"
|
||||
if err := os.WriteFile(settings, []byte(`{"merge-window": "1s", "merge-window-at-most": "10m"}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv(mergeWindowFileVar, settings)
|
||||
publish := theMergePathOn(t, open)
|
||||
ctx := t.Context()
|
||||
|
||||
publish(repoMerge("one", "c1", time.Now()))
|
||||
becomes(t, 15*time.Second, "the first rollout opening", func() bool {
|
||||
ws, _ := walks(t, open)
|
||||
return len(ws) == 1 && ws[0].Open()
|
||||
})
|
||||
ws, _ := walks(t, open)
|
||||
first := ws[0]
|
||||
|
||||
release, err := open.inventory.HoldPlans(ctx, true) // the other controller works the plans
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
publish(repoMerge("one", "c2", time.Now()))
|
||||
publish(repoMerge("one", "c3", time.Now()))
|
||||
time.Sleep(2500 * time.Millisecond) // both met the hold and were given back to the bus
|
||||
if mergeKept(open, "novox/one", "c2") || mergeKept(open, "novox/one", "c3") {
|
||||
t.Fatal("a merge was recorded while another holder had the plans")
|
||||
}
|
||||
release()
|
||||
becomes(t, 30*time.Second, "both merges recorded after the hold", func() bool {
|
||||
return mergeKept(open, "novox/one", "c2") && mergeKept(open, "novox/one", "c3")
|
||||
})
|
||||
|
||||
first.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The rollout ended: the look that cuts what waited behind it, once the batch's window is closed too.
|
||||
var bs []inventory.Plan
|
||||
becomes(t, 15*time.Second, "the next rollout being cut with both merges", func() bool {
|
||||
advanceHeld(ctx, open)
|
||||
ws, bs = walks(t, open)
|
||||
return len(bs) == 0 && len(ws) == 2
|
||||
})
|
||||
if ws[0].CommitOf("novox/one") != "c3" || len(ws[0].Delivery.Merges) != 2 {
|
||||
t.Fatalf("the next rollout does not carry both merges: commit %q, merges %+v", ws[0].CommitOf("novox/one"),
|
||||
ws[0].Delivery.Merges)
|
||||
}
|
||||
}
|
||||
|
||||
// A batch's window closes by the expiry of its key on the bus, heard by the watcher, and the batch is cut with
|
||||
// no cutter and no call to cut (ADR 0407 decision 1): a window of a second, a merge that waits for it.
|
||||
func TestABatchWindowClosesByTheExpiryOfItsKey(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
everyHeadWaits(t)
|
||||
settings := t.TempDir() + "/merge-window.json"
|
||||
if err := os.WriteFile(settings, []byte(`{"merge-window": "1s", "merge-window-at-most": "10m"}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv(mergeWindowFileVar, settings)
|
||||
publish := theMergePathOn(t, open)
|
||||
|
||||
start := time.Now()
|
||||
publish(repoMerge("one", "c1", start))
|
||||
becomes(t, 10*time.Second, "the merge waiting in a batch", func() bool {
|
||||
_, bs := walks(t, open)
|
||||
return len(bs) == 1 && bs[0].State == inventory.PlanAssembling
|
||||
})
|
||||
if ws, _ := walks(t, open); len(ws) != 0 {
|
||||
t.Fatalf("the batch was cut before its window closed: %+v", ws)
|
||||
}
|
||||
becomes(t, 15*time.Second, "the batch being cut by its key's expiry", func() bool {
|
||||
ws, bs := walks(t, open)
|
||||
return len(ws) == 1 && len(bs) == 0
|
||||
})
|
||||
if took := time.Since(start); took < time.Second {
|
||||
t.Fatalf("the batch was cut after %s, before its window of a second was up", took)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The merge window's clock is on the bus (novox/hq ADR 0407 decision 1, issue 973): an open batch has a key
|
||||
// `<batch>.quiet` that lives the window and is made again by each merge into the batch, and a key `<batch>.most`
|
||||
// made once when the batch opens that lives the window's most. The first of the two to expire closes the batch,
|
||||
// and the watcher hears it: no timer polls the batches. The store still keeps the times (ClosesAt, AtMost) as the
|
||||
// record of when a window closed, which `plans` reads; the keys are what wakes the controller.
|
||||
|
||||
// windowClock is the bus's side of the merge window.
|
||||
type windowClock interface {
|
||||
// Arm makes a batch's keys: its quiet key again, to live quiet; its most key if it has none, to live most.
|
||||
Arm(ctx context.Context, batch string, quiet, most time.Duration) error
|
||||
// Forget takes a batch's keys away: it was cut, or closed, and nothing is to wake the controller for it.
|
||||
Forget(ctx context.Context, batch string)
|
||||
}
|
||||
|
||||
// windowLook is the look a window's expiry takes: the watcher's, and the fallback timer's.
|
||||
var windowLook atomic.Pointer[func(batch string)]
|
||||
|
||||
var (
|
||||
windowMu sync.RWMutex
|
||||
windowKeyed windowClock // nil in a process that is not the serving controller
|
||||
)
|
||||
|
||||
// setWindowClock makes clock the merge window's; nil takes it away.
|
||||
func setWindowClock(c windowClock) {
|
||||
windowMu.Lock()
|
||||
defer windowMu.Unlock()
|
||||
windowKeyed = c
|
||||
}
|
||||
|
||||
func windowClockOf() windowClock {
|
||||
windowMu.RLock()
|
||||
defer windowMu.RUnlock()
|
||||
return windowKeyed
|
||||
}
|
||||
|
||||
// windowDeadlines are how long a batch's two keys are to live from now: what is left of its window and of its
|
||||
// most, each at least a second (the least a key can live). Not armed for a batch whose window is closed already
|
||||
// (by a stop, by its time, or because it is cut at once) or that is no batch.
|
||||
func windowDeadlines(b inventory.Plan, now time.Time) (quiet, most time.Duration, armed bool) {
|
||||
if b.Delivery == nil || b.Delivery.Batch == nil || windowClosed(b, now) {
|
||||
return 0, 0, false
|
||||
}
|
||||
// Rounded up to a whole second, at least one: the server rounds a key's time to live down to whole seconds, so
|
||||
// a fraction would close the key before the record's time and the look would cut nothing.
|
||||
least := func(d time.Duration) time.Duration {
|
||||
if d < time.Second {
|
||||
return time.Second
|
||||
}
|
||||
return (d + time.Second - 1) / time.Second * time.Second
|
||||
}
|
||||
w := b.Delivery.Batch
|
||||
return least(w.ClosesAt.Sub(now)), least(w.AtMost.Sub(now)), true
|
||||
}
|
||||
|
||||
// armWindow puts a batch's clock on the bus after its record was kept. Where the bus will not take it the window
|
||||
// closes by a timer of this batch alone, once, as long as is left: never silently never.
|
||||
func armWindow(ctx context.Context, b inventory.Plan, now time.Time) {
|
||||
c := windowClockOf()
|
||||
quiet, most, armed := windowDeadlines(b, now)
|
||||
if c == nil || !armed {
|
||||
return
|
||||
}
|
||||
if err := c.Arm(ctx, b.ID, quiet, most); err != nil {
|
||||
fmt.Printf("batches: the window of %s could not be put on the bus (%v); it closes by a timer of its own\n", b.ID, err)
|
||||
if look := windowLook.Load(); look != nil {
|
||||
time.AfterFunc(quiet, func() { (*look)(b.ID) })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// busWindow is the clock on the controller's bucket.
|
||||
type busWindow struct {
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
|
||||
// newBusWindow opens the bucket and starts the watcher, which names each batch whose key expired to look until
|
||||
// the context ends. Only expiry wakes it: a key taken away by Forget or made again by Arm leaves a delete.
|
||||
func newBusWindow(ctx context.Context, api jetstream.JetStream, look func(batch string)) (*busWindow, error) {
|
||||
kv, err := api.KeyValue(ctx, broker.MergeWindowBucket)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the merge window's bucket: %w", err)
|
||||
}
|
||||
w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("watching the merge window: %w", err)
|
||||
}
|
||||
go func() {
|
||||
defer w.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if e == nil || e.Operation() != jetstream.KeyValuePurge {
|
||||
continue // a put, or a delete this controller made
|
||||
}
|
||||
if batch, _, found := strings.Cut(e.Key(), "."); found && look != nil {
|
||||
look(batch)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return &busWindow{kv: kv}, nil
|
||||
}
|
||||
|
||||
func (w *busWindow) Arm(ctx context.Context, batch string, quiet, most time.Duration) error {
|
||||
// The most is made once, when the batch opens: it is there already for every merge after the first.
|
||||
if _, err := w.kv.Create(ctx, batch+".most", []byte("most"), jetstream.KeyTTL(most)); err != nil &&
|
||||
!errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return err
|
||||
}
|
||||
// The quiet key is made again, not changed: a key's time to live is set when it is made.
|
||||
if err := w.kv.Delete(ctx, batch+".quiet"); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return err
|
||||
}
|
||||
if _, err := w.kv.Create(ctx, batch+".quiet", []byte("quiet"), jetstream.KeyTTL(quiet)); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *busWindow) Forget(ctx context.Context, batch string) {
|
||||
for _, key := range []string{batch + ".quiet", batch + ".most"} {
|
||||
_ = w.kv.Delete(ctx, key)
|
||||
}
|
||||
}
|
||||
|
||||
// lookAtBatches is the look an expired window asks for: the plans held, as a merge heard holds them, and the batches
|
||||
// whose window closed cut (cutBatchesHeld). Waiting for the plans rather than giving up: an expiry is heard once.
|
||||
func lookAtBatches(ctx context.Context, open *stores) {
|
||||
release, err := open.inventory.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
fmt.Printf("batches: a window closed and the plans could not be held for the look: %v\n", err)
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
if err := cutBatchesHeld(ctx, open, time.Now().UTC()); err != nil {
|
||||
fmt.Printf("batches: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
// startMergeWindow puts the merge window's clock on the bus for as long as the context lives: the controller is
|
||||
// the lease holder here, the only one that writes it. The watcher looks at the batches when a key expires; the
|
||||
// batches open now get their keys, and one look is taken, so a window that closed while no controller served is
|
||||
// cut (ADR 0407 decision 1). No timer polls them.
|
||||
func startMergeWindow(ctx context.Context, open *stores, js *broker.JetStream) error {
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
look := func(string) { lookAtBatches(ctx, open) }
|
||||
windowLook.Store(&look)
|
||||
clock, err := newBusWindow(ctx, api, look)
|
||||
if err != nil {
|
||||
windowLook.Store(nil)
|
||||
return err
|
||||
}
|
||||
setWindowClock(clock)
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
setWindowClock(nil)
|
||||
windowLook.Store(nil)
|
||||
}()
|
||||
release, err := open.inventory.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer release()
|
||||
batches, err := open.inventory.Batches(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
for _, b := range batches {
|
||||
armWindow(ctx, b, now)
|
||||
}
|
||||
return cutBatchesHeld(ctx, open, now)
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// windowOnABus is the merge window's clock on a real bus, and the batches its watcher heard expire.
|
||||
func windowOnABus(t *testing.T) (*busWindow, <-chan string) {
|
||||
t.Helper()
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conn, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(conn.Close)
|
||||
api, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
heard := make(chan string, 16)
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
t.Cleanup(stop)
|
||||
w, err := newBusWindow(ctx, api, func(batch string) { heard <- batch })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return w, heard
|
||||
}
|
||||
|
||||
func hears(t *testing.T, heard <-chan string, within time.Duration) (string, bool) {
|
||||
t.Helper()
|
||||
select {
|
||||
case b := <-heard:
|
||||
return b, true
|
||||
case <-time.After(within):
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
// A batch's quiet key expires when no merge came for the window, and the watcher names the batch (issue 973,
|
||||
// novox/hq ADR 0407 decision 1).
|
||||
func TestTheQuietKeyExpiresAndTheWatcherNamesTheBatch(t *testing.T) {
|
||||
w, heard := windowOnABus(t)
|
||||
if err := w.Arm(t.Context(), "plan-1", time.Second, time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b, ok := hears(t, heard, 10*time.Second); !ok || b != "plan-1" {
|
||||
t.Fatalf("heard %q (%v); want plan-1", b, ok)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge into the batch makes the quiet key again, so the window restarts: it does not expire at the first
|
||||
// deadline, and the batch's most closes it all the same.
|
||||
func TestAMergeRestartsTheQuietKeyAndTheMostKeyClosesAllTheSame(t *testing.T) {
|
||||
w, heard := windowOnABus(t)
|
||||
ctx := t.Context()
|
||||
if err := w.Arm(ctx, "plan-2", 2*time.Second, 5*time.Second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(1200 * time.Millisecond)
|
||||
if err := w.Arm(ctx, "plan-2", 2*time.Second, 5*time.Second); err != nil { // a merge: quiet again, most as it was
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b, ok := hears(t, heard, 1500*time.Millisecond); ok {
|
||||
t.Fatalf("heard %q before the restarted window was up", b)
|
||||
}
|
||||
// Another merge keeps the window open; the most (5 s from the first) closes it at about 5 s.
|
||||
start := time.Now()
|
||||
for time.Since(start) < 2500*time.Millisecond {
|
||||
if err := w.Arm(ctx, "plan-2", 2*time.Second, 5*time.Second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(700 * time.Millisecond)
|
||||
}
|
||||
if b, ok := hears(t, heard, 8*time.Second); !ok || b != "plan-2" {
|
||||
t.Fatalf("the most did not close the batch: heard %q (%v)", b, ok)
|
||||
}
|
||||
}
|
||||
|
||||
// A batch cut is forgotten: its keys are taken away by a delete, which is no expiry and wakes nobody.
|
||||
func TestForgettingABatchWakesNobody(t *testing.T) {
|
||||
w, heard := windowOnABus(t)
|
||||
if err := w.Arm(t.Context(), "plan-3", time.Second, time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.Forget(t.Context(), "plan-3")
|
||||
if b, ok := hears(t, heard, 3*time.Second); ok {
|
||||
t.Fatalf("a forgotten batch was heard expire: %q", b)
|
||||
}
|
||||
}
|
||||
|
||||
// The deadlines a batch's keys are made with: what is left of its window and of its most, at least a second;
|
||||
// none for a batch whose window is closed or that is no batch.
|
||||
func TestTheDeadlinesOfABatchsKeys(t *testing.T) {
|
||||
now := time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC)
|
||||
batch := func(closes, most time.Duration, closed bool) inventory.Plan {
|
||||
return inventory.Plan{State: inventory.PlanAssembling, Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{
|
||||
ClosesAt: now.Add(closes), AtMost: now.Add(most), Closed: closed}}}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
b inventory.Plan
|
||||
quiet, most time.Duration
|
||||
armed bool
|
||||
}{
|
||||
{"open", batch(90*time.Second, 10*time.Minute, false), 90 * time.Second, 10 * time.Minute, true},
|
||||
{"almost up", batch(100*time.Millisecond, 5*time.Second, false), time.Second, 5 * time.Second, true},
|
||||
{"closed by a stop", batch(90*time.Second, 10*time.Minute, true), 0, 0, false},
|
||||
{"up", batch(-time.Second, 10*time.Minute, false), 0, 0, false},
|
||||
{"cut at once", func() inventory.Plan {
|
||||
b := batch(90*time.Second, 10*time.Minute, false)
|
||||
b.Delivery.Batch.CutAtOnce = true
|
||||
return b
|
||||
}(), 0, 0, false},
|
||||
{"no batch", inventory.Plan{}, 0, 0, false},
|
||||
} {
|
||||
quiet, most, armed := windowDeadlines(c.b, now)
|
||||
if armed != c.armed || quiet != c.quiet || most != c.most {
|
||||
t.Errorf("%s: %v, %v, %v; want %v, %v, %v", c.what, quiet, most, armed, c.quiet, c.most, c.armed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A deadline that is not a whole number of seconds is rounded up, never down: the server rounds a key's time to live
|
||||
// down to whole seconds, so 89.999999 s would expire at 89 s, before the window is closed by the record's times, and
|
||||
// the look would cut nothing (review of issue 973).
|
||||
func TestAFractionalDeadlineIsRoundedUpToTheNextWholeSecond(t *testing.T) {
|
||||
now := time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC)
|
||||
b := inventory.Plan{State: inventory.PlanAssembling, Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{
|
||||
ClosesAt: now.Add(89*time.Second + 999999*time.Microsecond), AtMost: now.Add(599*time.Second + time.Nanosecond)}}}
|
||||
quiet, most, armed := windowDeadlines(b, now)
|
||||
if !armed || quiet != 90*time.Second || most != 600*time.Second {
|
||||
t.Errorf("deadlines %v, %v (armed %v); want 1m30s, 10m0s", quiet, most, armed)
|
||||
}
|
||||
}
|
||||
@@ -1,243 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A merge the bus announced and the controller never acted on is caught up** (novox/hq issue 266).
|
||||
//
|
||||
// The forge's poll announces every merge on the events stream, and the controller acts on what its
|
||||
// consumer there hands it. On 2026-10-06 one merge was on the stream and never handed over: the bus
|
||||
// server moved the consumer past it — a fault of consumers with several filters in the server the
|
||||
// mesh ran — and the controller, which only acts on what it is handed, said nothing. The modules
|
||||
// built from that repository stayed behind and were built by hand.
|
||||
//
|
||||
// So the stream is read back on a timer, on a consumer of its own filtered on merges alone, and every
|
||||
// announcement older than mergeGrace is judged as SourceMoved would judge it. **No record of what
|
||||
// was handled is kept, because none is needed**: acting on a merge marks every module it moved as
|
||||
// looked at since, so an announcement already acted on reads as history and moves nothing. One that
|
||||
// would still move something was never acted on — it is said, and acted on now.
|
||||
const (
|
||||
// mergeGrace is how long an announcement is left to the controller's own consumer before it is
|
||||
// judged missed. That consumer hands over one event at a time, and a merge waits behind a build
|
||||
// outcome that is being acted on; acting on a merge itself asks builds and does not wait for them.
|
||||
mergeGrace = 10 * time.Minute
|
||||
// mergeLookBack is how far back a pass reads. A merge missed longer ago than this was missed by a
|
||||
// controller that was not running this, and is the operator's to look at, not a surprise rebuild.
|
||||
mergeLookBack = 24 * time.Hour
|
||||
// mergeCatchUpEvery is how often the stream is read back.
|
||||
mergeCatchUpEvery = 5 * time.Minute
|
||||
)
|
||||
|
||||
// merges is what reads back the forge's announcements; the link server, or a test's list.
|
||||
type merges interface {
|
||||
AnnouncedMerges(ctx context.Context, since time.Time) ([]link.AnnouncedMerge, error)
|
||||
}
|
||||
|
||||
// unheardMerges is the announcements of merges the controller has not heard (novox/hq ADR 0276): a merge kept
|
||||
// in a batch is not acted on until its batch is cut, which can be past mergeGrace behind a long walk, and is
|
||||
// not missed for that.
|
||||
type unheardMerges struct {
|
||||
merges
|
||||
inv *inventory.Inventory
|
||||
}
|
||||
|
||||
func (u unheardMerges) AnnouncedMerges(ctx context.Context, since time.Time) ([]link.AnnouncedMerge, error) {
|
||||
all, err := u.merges.AnnouncedMerges(ctx, since)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []link.AnnouncedMerge
|
||||
for _, a := range all {
|
||||
_, kept, err := u.inv.MergeOf(ctx, a.Owner+"/"+a.Repo, a.Commit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !kept {
|
||||
out = append(out, a)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// owedToTheRecord says a merge read as history is still owed to the record (novox/hq ADR 0276): the merges
|
||||
// reader knows, when it keeps them (unheardMerges).
|
||||
func owedToTheRecord(ctx context.Context, announced merges, m link.SourceMoved, entries []inventory.Entry,
|
||||
read map[string][]inventory.ReadRepository) bool {
|
||||
u, ok := announced.(unheardMerges)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
merged, err := time.Parse(time.RFC3339Nano, m.MergedAt)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
raw := m
|
||||
raw.MergedAt = ""
|
||||
owed, err := owedLate(ctx, u.inv, m, merged, wouldMove(raw, entries, read))
|
||||
return err == nil && owed
|
||||
}
|
||||
|
||||
// catchingUpOnMerges reads back the forge's announcements on a timer, until the context ends.
|
||||
func catchingUpOnMerges(ctx context.Context, open *stores, announced merges) {
|
||||
f := following{open: open}
|
||||
catalogued := func(ctx context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
|
||||
entries, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
read, err := readForPlanning(ctx, open.inventory)
|
||||
return entries, read, err
|
||||
}
|
||||
failing := ""
|
||||
tick := time.NewTicker(mergeCatchUpEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
watchedMerges.begin()
|
||||
err := catchUpOnMerges(ctx, time.Now(), unheardMerges{announced, open.inventory}, catalogued, f.SourceMoved, func(format string, args ...any) {
|
||||
fmt.Printf(format+"\n", args...)
|
||||
})
|
||||
// What the pass found is what S5 says (novox/hq to-be 45 §3); a pass that could not read
|
||||
// says that instead, and leaves what the last one found standing.
|
||||
watchedMerges.end(time.Now(), err)
|
||||
// A pass that cannot read says so once, not every five minutes, and says when it reads again.
|
||||
why := ""
|
||||
if err != nil {
|
||||
why = err.Error()
|
||||
}
|
||||
if why != failing {
|
||||
if why != "" {
|
||||
fmt.Printf("merges the bus may not have handed over cannot be looked for: %s\n", why)
|
||||
} else {
|
||||
fmt.Println("merges the bus may not have handed over are looked for again")
|
||||
}
|
||||
failing = why
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// catchUpOnMerges is one pass: every announcement older than mergeGrace that acting on would still
|
||||
// move something is said and acted on, oldest first.
|
||||
//
|
||||
// Judged twice: once against the catalogue as the pass found it, and again just before acting,
|
||||
// because acting on an earlier missed merge of the same repository may have moved what a later one
|
||||
// would have.
|
||||
func catchUpOnMerges(ctx context.Context, now time.Time, announced merges,
|
||||
catalogued func(context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error),
|
||||
act func(context.Context, link.SourceMoved) error, say func(string, ...any)) error {
|
||||
|
||||
all, err := announced.AnnouncedMerges(ctx, now.Add(-mergeLookBack))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
entries, read, err := catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, a := range all {
|
||||
// A merge the forge said no time of is dated by its announcement, so a packaging module's look can
|
||||
// make it history once acted on, and the catch-up does not act on it again every pass (ADR 0267).
|
||||
if a.SourceMoved.MergedAt == "" && !a.At.IsZero() {
|
||||
a.SourceMoved.MergedAt = a.At.UTC().Format(time.RFC3339)
|
||||
}
|
||||
if now.Sub(a.At) < mergeGrace {
|
||||
continue
|
||||
}
|
||||
if len(wouldMove(a.SourceMoved, entries, read)) == 0 && !owedToTheRecord(ctx, announced, a.SourceMoved, entries, read) {
|
||||
continue
|
||||
}
|
||||
if entries, read, err = catalogued(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
moves := wouldMove(a.SourceMoved, entries, read)
|
||||
if len(moves) == 0 {
|
||||
if !owedToTheRecord(ctx, announced, a.SourceMoved, entries, read) {
|
||||
continue
|
||||
}
|
||||
raw := a.SourceMoved
|
||||
raw.MergedAt = ""
|
||||
moves = wouldMove(raw, entries, read)
|
||||
}
|
||||
var names []string
|
||||
for _, e := range moves {
|
||||
names = append(names, e.Manifest.Module)
|
||||
}
|
||||
watchedMerges.found(missedMerge{Owner: a.Owner, Repo: a.Repo, Base: a.Base, Commit: a.Commit,
|
||||
At: a.At, Modules: names})
|
||||
say("%s/%s merged into %s (%.8s), announced %s ago, and the controller never acted on it: the bus "+
|
||||
"did not hand the announcement over (novox/hq issue 266). %s %s behind it; acting on it now",
|
||||
a.Owner, a.Repo, a.Base, a.Commit, now.Sub(a.At).Round(time.Minute), readableList(names),
|
||||
isAre(len(names)))
|
||||
if err := act(ctx, a.SourceMoved); err != nil {
|
||||
say("%s/%s moved to %.8s and the mesh could not act on it: %v; the next pass tries again",
|
||||
a.Owner, a.Repo, a.Commit, err)
|
||||
continue
|
||||
}
|
||||
if entries, read, err = catalogued(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// missedMerge is one merge the bus announced and never handed over, as a pass found it.
|
||||
type missedMerge struct {
|
||||
Owner, Repo, Base, Commit string
|
||||
// At is when the bus took the announcement.
|
||||
At time.Time
|
||||
Modules []string
|
||||
}
|
||||
|
||||
// mergeWatch is what the passes found, for S5 (novox/hq to-be 45 §3): **a merge nothing read is
|
||||
// said**, urgent, even though the pass acts on it at once — the bus skipping a message is a fault of
|
||||
// the transport the mesh's every change rides on, and acting late is the repair, not the absence of
|
||||
// the fault. The next pass, finding it acted on, clears it.
|
||||
type mergeWatch struct {
|
||||
mu sync.Mutex
|
||||
passed time.Time
|
||||
err error
|
||||
finding []missedMerge
|
||||
missed []missedMerge
|
||||
}
|
||||
|
||||
var watchedMerges = &mergeWatch{}
|
||||
|
||||
func (w *mergeWatch) begin() {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.finding = nil
|
||||
}
|
||||
|
||||
func (w *mergeWatch) found(m missedMerge) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.finding = append(w.finding, m)
|
||||
}
|
||||
|
||||
func (w *mergeWatch) end(at time.Time, err error) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.passed, w.err = at, err
|
||||
if err == nil {
|
||||
w.missed = w.finding
|
||||
}
|
||||
}
|
||||
|
||||
// last is when the last pass ended, what the last pass that read found, and what the last pass
|
||||
// could not read.
|
||||
func (w *mergeWatch) last() (time.Time, []missedMerge, error) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.passed, append([]missedMerge(nil), w.missed...), w.err
|
||||
}
|
||||
@@ -1,180 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// announcedList is the events stream's merges as a test gives them.
|
||||
type announcedList []link.AnnouncedMerge
|
||||
|
||||
func (a announcedList) AnnouncedMerges(_ context.Context, since time.Time) ([]link.AnnouncedMerge, error) {
|
||||
var out []link.AnnouncedMerge
|
||||
for _, m := range a {
|
||||
if !m.At.Before(since) {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// aCatalogue is what the inventory holds, changed the way acting on a merge changes it: every module
|
||||
// the merge moved is marked as looked at (inventory.SourceMoved writes source_seen = now()).
|
||||
type aCatalogue struct {
|
||||
entries []inventory.Entry
|
||||
acted []string
|
||||
fail error
|
||||
}
|
||||
|
||||
func (c *aCatalogue) read(context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
|
||||
return append([]inventory.Entry(nil), c.entries...), nil, nil
|
||||
}
|
||||
|
||||
func (c *aCatalogue) act(now func() time.Time) func(context.Context, link.SourceMoved) error {
|
||||
return func(_ context.Context, m link.SourceMoved) error {
|
||||
if c.fail != nil {
|
||||
return c.fail
|
||||
}
|
||||
c.acted = append(c.acted, m.Repo+"@"+m.Commit[:8])
|
||||
for _, moved := range wouldMove(m, c.entries, nil) {
|
||||
for i := range c.entries {
|
||||
if c.entries[i].Manifest.Module == moved.Manifest.Module {
|
||||
c.entries[i].Source.Head = m.Commit
|
||||
c.entries[i].Source.Seen = now()
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func at(s string) time.Time {
|
||||
t, err := time.Parse(time.RFC3339, s)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
func announced(repo, commit, mergedAt, onTheBus string, paths ...string) link.AnnouncedMerge {
|
||||
return link.AnnouncedMerge{
|
||||
SourceMoved: link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Commit: commit,
|
||||
MergedAt: mergedAt, Paths: paths,
|
||||
CloneURL: "http://forge.internal:20000/novox/" + repo + ".git"},
|
||||
At: at(onTheBus),
|
||||
}
|
||||
}
|
||||
|
||||
func built(module, repo, path, commit, seen string) inventory.Entry {
|
||||
e := fromRepo(module, "http://forge.internal:20000/novox/"+repo+".git", path)
|
||||
e.Source.BuiltFrom, e.Source.Head, e.Source.Seen = commit, commit, at(seen)
|
||||
return e
|
||||
}
|
||||
|
||||
// **novox/hq issue 266, as it happened.** The forge announced a merge of the tools repository on the
|
||||
// events stream; the bus never handed it to the controller, which acted on the merges around it and
|
||||
// not on this one, and said nothing. Read back from the stream, it is the one merge that would still
|
||||
// move something — so it is said and acted on, once, and only after the controller's own consumer
|
||||
// has had its time with it.
|
||||
func TestAMergeTheBusNeverHandedOverIsActedOnLate(t *testing.T) {
|
||||
cat := &aCatalogue{entries: []inventory.Entry{
|
||||
built("mesh-tools", "mesh-tools", "", "8b789578aaaaaaaa", "2026-10-04T15:24:32Z"),
|
||||
built("node-tools", "mesh-tools", "node-tools", "8b789578aaaaaaaa", "2026-10-04T15:24:32Z"),
|
||||
// Acted on when it was announced: looked at after it was merged.
|
||||
built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-05T22:39:21Z"),
|
||||
}}
|
||||
stream := announcedList{
|
||||
// Nothing the mesh holds is built from the records repository.
|
||||
announced("hq", "88f7f79fcccccccc", "2026-10-05T22:43:00Z", "2026-10-05T22:43:04Z", "04-ISSUES/x.md"),
|
||||
// Acted on: its module was looked at since.
|
||||
announced("mesh-catalog", "78328d4adddddddd", "2026-10-05T22:39:00Z", "2026-10-05T22:39:21Z", "modules/gitea/x.ts"),
|
||||
// Never handed over.
|
||||
announced("mesh-tools", "9730bd89c3e48d0e", "2026-10-05T22:46:47Z", "2026-10-05T22:47:06Z",
|
||||
"node-tools/internal/console/console.go"),
|
||||
}
|
||||
var said []string
|
||||
say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }
|
||||
clock := at("2026-10-05T22:50:00Z")
|
||||
now := func() time.Time { return clock }
|
||||
pass := func() {
|
||||
t.Helper()
|
||||
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
pass()
|
||||
if len(cat.acted) != 0 {
|
||||
t.Fatalf("a merge three minutes old was taken from the controller's own consumer: %v", cat.acted)
|
||||
}
|
||||
|
||||
clock = at("2026-10-05T22:58:00Z")
|
||||
pass()
|
||||
if strings.Join(cat.acted, ",") != "mesh-tools@9730bd89" {
|
||||
t.Fatalf("acted on %v, wanted the one merge never handed over", cat.acted)
|
||||
}
|
||||
if len(said) != 1 || !strings.Contains(said[0], "novox/mesh-tools merged into main (9730bd89)") ||
|
||||
!strings.Contains(said[0], "mesh-tools and node-tools are behind it") {
|
||||
t.Fatalf("the missed merge was not said as one: %q", said)
|
||||
}
|
||||
|
||||
clock = at("2026-10-05T23:03:00Z")
|
||||
pass()
|
||||
if len(cat.acted) != 1 || len(said) != 1 {
|
||||
t.Fatalf("a merge acted on was acted on again: %v %q", cat.acted, said)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge that changed none of the held modules' files moves nothing, so it is never "missed"; one
|
||||
// that could not be acted on is said and tried again on the next pass.
|
||||
func TestAMissedMergeThatCouldNotBeActedOnIsTriedAgain(t *testing.T) {
|
||||
cat := &aCatalogue{
|
||||
entries: []inventory.Entry{built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-05T20:00:00Z")},
|
||||
fail: errors.New("the store is restarting"),
|
||||
}
|
||||
stream := announcedList{
|
||||
announced("mesh-catalog", "aaaaaaaa11111111", "2026-10-05T21:00:00Z", "2026-10-05T21:00:10Z",
|
||||
"modules/plex/module.json", "modules/plex/x.ts"),
|
||||
announced("mesh-catalog", "bbbbbbbb22222222", "2026-10-05T21:10:00Z", "2026-10-05T21:10:10Z", "modules/gitea/x.ts"),
|
||||
}
|
||||
var said []string
|
||||
say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }
|
||||
clock := at("2026-10-05T22:00:00Z")
|
||||
now := func() time.Time { return clock }
|
||||
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(said) != 2 || !strings.Contains(said[1], "could not act on it") {
|
||||
t.Fatalf("a failed catch-up was not said: %q", said)
|
||||
}
|
||||
cat.fail = nil
|
||||
clock = at("2026-10-05T22:05:00Z")
|
||||
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(cat.acted, ",") != "mesh-catalog@bbbbbbbb" {
|
||||
t.Fatalf("acted on %v, wanted only the merge that changed a held module", cat.acted)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge older than the look-back is left to the operator: a controller that did not run this
|
||||
// missed it, and acting on it days later would be a surprise rebuild.
|
||||
func TestAMergeOlderThanTheLookBackIsLeftAlone(t *testing.T) {
|
||||
cat := &aCatalogue{entries: []inventory.Entry{built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-01T00:00:00Z")}}
|
||||
stream := announcedList{announced("mesh-catalog", "cccccccc33333333", "2026-10-03T00:00:00Z", "2026-10-03T00:00:05Z", "modules/gitea/x.ts")}
|
||||
clock := at("2026-10-05T22:00:00Z")
|
||||
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(func() time.Time { return clock }),
|
||||
func(string, ...any) {}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(cat.acted) != 0 {
|
||||
t.Fatalf("a merge of three days ago was acted on: %v", cat.acted)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The operator's Approve of a delivery group registers a moved module from its new repository (novox/hq ADR 0420
|
||||
// §7, issue 1025). ADR 0266 §44 lets a build's outcome register a known module only from the repository the
|
||||
// catalogue already builds it from. The one exception was a build asked at the controller's terminal. This is a
|
||||
// second, for a move: mesh-delivery announces each merge it makes with the ask whose Approve it merged on; the
|
||||
// controller keeps the announcement, and when the build of a merge of a known module comes from another repository
|
||||
// it reads that ask's warrant from the router and accepts only the operator's Approve of the merged head. A verb
|
||||
// alone, or a merge with no such warrant, never moves a module.
|
||||
|
||||
// theDelivery is the module that merges, and whose asks these are.
|
||||
const theDelivery = "mesh-delivery"
|
||||
|
||||
// optionApprove is the option of mesh-delivery's ask that merges (its OptionApprove).
|
||||
const optionApprove = "approve"
|
||||
|
||||
// deliveryMerge is mesh-delivery's announcement of a merge it made: the pull request and the head it merged, the
|
||||
// group it belongs to, and the ask whose answer it merged on with the option chosen and the heads that answer was
|
||||
// bound to (delivery id → commit).
|
||||
type deliveryMerge struct {
|
||||
Owner string `json:"owner"`
|
||||
Repo string `json:"repo"`
|
||||
Number int `json:"number"`
|
||||
HeadSHA string `json:"head_sha"`
|
||||
Group string `json:"group,omitempty"`
|
||||
AskID string `json:"ask_id"`
|
||||
Option string `json:"option"`
|
||||
Heads map[string]string `json:"heads"`
|
||||
}
|
||||
|
||||
// parseDeliveryMerge reads an announcement, refusing one that names no pull request, head or ask.
|
||||
func parseDeliveryMerge(body []byte) (deliveryMerge, error) {
|
||||
var d deliveryMerge
|
||||
if err := json.Unmarshal(body, &d); err != nil {
|
||||
return d, fmt.Errorf("mesh-delivery's word on a merge cannot be read: %w", err)
|
||||
}
|
||||
if d.Owner == "" || d.Repo == "" || d.Number <= 0 || d.HeadSHA == "" || d.AskID == "" || d.Option == "" {
|
||||
return d, errors.New("mesh-delivery's word on a merge names no pull request, head, ask or option")
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// deliveryMergeKey is where a merge is kept: bus keys allow no `#` or `@`, so the pull request and its head are
|
||||
// path-like tokens, as the forge spells the repository.
|
||||
func deliveryMergeKey(owner, repo string, number int, head string) string {
|
||||
return fmt.Sprintf("%s/%s/%d/%s", owner, repo, number, head)
|
||||
}
|
||||
|
||||
// mergeAct is what mesh-delivery's answer binds, spelled exactly as mesh-delivery spells it (its approval.go:
|
||||
// mergeAct and headsString): this ask, the option and the heads it covers.
|
||||
func mergeAct(ask, option string, heads map[string]string) asks.Act {
|
||||
var out []string
|
||||
for id, sha := range heads {
|
||||
out = append(out, id+"="+sha)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return asks.Act{"verb": theDelivery + ".merge", "ask": ask, "option": option, "heads": strings.Join(out, ",")}
|
||||
}
|
||||
|
||||
// movedOnApprove judges the warrant of the ask a merge was made on: it holds only if it is the operator's Approve,
|
||||
// proved by the channel it came through, of this very ask as it was published (digest), bound to the heads the
|
||||
// announcement names, and those heads hold the head merged. Answers why it holds, or why not.
|
||||
func movedOnApprove(d deliveryMerge, ask *asks.Ask, w *asks.Warrant) (string, error) {
|
||||
switch {
|
||||
case ask == nil || w == nil:
|
||||
return "", errors.New("the router holds no closed record of the ask the merge was made on")
|
||||
case ask.Who != asks.Operator:
|
||||
return "", fmt.Errorf("the ask %s is for %q to answer, not the operator", ask.ID, ask.Who)
|
||||
case d.Option != optionApprove:
|
||||
return "", fmt.Errorf("the merge was made on the option %q, not an Approve", d.Option)
|
||||
}
|
||||
option, err := w.For(theDelivery, *ask)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if option.ID != optionApprove || w.Option != optionApprove || d.Option != option.ID {
|
||||
return "", fmt.Errorf("the operator chose %q on the ask %s, not an Approve", w.Option, w.Ask)
|
||||
}
|
||||
if w.By == nil || w.By.Who != asks.Operator || w.By.Verified == "" {
|
||||
return "", fmt.Errorf("the ask %s was not answered by the operator on a channel that proves who answered", w.Ask)
|
||||
}
|
||||
if err := option.Performs(mergeAct(d.AskID, option.ID, d.Heads)); err != nil {
|
||||
return "", err
|
||||
}
|
||||
named := false
|
||||
for _, sha := range d.Heads {
|
||||
named = named || sha == d.HeadSHA
|
||||
}
|
||||
if !named {
|
||||
return "", fmt.Errorf("the Approve covers %v, and the head merged, %.8s, is not among them", d.Heads, d.HeadSHA)
|
||||
}
|
||||
return w.Says() + " (ask " + w.Ask + ")", nil
|
||||
}
|
||||
|
||||
// ---- keeping what mesh-delivery announced ----------------------------------------------------------------------
|
||||
|
||||
// keepingDeliveryMerges keeps each merge mesh-delivery announces in the controller's bucket for them.
|
||||
type keepingDeliveryMerges struct {
|
||||
js func() *broker.JetStream
|
||||
}
|
||||
|
||||
func (k keepingDeliveryMerges) KeepDeliveryMerge(ctx context.Context, body []byte) error {
|
||||
d, err := parseDeliveryMerge(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
js := k.js()
|
||||
if js == nil {
|
||||
return fmt.Errorf("%w: the bus is not held", link.ErrTryAgain)
|
||||
}
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, broker.DeliveryMergesBucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: the bucket of mesh-delivery's merges: %w", link.ErrTryAgain, err)
|
||||
}
|
||||
if _, err := kv.Put(ctx, deliveryMergeKey(d.Owner, d.Repo, d.Number, d.HeadSHA), body); err != nil {
|
||||
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// keptDeliveryMerge is the merge mesh-delivery announced for a pull request's head; false when none was kept.
|
||||
// A variable so a test can hand it one.
|
||||
var keptDeliveryMerge = func(ctx context.Context, owner, repo string, number int, head string) (deliveryMerge, bool, error) {
|
||||
serving := servingBus.Load()
|
||||
if serving == nil {
|
||||
return deliveryMerge{}, false, errors.New("only the serving controller keeps what mesh-delivery announced")
|
||||
}
|
||||
api, err := jetstream.New(serving.Conn())
|
||||
if err != nil {
|
||||
return deliveryMerge{}, false, err
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, broker.DeliveryMergesBucket)
|
||||
if err != nil {
|
||||
return deliveryMerge{}, false, err
|
||||
}
|
||||
e, err := kv.Get(ctx, deliveryMergeKey(owner, repo, number, head))
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return deliveryMerge{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return deliveryMerge{}, false, err
|
||||
}
|
||||
d, err := parseDeliveryMerge(e.Value())
|
||||
return d, err == nil, err
|
||||
}
|
||||
|
||||
// routerAskOf is the router's record of one of mesh-delivery's asks: the ask as published and its warrant. A
|
||||
// variable so a test can hand it one.
|
||||
var routerAskOf = func(ctx context.Context, inv *inventory.Inventory, id string) (*asks.Ask, *asks.Warrant, error) {
|
||||
serving := servingBus.Load()
|
||||
if serving == nil {
|
||||
return nil, nil, errors.New("only the serving controller reads the router's record")
|
||||
}
|
||||
bucket, err := asksRecords(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if bucket == "" {
|
||||
return nil, nil, errors.New("no module declares the operator channel's records")
|
||||
}
|
||||
_, ask, w, err := readRouterEntry(ctx, serving.Conn(), bucket, theDelivery, id)
|
||||
return ask, w, err
|
||||
}
|
||||
|
||||
// readRouterEntry reads the router's record of one asker's ask: its state, the ask as published, and its warrant;
|
||||
// "" when there is none. The controller's grant reaches the JetStream API whole (`$JS.API.>`), so it reads any
|
||||
// asker's record.
|
||||
func readRouterEntry(ctx context.Context, conn *nats.Conn, bucket, asker, ask string) (string, *asks.Ask, *asks.Warrant, error) {
|
||||
reply, err := conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET.KV_"+bucket+".$KV."+bucket+"."+asker+"."+ask, nil)
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
if status := reply.Header.Get("Status"); status != "" {
|
||||
if status == "404" {
|
||||
return "", nil, nil, nil
|
||||
}
|
||||
return "", nil, nil, fmt.Errorf("the router's record could not be read: %s %s", status, reply.Header.Get("Description"))
|
||||
}
|
||||
var rec struct {
|
||||
State string `json:"state"`
|
||||
Ask *asks.Ask `json:"ask"`
|
||||
Warrant *asks.Warrant `json:"warrant"`
|
||||
}
|
||||
if err := json.Unmarshal(reply.Data, &rec); err != nil {
|
||||
return "", nil, nil, fmt.Errorf("the router's record of %s's ask %s cannot be read: %w", asker, ask, err)
|
||||
}
|
||||
return rec.State, rec.Ask, rec.Warrant, nil
|
||||
}
|
||||
|
||||
// errNotAMerge says a build was not asked for a merge, so there is no move to look for.
|
||||
var errNotAMerge = errors.New("the build is not of a merge")
|
||||
|
||||
// approvedMove says whether a build of a known module from another repository is the move the operator approved
|
||||
// (ADR 0420 §7): the build is of a merge, that merge is a pull request mesh-delivery announced it made, and the
|
||||
// ask it made it on holds an Approve by the operator of that head. Answers why it was accepted; an error says why
|
||||
// it was not, and the refusal of ADR 0266 stands.
|
||||
func approvedMove(ctx context.Context, inv *inventory.Inventory, forms *sourceForms, buildID string,
|
||||
built inventory.Source) (string, error) {
|
||||
r, found, err := inv.BuildRequestByID(ctx, buildID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !found || r.For != "merge" || r.Commit == "" {
|
||||
return "", errNotAMerge
|
||||
}
|
||||
owner, name, onForge := forms.onTheForge(built.Repository, built.Seat)
|
||||
if !onForge {
|
||||
return "", errors.New("the repository is not on the mesh's forge")
|
||||
}
|
||||
merged, known, err := inv.MergeOf(ctx, owner+"/"+name, r.Commit)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !known {
|
||||
return "", fmt.Errorf("the controller kept no merge of %s/%s at %.8s", owner, name, r.Commit)
|
||||
}
|
||||
k := announcedOf(merged)
|
||||
if k.Number <= 0 || k.HeadSHA == "" {
|
||||
return "", errors.New("the forge's announcement of the merge names no pull request or head")
|
||||
}
|
||||
d, kept, err := keptDeliveryMerge(ctx, owner, name, k.Number, k.HeadSHA)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !kept {
|
||||
return "", fmt.Errorf("mesh-delivery announced no merge of %s/%s#%d at %.8s", owner, name, k.Number, k.HeadSHA)
|
||||
}
|
||||
ask, w, err := routerAskOf(ctx, inv, d.AskID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
why, err := movedOnApprove(d, ask, w)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("%s/%s#%d merged at %.8s on %s", owner, name, k.Number, k.HeadSHA, why), nil
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// The move whole, against a real bus and the store (novox/hq ADR 0420 §7, issue 1025): mesh-delivery's word that it
|
||||
// merged a pull request is published on the subject the controller consumes and kept in its bucket; the router's
|
||||
// record of the ask is read back from the router's bucket by a direct get; and the build of a module from the
|
||||
// repository that merge was made in is registered from there only when that record holds the operator's Approve of
|
||||
// the merged head. Store-backed: skips without one.
|
||||
func TestAModuleBuiltFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTheBus(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
answer bool // the router holds the operator's Approve of the merged heads
|
||||
moved bool
|
||||
}{
|
||||
{what: "the router holds the operator's Approve", answer: true, moved: true},
|
||||
{what: "the router holds no record of the ask"},
|
||||
} {
|
||||
t.Run(c.what, func(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
// The module declaring the router's records, as the controller finds them.
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "operator-channel", Version: "1",
|
||||
DefinesSeats: []catalogue.SeatDeclaration{{Name: broker.AsksSeat, Records: []string{"asks"}}}},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/operator-channel", BuiltFrom: "c0ffee"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
records, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: broker.BucketName("operator-channel", "asks")})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
a := approvedMerge(t)
|
||||
a.merge.Owner, a.merge.Repo, a.merge.Number = "novox", "mesh-host", 7
|
||||
if c.answer {
|
||||
record, _ := json.Marshal(map[string]any{"state": "closed", "ask": a.ask, "warrant": a.warrant})
|
||||
if _, err := records.Put(ctx, theDelivery+"."+a.merge.AskID, record); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
server := link.ConnectNats(js, nil, noListener{})
|
||||
if err := server.KeepsDeliveryMerges(keepingDeliveryMerges{js: server.JetStream}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The events consumer exists, from now on, before the word is published: Receive would make it too late.
|
||||
if err := broker.AssertServingConsumers(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
serving, stop := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() { defer close(done); _ = server.Serve(serving) }()
|
||||
t.Cleanup(func() { stop(); <-done })
|
||||
servingBus.Store(js)
|
||||
t.Cleanup(func() { servingBus.Store(nil) })
|
||||
|
||||
// mesh-delivery says it merged the pull request, and the controller keeps what it said.
|
||||
body, _ := json.Marshal(a.merge)
|
||||
if _, err := js.Context().Publish(broker.DeliveryMergedSubject, body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
becomes(t, 15*time.Second, "the merge being kept", func() bool {
|
||||
_, kept, err := keptDeliveryMerge(ctx, "novox", "mesh-host", 7, a.merge.HeadSHA)
|
||||
return err == nil && kept
|
||||
})
|
||||
|
||||
// The forge's merge of that pull request, as the controller heard it, and the build it asked for.
|
||||
const commit = "c0ffee0123456789"
|
||||
event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-host", "base": "main", "number": 7,
|
||||
"head_sha": a.merge.HeadSHA, "merge_commit_sha": commit})
|
||||
if _, err := inv.AddMerge(ctx, inventory.BatchedMerge{Repository: "novox/mesh-host", Branch: "main", Commit: commit,
|
||||
Merged: time.Now(), Heard: time.Now(), Event: event}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id := fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano())
|
||||
if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: id, Repository: "novox/mesh-host", Seat: "git",
|
||||
Path: "modules/sudo", Ref: "main", Commit: commit, For: "merge"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
moved := onTrunk(id, "novox/mesh-host", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "2"})
|
||||
moved.Commit = commit
|
||||
_, _, err = takeIn(ctx, inv, moved)
|
||||
src, _ := inv.SourceOf(ctx, "sudo")
|
||||
if c.moved && (err != nil || src.Repository != "novox/mesh-host") {
|
||||
t.Fatalf("the Approved move was not registered from its new repository: %v (built from %q)", err, src.Repository)
|
||||
}
|
||||
if !c.moved && (!errors.Is(err, errNotItsSource) || src.Repository != "novox/mesh-catalog") {
|
||||
t.Fatalf("a move with no Approve was not refused as it is: %v (built from %q)", err, src.Repository)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
)
|
||||
|
||||
// A merge mesh-delivery made, with the ask whose answer it merged on: the heads that answer was bound to, the
|
||||
// ask as the router published it, and the warrant the operator's Approve left (novox/hq ADR 0420 §7, issue 1025).
|
||||
type anApprovedMerge struct {
|
||||
merge deliveryMerge
|
||||
ask asks.Ask
|
||||
warrant asks.Warrant
|
||||
}
|
||||
|
||||
func approvedMerge(t *testing.T) anApprovedMerge {
|
||||
t.Helper()
|
||||
heads := map[string]string{"novox/mesh-controller@aaaaaaaaaaaa": "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1",
|
||||
"novox/mesh-catalog@bbbbbbbbbbbb": "b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2"}
|
||||
m := deliveryMerge{Owner: "novox", Repo: "mesh-controller", Number: 7, HeadSHA: "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1",
|
||||
Group: "g1", AskID: "merge-g1", Option: optionApprove, Heads: heads}
|
||||
bind := func(option string, h map[string]string) string {
|
||||
d, err := asks.ActDigest(mergeAct(m.AskID, option, h))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return d
|
||||
}
|
||||
at := time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC)
|
||||
ask := asks.Ask{ID: m.AskID, Headline: "Merge the group", Explanation: "Two pull requests move a module.", Who: asks.Operator,
|
||||
Expires: at.Add(time.Hour), OnExpiry: "asked again",
|
||||
Options: []asks.Option{
|
||||
{ID: optionApprove, Label: "Approve", Does: "merges the group", Level: asks.Approve, Binds: bind(optionApprove, heads)},
|
||||
{ID: "decline", Label: "Decline", Does: "leaves it", Level: asks.Approve, Binds: bind("decline", heads)}}}
|
||||
w := asks.Warrant{Ask: m.AskID, Asker: theDelivery, Outcome: asks.OutcomeChosen, Option: optionApprove, Label: "Approve",
|
||||
Level: asks.Approve, By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "1", Verified: "user id"},
|
||||
At: at, AskDigest: ask.Digest()}
|
||||
return anApprovedMerge{merge: m, ask: ask, warrant: w}
|
||||
}
|
||||
|
||||
// The operator's Approve of the merged head holds, and every other answer does not: a Decline, a person who is not the
|
||||
// operator or not proved by their channel, a warrant for other heads or another ask than the router published, the
|
||||
// merged head outside the heads the Approve covered, and a router that holds nothing.
|
||||
func TestAMovesWarrantHoldsOnlyForTheOperatorsApproveOfTheMergedHead(t *testing.T) {
|
||||
ok := approvedMerge(t)
|
||||
if _, err := movedOnApprove(ok.merge, &ok.ask, &ok.warrant); err != nil {
|
||||
t.Fatalf("the operator's Approve of the merged heads was refused: %v", err)
|
||||
}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
change func(*anApprovedMerge)
|
||||
}{
|
||||
{"a Decline", func(a *anApprovedMerge) {
|
||||
a.warrant.Option, a.warrant.Label = "decline", "Decline"
|
||||
a.merge.Option = "decline"
|
||||
}},
|
||||
{"no person chose", func(a *anApprovedMerge) { a.warrant.Outcome = asks.OutcomeExpired }},
|
||||
{"a person who is not the operator", func(a *anApprovedMerge) { a.warrant.By.Who = "agent" }},
|
||||
{"a channel that proves nobody", func(a *anApprovedMerge) { a.warrant.By.Verified = "" }},
|
||||
{"other heads than the ask was bound to", func(a *anApprovedMerge) {
|
||||
a.merge.Heads = map[string]string{"novox/mesh-controller@aaaaaaaaaaaa": "c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3"}
|
||||
}},
|
||||
{"a head merged that the Approve did not cover", func(a *anApprovedMerge) { a.merge.HeadSHA = "d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4" }},
|
||||
{"another ask than the one the warrant answers", func(a *anApprovedMerge) { a.ask.Headline = "Something else" }},
|
||||
{"a warrant of another asker", func(a *anApprovedMerge) { a.warrant.Asker = "claude-code" }},
|
||||
{"an ask for no operator to answer", func(a *anApprovedMerge) { a.ask.Who = "agent" }},
|
||||
{"an Approve given after the ask expired", func(a *anApprovedMerge) { a.warrant.At = a.ask.Expires.Add(time.Minute) }},
|
||||
} {
|
||||
a := approvedMerge(t)
|
||||
c.change(&a)
|
||||
if why, err := movedOnApprove(a.merge, &a.ask, &a.warrant); err == nil {
|
||||
t.Errorf("%s was accepted: %s", c.what, why)
|
||||
}
|
||||
}
|
||||
if _, err := movedOnApprove(ok.merge, nil, &ok.warrant); err == nil {
|
||||
t.Error("a warrant with no ask to check it against was accepted")
|
||||
}
|
||||
if _, err := movedOnApprove(ok.merge, &ok.ask, nil); err == nil {
|
||||
t.Error("an ask with no warrant was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// mesh-delivery's word is read whole or refused, and kept under a key the bus allows.
|
||||
func TestADeliveryMergesWordIsReadWholeAndKeptUnderABusKey(t *testing.T) {
|
||||
body := `{"owner":"novox","repo":"mesh-controller","number":7,"head_sha":"abc","group":"g","ask_id":"merge-g",` +
|
||||
`"option":"approve","heads":{"novox/mesh-controller@abc":"abc"}}`
|
||||
d, err := parseDeliveryMerge([]byte(body))
|
||||
if err != nil || d.Number != 7 || d.Heads["novox/mesh-controller@abc"] != "abc" {
|
||||
t.Fatalf("read %+v, %v", d, err)
|
||||
}
|
||||
if key := deliveryMergeKey(d.Owner, d.Repo, d.Number, d.HeadSHA); key != "novox/mesh-controller/7/abc" ||
|
||||
strings.ContainsAny(key, "#@") {
|
||||
t.Errorf("key %q", key)
|
||||
}
|
||||
for _, bad := range []string{`{`, `{"owner":"novox","repo":"r","number":0,"head_sha":"a","ask_id":"x","option":"approve"}`,
|
||||
`{"owner":"novox","repo":"r","number":1,"head_sha":"a","option":"approve"}`} {
|
||||
if _, err := parseDeliveryMerge([]byte(bad)); err == nil {
|
||||
t.Errorf("%s was read", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/validate"
|
||||
)
|
||||
|
||||
// A directory or an archive that says how it reacts to files it does not know (novox/hq ADR 0422) is a
|
||||
// declaration the node-engine takes, so the gate, which judges every machine with the node-engine's own
|
||||
// validator, takes it too. Before the pin moved, every catalogue manifest using the words was refused
|
||||
// on every machine (issue 1024).
|
||||
func TestTheGateTakesADirectoryThatSaysHowItReacts(t *testing.T) {
|
||||
decl := fmt.Sprintf(`{"declaration":%d,"resources":[
|
||||
{"id":"xorg.config","type":"directory","path":"/home/a/.config/xorg","mode":"0755","owner":"a","unknown":"move-out","knows":["scripts/"]},
|
||||
{"id":"p10k.theme","type":"archive","source":"https://example.invalid/a.tar","digest":"sha256:`+
|
||||
"0000000000000000000000000000000000000000000000000000000000000000"+
|
||||
`","path":"/home/a/.local/share/powerlevel10k","unknown":"move-out"}]}`, validate.Version)
|
||||
if problems := validate.Declaration([]byte(decl)); len(problems) != 0 {
|
||||
t.Fatalf("refused: %q", problems)
|
||||
}
|
||||
}
|
||||
@@ -429,12 +429,6 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
||||
"out) has not let it start yet.",
|
||||
Resolved: "Resolved: the delivery is no longer waiting"}
|
||||
}),
|
||||
"merge-not-acted": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A merge was never picked up",
|
||||
Explanation: "A pull request was merged, and the controller never heard of it, so nothing was built or " +
|
||||
"sent. The mesh catches up on missed merges by itself.",
|
||||
Resolved: "Resolved: the merge was picked up"}
|
||||
}),
|
||||
"ask-lost": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A build has no result",
|
||||
Explanation: "A build was asked for and never answered. It is asked again with the next delivery.",
|
||||
|
||||
@@ -1126,7 +1126,13 @@ func engineTakesHanded(ctx context.Context, inv *inventory.Inventory, node strin
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return had && stated.Contract >= link.HandedContract, nil
|
||||
return takesHanded(stated, had), nil
|
||||
}
|
||||
|
||||
// takesHanded is whether a machine's newest statement (had: it ever stated one) is of an engine that writes a
|
||||
// filled file on the provisioner client's word alone.
|
||||
func takesHanded(stated inventory.NodeHealth, had bool) bool {
|
||||
return had && stated.Contract >= link.HandedContract
|
||||
}
|
||||
|
||||
// provisionerAccount is the account the provisioner's client runs as on this machine (novox/hq ADR 0349 §4): the user
|
||||
|
||||
@@ -0,0 +1,674 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A rollout advances on events, not on a timer (novox/hq ADR 0406 §5, issue 974). One look at the open plans runs on
|
||||
// each thing that can let a plan move: a plan saved (a build's `built`, a tier's send kept, a command's move), a
|
||||
// machine's report on a send, a silent key's expiry, and the controller taking the lease. The looks are asked for
|
||||
// with askToLook and run one at a time by a worker, so an ask made while a look runs is not lost and many asks are
|
||||
// one look. Only two things are still started by time, both named and both one-shot: a failed advance step is
|
||||
// looked at again (retryLook), and the end of a walk whose end is not known is recorded when it can be
|
||||
// (armWalkEnd).
|
||||
|
||||
// looker runs the look at the plans that events ask for.
|
||||
type looker struct {
|
||||
signal chan struct{} // one place: asks made together are one look
|
||||
look func(ctx context.Context) (failed bool)
|
||||
// gap is the least time between two looks, so a plan that kept changing could not make the worker spin.
|
||||
gap time.Duration
|
||||
// first runs before the first look: what a controller that took the lease sets up.
|
||||
first func(ctx context.Context)
|
||||
|
||||
mu sync.Mutex
|
||||
retry *time.Timer
|
||||
backoff time.Duration
|
||||
base time.Duration
|
||||
most time.Duration
|
||||
}
|
||||
|
||||
const (
|
||||
retryLookFirst = 5 * time.Second
|
||||
retryLookMost = time.Minute
|
||||
lookGap = 200 * time.Millisecond
|
||||
)
|
||||
|
||||
func newLooker(look func(ctx context.Context) bool) *looker {
|
||||
return &looker{signal: make(chan struct{}, 1), look: look, gap: lookGap, base: retryLookFirst, most: retryLookMost}
|
||||
}
|
||||
|
||||
// ask has a look run, soon; never blocks.
|
||||
func (l *looker) ask() {
|
||||
select {
|
||||
case l.signal <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// run takes the asks one at a time until the context ends.
|
||||
func (l *looker) run(ctx context.Context) {
|
||||
if l.first != nil {
|
||||
l.first(ctx)
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
l.mu.Lock()
|
||||
if l.retry != nil {
|
||||
l.retry.Stop()
|
||||
}
|
||||
l.mu.Unlock()
|
||||
return
|
||||
case <-l.signal:
|
||||
}
|
||||
l.after(l.look(ctx))
|
||||
if l.gap > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(l.gap):
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// after takes the outcome of a look: a failed one is looked at again later, a good one forgets the failures.
|
||||
func (l *looker) after(failed bool) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if !failed {
|
||||
l.backoff = 0
|
||||
if l.retry != nil {
|
||||
l.retry.Stop()
|
||||
l.retry = nil
|
||||
}
|
||||
return
|
||||
}
|
||||
if l.retry != nil {
|
||||
return // one is armed already
|
||||
}
|
||||
switch {
|
||||
case l.backoff == 0:
|
||||
l.backoff = l.base
|
||||
case l.backoff*2 > l.most:
|
||||
l.backoff = l.most
|
||||
default:
|
||||
l.backoff *= 2
|
||||
}
|
||||
// retries: an advance step that failed (store, bus or send). One at a time, five seconds doubling to a minute, forgotten by
|
||||
// the first look that fails nothing; the step is otherwise started by an event, and a failure is none.
|
||||
l.retry = time.AfterFunc(l.backoff, func() {
|
||||
l.mu.Lock()
|
||||
l.retry = nil
|
||||
l.mu.Unlock()
|
||||
l.ask()
|
||||
})
|
||||
}
|
||||
|
||||
// theLooker is the serving controller's; nil in a command, whose looks are its own.
|
||||
var (
|
||||
lookerMu sync.RWMutex
|
||||
theLooker *looker
|
||||
)
|
||||
|
||||
// askToLook has the serving controller look at the open plans soon: a plan was saved, a machine reported on a send,
|
||||
// a silent key expired. Nothing in a process that is not serving.
|
||||
func askToLook() {
|
||||
lookerMu.RLock()
|
||||
l := theLooker
|
||||
lookerMu.RUnlock()
|
||||
if l != nil {
|
||||
l.ask()
|
||||
}
|
||||
}
|
||||
|
||||
// startPlanLooks starts the look at the open plans, and asks for the first: the controller took the lease, and a
|
||||
// plan left by a replaced controller resumes from there (ADR 0406 §6). It replaces the rollout's 30 second ticker.
|
||||
func startPlanLooks(ctx context.Context, open *stores) {
|
||||
l := newLooker(func(ctx context.Context) bool { return lookAtPlans(ctx, open) })
|
||||
l.first = func(ctx context.Context) { rearmWalkEnds(ctx, open) }
|
||||
lookerMu.Lock()
|
||||
theLooker = l
|
||||
lookerMu.Unlock()
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
lookerMu.Lock()
|
||||
if theLooker == l {
|
||||
theLooker = nil
|
||||
}
|
||||
lookerMu.Unlock()
|
||||
}()
|
||||
go l.run(ctx)
|
||||
l.ask()
|
||||
}
|
||||
|
||||
// applySilentAfter is how long a machine may say nothing on a send before the plan names it silent: the mesh's
|
||||
// ApplySilentAfter, a variable so a test of the wait needs no quarter of an hour.
|
||||
var applySilentAfter = inventory.ApplySilentAfter
|
||||
|
||||
// startRolloutEvents starts what a rollout advances on, for as long as the context lives (novox/hq ADR 0406 §5, issue 974):
|
||||
// the wait for each machine's report as a key on the bus that expires (where the bus will not take it a timer of its own
|
||||
// for that wait looks instead), a provisioner client's words, and the look at the plans, asked for first so a plan left
|
||||
// by a replaced controller resumes. The serving controller's start and the tests of a rollout call this one.
|
||||
func startRolloutEvents(ctx context.Context, open *stores, js *broker.JetStream) {
|
||||
if err := startRolloutSilent(ctx, js); err != nil {
|
||||
fmt.Printf("plans: the wait for the machines' reports could not be put on the bus (%v); each is looked at by a timer of its own\n", err)
|
||||
}
|
||||
if err := startProvAppliedLooks(ctx, js); err != nil {
|
||||
fmt.Printf("plans: the provisioner clients' words could not be watched (%v); a walk waits for them until the next look\n", err)
|
||||
}
|
||||
if err := startPlanMovedLooks(ctx, js); err != nil {
|
||||
fmt.Printf("plans: the plans moved by other processes could not be followed (%v)\n", err)
|
||||
}
|
||||
startPlanLooks(ctx, open)
|
||||
}
|
||||
|
||||
// servingPlanSaved is what the serving controller does for each plan it keeps: said as plan-moved, looked at (a build, a
|
||||
// send kept or a command's move is followed by the look that asks the next tier, in place of the timer that did), and,
|
||||
// for a walk that ended, its merges' check builds forgotten (novox/hq ADR 0350 §5).
|
||||
func servingPlanSaved(ctx context.Context, bus link.Bus, open *stores) func(*inventory.Inventory, inventory.Plan) {
|
||||
return func(_ *inventory.Inventory, p inventory.Plan) {
|
||||
sayPlanMoved(ctx, bus, open.inventory, p)
|
||||
planKept(p)
|
||||
if p.WalkSaid == inventory.WalkEnded {
|
||||
go forgetChecksOfWalk(context.WithoutCancel(ctx), open.inventory, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// planLookPasses is the most times one look goes over the plans again because the first pass changed which are open.
|
||||
const planLookPasses = 6
|
||||
|
||||
// lookAtPlans moves every open plan as far as the facts allow, as the ticker did each pass: what waits for a gate is
|
||||
// released, the plans advanced, the wait for each machine's report kept as a key on the bus, and the phases of the
|
||||
// walks that ended recorded. It says whether a step failed. Waits for the plans rather than giving up: an ask is
|
||||
// answered once.
|
||||
func lookAtPlans(ctx context.Context, open *stores) (failed bool) {
|
||||
release, err := open.inventory.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
fmt.Printf("plans: cannot hold them: %v\n", err)
|
||||
return true
|
||||
}
|
||||
func() {
|
||||
defer release()
|
||||
// What waits for a gate, released one machine first, then the rest at once (ADR 0236, ADR 0353).
|
||||
if _, err := releaseBacklog(ctx, open, ""); err != nil {
|
||||
fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err)
|
||||
failed = true
|
||||
}
|
||||
// Again while the plans that are open changed: a plan that ended lets a deferred one before it start, and a
|
||||
// batch cut makes one.
|
||||
for pass := 0; pass < planLookPasses; pass++ {
|
||||
before := openPlansNow(ctx, open.inventory)
|
||||
if advanceHeld(ctx, open) {
|
||||
failed = true
|
||||
}
|
||||
if openPlansNow(ctx, open.inventory) == before {
|
||||
break
|
||||
}
|
||||
}
|
||||
if err := syncSilentKeys(ctx, open); err != nil {
|
||||
fmt.Printf("plans: the wait for the machines' reports could not be kept on the bus: %v\n", err)
|
||||
failed = true // so the look is retried
|
||||
}
|
||||
}()
|
||||
keepWalkPhases(ctx, open)
|
||||
return failed
|
||||
}
|
||||
|
||||
// advancePlans is one pass over the open plans taking the plans without waiting for them, for a caller that wants a
|
||||
// single step rather than the look (the tests of the steps).
|
||||
func advancePlans(ctx context.Context, open *stores) {
|
||||
release, err := open.inventory.HoldPlans(ctx, false)
|
||||
if err != nil {
|
||||
if !errors.Is(err, inventory.ErrPlansBusy) {
|
||||
fmt.Printf("plans: cannot hold them: %v\n", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
if _, err := releaseBacklog(ctx, open, ""); err != nil {
|
||||
fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err)
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
}
|
||||
|
||||
// openPlansNow is which plans are open and in what state, for telling a pass that changed them from one that did not.
|
||||
func openPlansNow(ctx context.Context, inv *inventory.Inventory) string {
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return "unreadable " + time.Now().String()
|
||||
}
|
||||
ids := make([]string, 0, len(plans))
|
||||
for _, p := range plans {
|
||||
ids = append(ids, p.ID+":"+string(p.State))
|
||||
}
|
||||
sort.Strings(ids)
|
||||
return strings.Join(ids, ",")
|
||||
}
|
||||
|
||||
// --- the wait for a machine's report, as keys on the bus ---
|
||||
|
||||
var (
|
||||
silentMu sync.RWMutex
|
||||
silentKeyed silentClock // nil in a process that is not the serving controller
|
||||
)
|
||||
|
||||
// setSilentClock makes clock the rollout's; nil takes it away.
|
||||
func setSilentClock(c silentClock) {
|
||||
silentMu.Lock()
|
||||
defer silentMu.Unlock()
|
||||
silentKeyed = c
|
||||
}
|
||||
|
||||
func silentClockOf() silentClock {
|
||||
silentMu.RLock()
|
||||
defer silentMu.RUnlock()
|
||||
return silentKeyed
|
||||
}
|
||||
|
||||
// silentWaitsOf is every machine an open plan waits on for a report, with when it was sent: the machines a walk sent
|
||||
// that have not reported on the send and are not silent yet (walkReports), the machine a gate of a first machine
|
||||
// judges, and the machines a release's gates wait on. Pure.
|
||||
func silentWaitsOf(p inventory.Plan, reports []inventory.Reported, now time.Time, prov inventory.ProvLookup) map[silentWait]time.Time {
|
||||
waits := map[silentWait]time.Time{}
|
||||
put := func(w silentWait, at time.Time) {
|
||||
if at.IsZero() {
|
||||
return
|
||||
}
|
||||
if was, ok := waits[w]; !ok || at.Before(was) {
|
||||
waits[w] = at
|
||||
}
|
||||
}
|
||||
if !p.Open() {
|
||||
return waits
|
||||
}
|
||||
if r := p.Release; r != nil {
|
||||
gates := []*inventory.PlanGate{r.Gate}
|
||||
for _, g := range r.Gates {
|
||||
gates = append(gates, g)
|
||||
}
|
||||
for _, g := range gates {
|
||||
if g == nil || g.Verdict != "" || g.Silent || g.Pending || g.Since == nil {
|
||||
continue
|
||||
}
|
||||
for _, node := range g.Machines {
|
||||
put(silentWait{Plan: p.ID, Tier: 0, Node: node}, *g.Since)
|
||||
}
|
||||
}
|
||||
return waits
|
||||
}
|
||||
if p.Waiting() {
|
||||
return waits
|
||||
}
|
||||
tierOf := func(module string) int {
|
||||
for i, tier := range p.Tiers {
|
||||
if slices.Contains(tier, module) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
waiting, _, _ := walkReports(p, reports, now, prov)
|
||||
for module, s := range p.Modules {
|
||||
if s == nil {
|
||||
continue
|
||||
}
|
||||
for _, node := range waiting {
|
||||
sent, ok := s.Rest[node]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
at := sent.At
|
||||
if at.IsZero() && s.SentAt != nil {
|
||||
at = *s.SentAt
|
||||
}
|
||||
put(silentWait{Plan: p.ID, Tier: tierOf(module), Node: node}, at)
|
||||
}
|
||||
if g := s.Gate; g != nil && g.Verdict == "" && !g.Silent && g.Since != nil && s.FirstAt != nil {
|
||||
for _, node := range g.Machines {
|
||||
put(silentWait{Plan: p.ID, Tier: tierOf(module), Node: node}, *g.Since)
|
||||
}
|
||||
}
|
||||
}
|
||||
return waits
|
||||
}
|
||||
|
||||
// silentTTL is how long a wait's key lives: what is left of ApplySilentAfter from the send, and a second over, so the
|
||||
// look it wakes finds the machine silent and not a fraction early.
|
||||
func silentTTL(sent, now time.Time) time.Duration {
|
||||
left := sent.Add(applySilentAfter).Sub(now) + time.Second
|
||||
if left < time.Second {
|
||||
return time.Second
|
||||
}
|
||||
return left
|
||||
}
|
||||
|
||||
// silentDiff is what the keys on the bus need to be as the waits are: the waits to put a key for (none yet, or the
|
||||
// key is of an earlier send) and the keys to delete (the wait is over). Pure.
|
||||
func silentDiff(want, held map[silentWait]time.Time) (arm, drop []silentWait) {
|
||||
for w, sent := range want {
|
||||
if was, ok := held[w]; !ok || !was.Equal(sent) {
|
||||
arm = append(arm, w)
|
||||
}
|
||||
}
|
||||
for w := range held {
|
||||
if _, ok := want[w]; !ok {
|
||||
drop = append(drop, w)
|
||||
}
|
||||
}
|
||||
return arm, drop
|
||||
}
|
||||
|
||||
// syncSilentKeys keeps the keys on the bus as the open plans' waits are: a wait without a key gets one that lives out
|
||||
// its time, a send made again gets its key again, and a key whose wait is over — the machine reported on the send, the
|
||||
// gate has its verdict, the plan ended — is deleted. A report deletes the key by way of the look it asks for (the
|
||||
// look alone knows the report was on that send). Only the lease holder looks, so only it writes.
|
||||
func syncSilentKeys(ctx context.Context, open *stores) error {
|
||||
c := silentClockOf()
|
||||
inv := open.inventory
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
want := map[silentWait]time.Time{}
|
||||
for _, p := range plans {
|
||||
for w, at := range silentWaitsOf(p, reports, now, provLookup(ctx)) {
|
||||
want[w] = at
|
||||
}
|
||||
}
|
||||
return keepSilentKeys(ctx, c, want, now)
|
||||
}
|
||||
|
||||
// keepSilentKeys puts the keys on the bus as the waits are. Where the bus will not take a key — no bucket, the keys
|
||||
// cannot be read, one cannot be put — the wait is looked at by a timer of its own, once, and the error is given back so
|
||||
// the look is retried.
|
||||
func keepSilentKeys(ctx context.Context, c silentClock, want map[silentWait]time.Time, now time.Time) error {
|
||||
byTimer := func() {
|
||||
for w, sent := range want {
|
||||
silentByTimer(w, sent, silentTTL(sent, now))
|
||||
}
|
||||
}
|
||||
if c == nil {
|
||||
byTimer()
|
||||
return nil
|
||||
}
|
||||
held, err := c.Held(ctx)
|
||||
if err != nil {
|
||||
byTimer()
|
||||
return err
|
||||
}
|
||||
arm, drop := silentDiff(want, held)
|
||||
var first error
|
||||
for _, w := range arm {
|
||||
if err := c.Arm(ctx, w, silentTTL(want[w], now), want[w]); err != nil {
|
||||
silentByTimer(w, want[w], silentTTL(want[w], now))
|
||||
if first == nil {
|
||||
first = err
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, w := range drop {
|
||||
if err := c.Reported(ctx, w); err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
var silentTimers = struct {
|
||||
mu sync.Mutex
|
||||
timers map[silentWait]time.Time
|
||||
}{timers: map[silentWait]time.Time{}}
|
||||
|
||||
// silentByTimer looks at the plans when a wait's time is up, for a process whose bus will not take the key
|
||||
// (DegradableGrants says why that can be): never silently never. Once for each send.
|
||||
func silentByTimer(w silentWait, sent time.Time, ttl time.Duration) {
|
||||
silentTimers.mu.Lock()
|
||||
defer silentTimers.mu.Unlock()
|
||||
if at, ok := silentTimers.timers[w]; ok && at.Equal(sent) {
|
||||
return
|
||||
}
|
||||
silentTimers.timers[w] = sent
|
||||
// retries: a rollout's wait for a machine, where the bus would not take its key. One per send, once.
|
||||
time.AfterFunc(ttl, func() {
|
||||
silentTimers.mu.Lock()
|
||||
if silentTimers.timers[w].Equal(sent) {
|
||||
delete(silentTimers.timers, w)
|
||||
}
|
||||
silentTimers.mu.Unlock()
|
||||
askToLook()
|
||||
})
|
||||
}
|
||||
|
||||
// startRolloutSilent puts the wait for the machines' reports on the bus for as long as the context lives: the
|
||||
// controller is the lease holder here, the only one that writes it. The watcher asks for a look when a key expires.
|
||||
func startRolloutSilent(ctx context.Context, js *broker.JetStream) error {
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
clock, err := newBusSilent(ctx, api, func(silentWait) { askToLook() })
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
setSilentClock(clock)
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
silentMu.Lock()
|
||||
if silentKeyed == silentClock(clock) {
|
||||
silentKeyed = nil
|
||||
}
|
||||
silentMu.Unlock()
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// startProvAppliedLooks has the plans looked at when a machine's provisioner client says a word (`prov.applied.<node>.<module>`):
|
||||
// a send that handed modules to the provisioner is applied only once the client said so too, and without this the walk
|
||||
// would wait for its silent key (ADR 0362, ADR 0406 §5, issue 974). New words only: the first look reads the rest.
|
||||
func startProvAppliedLooks(ctx context.Context, js *broker.JetStream) error {
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stream, err := api.Stream(ctx, broker.ProvAppliedStream)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the provisioner clients' words: %w", err)
|
||||
}
|
||||
consumer, err := stream.OrderedConsumer(ctx, jetstream.OrderedConsumerConfig{DeliverPolicy: jetstream.DeliverNewPolicy})
|
||||
if err != nil {
|
||||
return fmt.Errorf("watching the provisioner clients' words: %w", err)
|
||||
}
|
||||
consuming, err := consumer.Consume(func(jetstream.Msg) { askToLook() })
|
||||
if err != nil {
|
||||
return fmt.Errorf("watching the provisioner clients' words: %w", err)
|
||||
}
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
consuming.Stop()
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// startPlanMovedLooks has the plans looked at when any process says one moved: the serving controller follows its own
|
||||
// `plan-moved`, so a plan a person moved with a verb at the control node's terminal — a process this one did not spawn,
|
||||
// whose saves it does not hear — is looked at all the same (novox/hq ADR 0406 §2, §5, issue 974). New messages only; the
|
||||
// first look reads the rest. A look its own save asks is the same look: asks are one.
|
||||
func startPlanMovedLooks(ctx context.Context, js *broker.JetStream) error {
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stream, err := api.Stream(ctx, broker.EventsStream)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the events stream: %w", err)
|
||||
}
|
||||
consumer, err := stream.OrderedConsumer(ctx, jetstream.OrderedConsumerConfig{
|
||||
FilterSubjects: []string{link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved)},
|
||||
DeliverPolicy: jetstream.DeliverNewPolicy})
|
||||
if err != nil {
|
||||
return fmt.Errorf("following the plans moved: %w", err)
|
||||
}
|
||||
consuming, err := consumer.Consume(func(jetstream.Msg) { askToLook() })
|
||||
if err != nil {
|
||||
return fmt.Errorf("following the plans moved: %w", err)
|
||||
}
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
consuming.Stop()
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- the end of a walk whose end is not known ---
|
||||
|
||||
var walkEnds = struct {
|
||||
mu sync.Mutex
|
||||
timers map[string]*time.Timer
|
||||
}{timers: map[string]*time.Timer{}}
|
||||
|
||||
// armWalkEnd has the walk recorded once its end can be known: a walk that ended with a machine silent is recorded
|
||||
// without its total ApplySilentAfter and a minute after its last save (recordWalkPhases), which no event says. Armed at
|
||||
// each save of the done walk, so it counts from the last.
|
||||
func armWalkEnd(id string, savedAt time.Time) {
|
||||
d := time.Until(savedAt.Add(inventory.ApplySilentAfter + time.Minute + time.Second))
|
||||
if d <= 0 {
|
||||
return
|
||||
}
|
||||
walkEnds.mu.Lock()
|
||||
defer walkEnds.mu.Unlock()
|
||||
if t := walkEnds.timers[id]; t != nil {
|
||||
t.Stop()
|
||||
}
|
||||
// retries: recording the end of a walk whose end is not known. One per walk, once, from its last save; the first
|
||||
// look after the lease was taken arms it again for the walks still unrecorded.
|
||||
walkEnds.timers[id] = time.AfterFunc(d, func() {
|
||||
walkEnds.mu.Lock()
|
||||
delete(walkEnds.timers, id)
|
||||
walkEnds.mu.Unlock()
|
||||
askToLook()
|
||||
})
|
||||
}
|
||||
|
||||
// rearmWalkEnds is armWalkEnd for the walks that ended lately, for a controller that has just taken the lease.
|
||||
func rearmWalkEnds(ctx context.Context, open *stores) {
|
||||
recent, err := open.inventory.RecentPlans(ctx, 30)
|
||||
if err != nil {
|
||||
fmt.Printf("plans: the walks ended lately could not be read to record their phases: %v\n", err)
|
||||
return
|
||||
}
|
||||
for _, p := range recent {
|
||||
if p.State == inventory.PlanDone && p.Release == nil && time.Since(p.Updated) <= 2*time.Hour {
|
||||
armWalkEnd(p.ID, p.Updated)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// planKept is what the serving controller does when a plan was saved: a look is asked for, so a send, a built tier or
|
||||
// a command's move is followed by the plan's next step; and a walk that ended is armed to be recorded.
|
||||
func planKept(p inventory.Plan) {
|
||||
if p.State == inventory.PlanDone && p.Release == nil {
|
||||
armWalkEnd(p.ID, time.Now())
|
||||
}
|
||||
askToLook()
|
||||
}
|
||||
|
||||
// --- `sent`: a tier sent to a machine, said ---
|
||||
|
||||
// sendMade is one machine a plan's tier was sent to.
|
||||
type sendMade struct {
|
||||
Tier int
|
||||
Node string
|
||||
Modules []string
|
||||
At time.Time
|
||||
}
|
||||
|
||||
// sendsMade collects the sends one step of a plan made, to be said once the plan is saved.
|
||||
type sendsMade struct {
|
||||
mu sync.Mutex
|
||||
list []sendMade
|
||||
}
|
||||
|
||||
type sendsMadeKey struct{}
|
||||
|
||||
func withSendsMade(ctx context.Context, s *sendsMade) context.Context {
|
||||
return context.WithValue(ctx, sendsMadeKey{}, s)
|
||||
}
|
||||
|
||||
// noteSends records that modules of the plan's tier were sent to the nodes.
|
||||
func noteSends(ctx context.Context, tier int, nodes, modules []string, at time.Time) {
|
||||
s, _ := ctx.Value(sendsMadeKey{}).(*sendsMade)
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for _, n := range nodes {
|
||||
s.list = append(s.list, sendMade{Tier: tier, Node: n, Modules: slices.Clone(modules), At: at})
|
||||
}
|
||||
}
|
||||
|
||||
// planSentBody is the body of the seat event `sent` (novox/hq ADR 0406 §5, issue 974): the plan, the tier, the machine
|
||||
// and the modules it was sent, and when.
|
||||
type planSentBody struct {
|
||||
Event string `json:"event"`
|
||||
Plan string `json:"plan"`
|
||||
Tier int `json:"tier"`
|
||||
Node string `json:"node"`
|
||||
Modules []string `json:"modules"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// sentBodies are the events a step's sends are said as.
|
||||
func sentBodies(plan string, sends *sendsMade) []planSentBody {
|
||||
sends.mu.Lock()
|
||||
defer sends.mu.Unlock()
|
||||
out := make([]planSentBody, 0, len(sends.list))
|
||||
for _, s := range sends.list {
|
||||
out = append(out, planSentBody{Event: link.KeySent, Plan: plan, Tier: s.Tier, Node: s.Node,
|
||||
Modules: s.Modules, At: s.At})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// saySends says each send a step made as `sent`, once the plan that records it is saved. What an outsider hears; the
|
||||
// controller's own next step does not wait for it.
|
||||
func saySends(ctx context.Context, p inventory.Plan, sends *sendsMade) {
|
||||
bus := checkEvents
|
||||
if bus == nil {
|
||||
return
|
||||
}
|
||||
for _, body := range sentBodies(p.ID, sends) {
|
||||
raw, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
go func() {
|
||||
stating, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer stop()
|
||||
if err := bus.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeySent, raw); err != nil {
|
||||
fmt.Printf("%s: tier %d was sent to %s, and could not be said as sent: %v\n", body.Plan, body.Tier, body.Node, err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,351 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
func rolloutOf(sentAgo time.Duration, reported bool) (inventory.Plan, []inventory.Reported) {
|
||||
sent := time.Now().Add(-sentAgo).UTC()
|
||||
p := inventory.Plan{ID: "plan-9", State: inventory.PlanRolling, Tiers: [][]string{{"a", "b"}}, Tier: 0,
|
||||
Modules: map[string]*inventory.PlanModule{
|
||||
"a": {SentAt: &sent, Rest: map[string]inventory.SentDeclaration{"ace": {Digest: "d1", At: sent}}},
|
||||
"b": {SentAt: &sent, Rest: map[string]inventory.SentDeclaration{"ace": {Digest: "d1", At: sent}}},
|
||||
}}
|
||||
r := inventory.Reported{Node: "ace", Sent: &sent}
|
||||
if reported {
|
||||
at := sent.Add(time.Minute)
|
||||
r.At, r.Current, r.Declared, r.Outcome = &at, true, "d1", inventory.OutcomeApplied
|
||||
}
|
||||
return p, []inventory.Reported{r}
|
||||
}
|
||||
|
||||
// A machine a plan sent and heard nothing from is a wait with a key, from when it was sent (issue 974, novox/hq ADR
|
||||
// 0406 §5); one that reported on the send, or that is silent already, is none.
|
||||
func TestAPlanWaitsOnTheMachinesItSentAndHeardNothingFrom(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
p, reports := rolloutOf(5*time.Minute, false)
|
||||
waits := silentWaitsOf(p, reports, now, nil)
|
||||
if len(waits) != 1 {
|
||||
t.Fatalf("waits = %v; want the one machine", waits)
|
||||
}
|
||||
for w, at := range waits {
|
||||
if w.Plan != "plan-9" || w.Node != "ace" || w.Tier != 0 || now.Sub(at) < 4*time.Minute || now.Sub(at) > 6*time.Minute {
|
||||
t.Errorf("wait %+v sent at %v is wrong", w, at)
|
||||
}
|
||||
}
|
||||
p, reports = rolloutOf(5*time.Minute, true)
|
||||
if waits := silentWaitsOf(p, reports, now, nil); len(waits) != 0 {
|
||||
t.Errorf("a machine that reported on its send is waited on: %v", waits)
|
||||
}
|
||||
p, reports = rolloutOf(inventory.ApplySilentAfter+time.Minute, false)
|
||||
if waits := silentWaitsOf(p, reports, now, nil); len(waits) != 0 {
|
||||
t.Errorf("a machine silent past its time is waited on again: %v", waits)
|
||||
}
|
||||
p, reports = rolloutOf(time.Minute, false)
|
||||
p.State = inventory.PlanDone
|
||||
if waits := silentWaitsOf(p, reports, now, nil); len(waits) != 0 {
|
||||
t.Errorf("a plan that ended is waiting: %v", waits)
|
||||
}
|
||||
}
|
||||
|
||||
// A release's open gate waits on its machines from when it began judging, and its verdict ends the wait.
|
||||
func TestAReleasesOpenGateWaitsOnItsMachines(t *testing.T) {
|
||||
since := time.Now().Add(-time.Minute).UTC()
|
||||
g := &inventory.PlanGate{Machines: []string{"g14", "novox"}, Since: &since}
|
||||
p := inventory.Plan{ID: "release-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{Gate: g}}
|
||||
waits := silentWaitsOf(p, nil, time.Now(), nil)
|
||||
if len(waits) != 2 {
|
||||
t.Fatalf("waits = %v; want g14 and novox", waits)
|
||||
}
|
||||
g.Verdict = inventory.GatePassed
|
||||
if waits := silentWaitsOf(p, nil, time.Now(), nil); len(waits) != 0 {
|
||||
t.Errorf("a gate with its verdict still waits: %v", waits)
|
||||
}
|
||||
}
|
||||
|
||||
// The keys follow the waits: a wait with no key, or a key of an earlier send, is armed; a key with no wait is deleted.
|
||||
func TestTheKeysFollowTheWaits(t *testing.T) {
|
||||
t1 := time.Now().UTC()
|
||||
t2 := t1.Add(time.Minute)
|
||||
a, b, c := silentWait{"p", 0, "a"}, silentWait{"p", 0, "b"}, silentWait{"p", 0, "c"}
|
||||
arm, drop := silentDiff(map[silentWait]time.Time{a: t1, b: t2, c: t1}, map[silentWait]time.Time{a: t1, b: t1, silentWait{"p", 1, "a"}: t1})
|
||||
if len(arm) != 2 || !(arm[0] == b || arm[1] == b) || !(arm[0] == c || arm[1] == c) {
|
||||
t.Errorf("armed %v; want b (sent again) and c (no key)", arm)
|
||||
}
|
||||
if len(drop) != 1 || drop[0] != (silentWait{"p", 1, "a"}) {
|
||||
t.Errorf("dropped %v; want the key with no wait", drop)
|
||||
}
|
||||
if ttl := silentTTL(t1, t1.Add(inventory.ApplySilentAfter+time.Hour)); ttl != time.Second {
|
||||
t.Errorf("a wait long over has a key of %v; want the least, one second", ttl)
|
||||
}
|
||||
}
|
||||
|
||||
func lookerCounting(failures *atomic.Int32, looks *atomic.Int32, block chan struct{}) *looker {
|
||||
l := newLooker(func(ctx context.Context) bool {
|
||||
looks.Add(1)
|
||||
if block != nil {
|
||||
<-block
|
||||
}
|
||||
return failures.Add(-1) >= 0
|
||||
})
|
||||
l.gap, l.base, l.most = 0, 20*time.Millisecond, 80*time.Millisecond
|
||||
return l
|
||||
}
|
||||
|
||||
func eventually(t *testing.T, what string, ok func() bool) {
|
||||
t.Helper()
|
||||
for end := time.Now().Add(15 * time.Second); time.Now().Before(end); time.Sleep(5 * time.Millisecond) {
|
||||
if ok() {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("never: %s", what)
|
||||
}
|
||||
|
||||
// Asks made while a look runs are not lost and are one look after it, not one each.
|
||||
func TestAsksMadeWhileALookRunsAreOneLookAfterIt(t *testing.T) {
|
||||
var failures, looks atomic.Int32
|
||||
block := make(chan struct{})
|
||||
l := lookerCounting(&failures, &looks, block)
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
go l.run(ctx)
|
||||
l.ask()
|
||||
eventually(t, "the first look starts", func() bool { return looks.Load() == 1 })
|
||||
for i := 0; i < 20; i++ {
|
||||
l.ask()
|
||||
}
|
||||
block <- struct{}{} // the first look ends
|
||||
eventually(t, "the asks made meanwhile are looked at", func() bool { return looks.Load() == 2 })
|
||||
block <- struct{}{}
|
||||
time.Sleep(60 * time.Millisecond)
|
||||
if n := looks.Load(); n != 2 {
|
||||
t.Errorf("%d looks for 21 asks; want 2", n)
|
||||
}
|
||||
}
|
||||
|
||||
// A look that failed a step is looked at again, one at a time with a longer wait each time, and a look that fails
|
||||
// nothing ends the retries (novox/hq ADR 0406 §5; the one named retry).
|
||||
func TestAFailedLookIsLookedAtAgainAndAGoodOneEndsTheRetries(t *testing.T) {
|
||||
var failures, looks atomic.Int32
|
||||
failures.Store(3) // three looks fail, then one is good
|
||||
l := lookerCounting(&failures, &looks, nil)
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
go l.run(ctx)
|
||||
l.ask()
|
||||
eventually(t, "four looks", func() bool { return looks.Load() == 4 })
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
if n := looks.Load(); n != 4 {
|
||||
t.Errorf("%d looks; the good one should have ended the retries at 4", n)
|
||||
}
|
||||
l.mu.Lock()
|
||||
armed, backoff := l.retry != nil, l.backoff
|
||||
l.mu.Unlock()
|
||||
if armed || backoff != 0 {
|
||||
t.Errorf("a retry is armed (%v) or the wait kept (%v) after a good look", armed, backoff)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheRetryWaitDoublesToItsMost(t *testing.T) {
|
||||
l := newLooker(func(context.Context) bool { return true })
|
||||
l.base, l.most = 5*time.Second, time.Minute
|
||||
var seen []time.Duration
|
||||
for i := 0; i < 6; i++ {
|
||||
l.after(true)
|
||||
seen = append(seen, l.backoff)
|
||||
l.retry.Stop()
|
||||
l.retry = nil
|
||||
}
|
||||
want := []time.Duration{5 * time.Second, 10 * time.Second, 20 * time.Second, 40 * time.Second, time.Minute, time.Minute}
|
||||
for i := range want {
|
||||
if seen[i] != want[i] {
|
||||
t.Fatalf("waits %v; want %v", seen, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A plan saved asks for a look, and a walk saved done is armed to be recorded once its end can be known.
|
||||
func TestAPlanSavedAsksForALookAndADoneWalkIsArmedToBeRecorded(t *testing.T) {
|
||||
var failures, looks atomic.Int32
|
||||
l := lookerCounting(&failures, &looks, nil)
|
||||
lookerMu.Lock()
|
||||
theLooker = l
|
||||
lookerMu.Unlock()
|
||||
t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() })
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
go l.run(ctx)
|
||||
|
||||
planKept(inventory.Plan{ID: "walk-1", State: inventory.PlanDone})
|
||||
t.Cleanup(func() {
|
||||
walkEnds.mu.Lock()
|
||||
defer walkEnds.mu.Unlock()
|
||||
if t := walkEnds.timers["walk-1"]; t != nil {
|
||||
t.Stop()
|
||||
}
|
||||
delete(walkEnds.timers, "walk-1")
|
||||
})
|
||||
eventually(t, "a look", func() bool { return looks.Load() == 1 })
|
||||
walkEnds.mu.Lock()
|
||||
_, armed := walkEnds.timers["walk-1"]
|
||||
walkEnds.mu.Unlock()
|
||||
if !armed {
|
||||
t.Error("a walk saved done is not armed to be recorded")
|
||||
}
|
||||
askToLook() // no panic, no looker, once it is taken away
|
||||
}
|
||||
|
||||
// The tier-sends a step made are said once each, per machine, with the modules it was sent.
|
||||
func TestTheSendsAStepMadeAreSaidPerMachine(t *testing.T) {
|
||||
sends := &sendsMade{}
|
||||
ctx := withSendsMade(context.Background(), sends)
|
||||
at := time.Now().UTC()
|
||||
noteSends(ctx, 2, []string{"ace"}, []string{"a", "b"}, at)
|
||||
noteSends(ctx, 2, []string{"g14"}, []string{"a"}, at)
|
||||
noteSends(context.Background(), 2, []string{"nobody"}, []string{"a"}, at) // outside a step: nothing kept
|
||||
got := sentBodies("plan-9", sends)
|
||||
if len(got) != 2 || got[0].Node != "ace" || len(got[0].Modules) != 2 || got[0].Tier != 2 || got[0].Plan != "plan-9" ||
|
||||
got[0].Event != "sent" || !got[0].At.Equal(at) || got[1].Node != "g14" {
|
||||
t.Errorf("sent bodies = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A silent key that expires asks for a look; a key a report deleted does not (issue 974, novox/hq ADR 0406 §5).
|
||||
func TestAnExpiredSilentKeyAsksForALookAndAReportsDeleteDoesNot(t *testing.T) {
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var failures, looks atomic.Int32
|
||||
l := lookerCounting(&failures, &looks, nil)
|
||||
lookerMu.Lock()
|
||||
theLooker = l
|
||||
lookerMu.Unlock()
|
||||
t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock(); setSilentClock(nil) })
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
go l.run(ctx)
|
||||
if err := startRolloutSilent(ctx, js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c := silentClockOf()
|
||||
w := silentWait{Plan: "plan-5", Tier: 0, Node: "ace"}
|
||||
if err := c.Arm(ctx, w, time.Second, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Reported(ctx, w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(1800 * time.Millisecond)
|
||||
if n := looks.Load(); n != 0 {
|
||||
t.Fatalf("a key a report deleted asked for %d looks", n)
|
||||
}
|
||||
if err := c.Arm(ctx, w, time.Second, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
eventually(t, "the expiry asks for a look", func() bool { return looks.Load() == 1 })
|
||||
}
|
||||
|
||||
// A provisioner client's word on a module's files asks for a look, as a send that handed it modules is applied only
|
||||
// once it has said so (issue 974, review 1): no walk waits for its silent key for what the client already said.
|
||||
func TestAProvisionerClientsWordAsksForALook(t *testing.T) {
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var failures, looks atomic.Int32
|
||||
l := lookerCounting(&failures, &looks, nil)
|
||||
lookerMu.Lock()
|
||||
theLooker = l
|
||||
lookerMu.Unlock()
|
||||
t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() })
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
go l.run(ctx)
|
||||
if err := startProvAppliedLooks(ctx, js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
legacy, err := js.Conn().JetStream()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := legacy.Publish(broker.ProvisionerAppliedSubject("ace", "plex"), []byte(`{"status":"applied"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
eventually(t, "a look on the client's word", func() bool { return looks.Load() >= 1 })
|
||||
}
|
||||
|
||||
type brokenClock struct{ silentClock }
|
||||
|
||||
func (brokenClock) Held(context.Context) (map[silentWait]time.Time, error) {
|
||||
return nil, errors.New("the bus is down")
|
||||
}
|
||||
|
||||
// A bus that will not say which keys it holds does not leave the waits unwatched: each is looked at by a timer of its own
|
||||
// and the error is given back, so the look is marked failed and retried (issue 974, review 1).
|
||||
func TestKeysThatCannotBeReadAreWatchedByTimersAndTheLookFails(t *testing.T) {
|
||||
w := silentWait{Plan: "plan-held", Tier: 0, Node: "ace"}
|
||||
sent := time.Now().UTC()
|
||||
t.Cleanup(func() { silentTimers.mu.Lock(); delete(silentTimers.timers, w); silentTimers.mu.Unlock() })
|
||||
err := keepSilentKeys(t.Context(), brokenClock{}, map[silentWait]time.Time{w: sent}, sent)
|
||||
if err == nil {
|
||||
t.Fatal("no error given back")
|
||||
}
|
||||
silentTimers.mu.Lock()
|
||||
armed := silentTimers.timers[w].Equal(sent)
|
||||
silentTimers.mu.Unlock()
|
||||
if !armed {
|
||||
t.Error("no timer of its own watches the wait")
|
||||
}
|
||||
}
|
||||
|
||||
// A plan moved by a process the serving controller did not spawn — a verb run at the control node's terminal — is
|
||||
// looked at: the controller follows its own plan-moved (issue 974, round 1).
|
||||
func TestAPlanMovedByAnotherProcessAsksForALook(t *testing.T) {
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var failures, looks atomic.Int32
|
||||
l := lookerCounting(&failures, &looks, nil)
|
||||
lookerMu.Lock()
|
||||
theLooker = l
|
||||
lookerMu.Unlock()
|
||||
t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() })
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
go l.run(ctx)
|
||||
if err := startPlanMovedLooks(ctx, js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
legacy, err := js.Conn().JetStream()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := legacy.Publish(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), []byte(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
eventually(t, "a look on the plan moved", func() bool { return looks.Load() >= 1 })
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **Every plan is written through SavePlansMoving, which says it** (issue 974, novox/hq ADR 0406 §2: every move of a plan
|
||||
// is said as plan-moved, so no move is found only by comparing). The one hook, inventory.PlanSaved, is called there
|
||||
// after the commit; so no statement that writes the plan table may stand anywhere else, and writePlan, the one that
|
||||
// does, may be called from nowhere but SavePlansMoving. Checked by reading the sources, no store needed.
|
||||
func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) {
|
||||
write := regexp.MustCompile(`(?is)(insert\s+into|update|delete\s+from|truncate)\s+(table\s+)?release_plan\b`)
|
||||
var found []string
|
||||
for _, root := range []string{"../../cmd", "../../internal"} {
|
||||
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
|
||||
return nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
text := string(raw)
|
||||
if write.MatchString(text) && filepath.ToSlash(path) != "../../internal/inventory/plans.go" {
|
||||
found = append(found, path+" writes the plan table")
|
||||
}
|
||||
if strings.Contains(text, "writePlan(") && filepath.ToSlash(path) != "../../internal/inventory/plans.go" {
|
||||
found = append(found, path+" calls writePlan")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if len(found) > 0 {
|
||||
t.Fatalf("a plan is written without plan-moved being said:\n%s", strings.Join(found, "\n"))
|
||||
}
|
||||
raw, err := os.ReadFile("../../internal/inventory/plans.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := string(raw)
|
||||
if n := len(write.FindAllString(text, -1)); n != 1 {
|
||||
t.Errorf("plans.go has %d statements that write the plan table; want the one in writePlan", n)
|
||||
}
|
||||
calls := regexp.MustCompile(`writePlan\(ctx, tx`).FindAllStringIndex(text, -1)
|
||||
save := strings.Index(text, "func (i *Inventory) SavePlansMoving")
|
||||
next := strings.Index(text[save:], "\nfunc ")
|
||||
if len(calls) != 1 || calls[0][0] < save || calls[0][0] > save+next {
|
||||
t.Errorf("writePlan is called %d times, and not only from SavePlansMoving", len(calls))
|
||||
}
|
||||
if !strings.Contains(text[save:save+next], "PlanSaved(i, *p)") {
|
||||
t.Error("SavePlansMoving does not say each plan it wrote")
|
||||
}
|
||||
}
|
||||
|
||||
// movesSaid records what the hook hears: for each plan, the revisions it was said at.
|
||||
type movesSaid struct {
|
||||
mu sync.Mutex
|
||||
revs map[string][]int64
|
||||
}
|
||||
|
||||
func (s *movesSaid) hear(_ *inventory.Inventory, p inventory.Plan) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.revs[p.ID] = append(s.revs[p.ID], p.Revision)
|
||||
}
|
||||
|
||||
// sayingEverySave installs a PlanSaved that records, as the serving controller's says plan-moved.
|
||||
func sayingEverySave(t *testing.T) *movesSaid {
|
||||
t.Helper()
|
||||
s := &movesSaid{revs: map[string][]int64{}}
|
||||
was := inventory.PlanSaved
|
||||
inventory.PlanSaved = s.hear
|
||||
t.Cleanup(func() { inventory.PlanSaved = was })
|
||||
return s
|
||||
}
|
||||
|
||||
// noUnsaid fails the test for a plan whose stored revision is not one the hook heard: a move nobody said.
|
||||
func (s *movesSaid) noUnsaid(t *testing.T, open *stores) {
|
||||
t.Helper()
|
||||
recent, err := open.inventory.RecentPlans(t.Context(), 50)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if len(recent) == 0 {
|
||||
t.Fatal("no plan was kept")
|
||||
}
|
||||
for _, p := range recent {
|
||||
revs := s.revs[p.ID]
|
||||
if len(revs) == 0 || revs[len(revs)-1] != p.Revision {
|
||||
t.Errorf("%s is kept at revision %d and was said at %v: a move nobody said", p.ID, p.Revision, revs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Live, with the store: the moves a merge makes of a plan — the batch it opens, the walk it is cut into, the tier that
|
||||
// is asked — are each said, and a look that changes nothing says nothing (novox/hq ADR 0406 §2).
|
||||
func TestEveryMoveAMergeAndTheCutMakeOfAPlanIsSaid(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksWithPaths(t)
|
||||
heard := sayingEverySave(t)
|
||||
hear(t, open, catalogueMerge("553b7191claude", "app", t0), t0.Add(time.Second))
|
||||
if _, bs := walks(t, open); len(bs) != 1 {
|
||||
t.Fatalf("%d batch(es) opened", len(bs))
|
||||
}
|
||||
heard.noUnsaid(t, open)
|
||||
cutAt(t, open, t0.Add(91*time.Second))
|
||||
if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 {
|
||||
t.Fatalf("%d walk(s), %d batch(es) after the cut", len(ws), len(bs))
|
||||
}
|
||||
heard.noUnsaid(t, open)
|
||||
counted := 0
|
||||
for _, r := range heard.revs {
|
||||
counted += len(r)
|
||||
}
|
||||
cutAt(t, open, t0.Add(92*time.Second))
|
||||
again := 0
|
||||
for _, r := range heard.revs {
|
||||
again += len(r)
|
||||
}
|
||||
if again != counted {
|
||||
t.Errorf("a look that changed nothing said %d more move(s)", again-counted)
|
||||
}
|
||||
}
|
||||
@@ -197,12 +197,23 @@ func retryRefusal(p inventory.Plan, plans []inventory.Plan) error {
|
||||
return oneWalkAtATime(p, plans)
|
||||
}
|
||||
|
||||
// oneWalkAtATime refuses a retry while another walk is open, started or waiting for its word (novox/hq ADR
|
||||
// 0276): a walk retried beside it would be two walks at once.
|
||||
// oneWalkAtATime refuses a retry while an open walk, started or waiting for its word, is related to it (novox/hq
|
||||
// ADR 0276, ADR 0407 decision 4, issue 973): a walk retried beside a related one would be two walks on the same
|
||||
// things at once. A walk related to none may be retried beside them. The retried walk is judged by the modules it
|
||||
// plans, which already hold their dependents, and the repositories it carries.
|
||||
func oneWalkAtATime(p inventory.Plan, plans []inventory.Plan) error {
|
||||
mine := footprintOfWalk(p, nil)
|
||||
for _, q := range plans {
|
||||
if q.ID != p.ID && q.Open() && q.Release == nil {
|
||||
return fmt.Errorf("%s is open (%s): one walk at a time — retry %s once it ended", q.ID, q.Named(), p.ID)
|
||||
if q.ID == p.ID || !q.Open() || q.Release != nil {
|
||||
continue
|
||||
}
|
||||
if ownWalk(p) || ownWalk(q) {
|
||||
return fmt.Errorf("%s is open (%s) and one of them is on the controller's own path: one walk at a time — "+
|
||||
"retry %s once it ended", q.ID, q.Named(), p.ID)
|
||||
}
|
||||
if ok, why := related(mine, footprintOfWalk(q, nil), nil); ok {
|
||||
return fmt.Errorf("%s is open (%s) and %s: one walk at a time beside a related one — retry %s once it ended",
|
||||
q.ID, q.Named(), why, p.ID)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -680,3 +680,35 @@ func TestAPlanSaysWhyEachModuleIsInIt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **A context naming the module's own repository is the module's own build source** (novox/hq ADR 0420 §4,
|
||||
// issue 1025). A module registered from the controller's repository whose image compiles cmd/x through a
|
||||
// context naming that same repository is moved by a change to the program, and by none outside its closure.
|
||||
func TestAContextNamingTheModulesOwnRepositoryIsItsOwnBuildSource(t *testing.T) {
|
||||
const controllerRepo = "http://forge.internal:20000/novox/mesh-controller.git"
|
||||
entries := []inventory.Entry{fromRepo("mover", controllerRepo, "modules/mover")}
|
||||
said := []inventory.ReadRepository{
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"go.mod", "cmd/x/", "internal/y/"}},
|
||||
{Own: true, Paths: []string{"modules/mover/module.json"}},
|
||||
}
|
||||
read := map[string][]inventory.ReadRepository{"mover": inventory.OwnRepositoryAsOwn(said, controllerRepo)}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
paths []string
|
||||
want string
|
||||
}{
|
||||
{"the program its image compiles", []string{"cmd/x/main.go"}, "mover"},
|
||||
{"a package the program imports", []string{"internal/y/y.go"}, "mover"},
|
||||
{"its own manifest", []string{"modules/mover/module.json"}, "mover"},
|
||||
{"a file outside its closure", []string{"cmd/other/main.go"}, ""},
|
||||
} {
|
||||
if _, got := planMerge(t, "mesh-controller", c.paths, entries, read, nil); got != c.want {
|
||||
t.Errorf("%s: planned %q, wanted %q", c.what, got, c.want)
|
||||
}
|
||||
}
|
||||
// A context of another repository stays a context read, and a context of this one that said no paths too.
|
||||
other := []inventory.ReadRepository{{Repository: "novox/mesh-catalog", Paths: []string{"a/"}}, {Repository: "novox/mesh-controller"}}
|
||||
if got := inventory.OwnRepositoryAsOwn(other, controllerRepo); len(got) != 2 || got[0].Own || got[1].Own {
|
||||
t.Errorf("a context that is not this repository's with paths became its own: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -320,7 +320,11 @@ func probeAgentRoot(ctx context.Context, d *doctor) ([]conditions.Observation, e
|
||||
quiet, err := searchStillRunning(ctx, inv, node, n.AgentAccount, h, had, now)
|
||||
return err == nil && quiet
|
||||
}
|
||||
return rootObservations(ctx, r, trustedMachines(r.entries), time.Now()), nil
|
||||
obs := rootObservations(ctx, r, trustedMachines(r.entries), time.Now())
|
||||
// A membership that lost its verified sender to a judgement that failed at a push gets it back here, once
|
||||
// the machines pass (issue 930); the bound is one self-check.
|
||||
giveVerifiedSendersBack(ctx, d)
|
||||
return obs, nil
|
||||
}
|
||||
|
||||
// rootObservations judges the machines and says each that fails.
|
||||
@@ -365,14 +369,3 @@ func rootFreeAnswer(ctx context.Context, machines string, now time.Time) (any, e
|
||||
}
|
||||
return map[string]any{"machines": judgeRootFree(ctx, newRootReader(ctx, d.open.inventory, conn), names, now)}, nil
|
||||
}
|
||||
|
||||
// rootFreeNow is the machines judged root-free, for composing a push's memberships: only those that pass.
|
||||
func rootFreeNow(ctx context.Context, r *rootReader, machines []string, now time.Time) map[string]bool {
|
||||
free := map[string]bool{}
|
||||
for _, v := range judgeRootFree(ctx, r, machines, now) {
|
||||
if v.Free {
|
||||
free[v.Machine] = true
|
||||
}
|
||||
}
|
||||
return free
|
||||
}
|
||||
|
||||
+25
-17
@@ -180,12 +180,16 @@ func serve(ctx context.Context) (err error) {
|
||||
// And build results nobody was waiting for. A build triggered any other way than `build`
|
||||
// would otherwise be reported into the void, which is the same as not reporting it.
|
||||
server.Records(builds{inv, open})
|
||||
// Open plans move on a timer as well as on outcomes (novox/hq ADR 0162): a tier waiting for
|
||||
// machines to report moves when they have, and a plan left by a replaced controller resumes.
|
||||
go planTicker(ctx, open)
|
||||
// And the merge window (novox/hq ADR 0276): a batch is cut into its walk when the window closes, which no
|
||||
// merge or outcome says.
|
||||
go batchCutter(ctx, open)
|
||||
// Open plans move on events (novox/hq ADR 0406 §5, issue 974): a tier is asked on the `built` and the send of the
|
||||
// tier before it, a walk ends on its machines' reports, a silent machine is a key that expired, and a plan left
|
||||
// by a replaced controller resumes from the first look, which this asks for. No timer looks every so often.
|
||||
startRolloutEvents(ctx, open, server.JetStream())
|
||||
// And the merge window (novox/hq ADR 0276, ADR 0407): a batch is cut into its walk when the window closes,
|
||||
// which no merge or outcome says. The window is two keys on the bus that expire and a watcher that hears it;
|
||||
// nothing polls (issue 973).
|
||||
if err := startMergeWindow(ctx, open, server.JetStream()); err != nil {
|
||||
return fmt.Errorf("the merge window could not be put on the bus: %w", err)
|
||||
}
|
||||
// And the pending assignments settled on a tick of their own (novox/hq ADR 0261): made once their module
|
||||
// is registered, ended with why when its build will not register it, raised and cleared as conditions.
|
||||
// Never by a read.
|
||||
@@ -198,9 +202,13 @@ func serve(ctx context.Context) (err error) {
|
||||
if err := server.Follows(following{open: open}); err != nil {
|
||||
return err
|
||||
}
|
||||
// And the merges the bus announced and never handed over, read back on a timer and acted on late
|
||||
// rather than never (novox/hq issue 266).
|
||||
go catchingUpOnMerges(ctx, open, server)
|
||||
// And the merges mesh-delivery made, kept a week with the ask each was made on: what registers a module that
|
||||
// moved between repositories on the operator's Approve (novox/hq ADR 0420 §7, issue 1025).
|
||||
if err := server.KeepsDeliveryMerges(keepingDeliveryMerges{js: server.JetStream}); err != nil {
|
||||
return err
|
||||
}
|
||||
// No timer reads the merges back (novox/hq ADR 0407 decision 3, issue 973): the forge's event and the durable
|
||||
// consumer on the bus keep every merge, and one the controller cannot record yet is held and handed over again.
|
||||
// And a catalogue that has just started, asking for what it missed. The same type answers
|
||||
// both: what a build meant and what the builds were are two questions about one record.
|
||||
if err := server.Answers(following{open: open}); err != nil {
|
||||
@@ -250,13 +258,7 @@ func serve(ctx context.Context) (err error) {
|
||||
// (novox/hq to-be 45 §9).
|
||||
checkEvents = bus
|
||||
// And every walk this controller keeps in a new state, for the delivery it walks (novox/hq ADR 0239).
|
||||
inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) {
|
||||
sayPlanMoved(ctx, bus, open.inventory, p)
|
||||
// A walk ended has taken what it reuses: its merges' check builds go (novox/hq ADR 0350 §5).
|
||||
if p.WalkSaid == inventory.WalkEnded {
|
||||
go forgetChecksOfWalk(context.WithoutCancel(ctx), open.inventory, p)
|
||||
}
|
||||
}
|
||||
inventory.PlanSaved = servingPlanSaved(ctx, bus, open)
|
||||
if err := server.Checks(following{open: open}); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1333,8 +1335,14 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, se
|
||||
}
|
||||
// Which machines are root-free now (novox/hq ADR 0259 §8): a channel's verified sender is composed for the
|
||||
// router only from one, beside a router on one. Judged once per push, by the root-free verb's judgement.
|
||||
records.RootFree = rootFreeNow(ctx, newRootReader(ctx, open.inventory, bus.Conn), records.Nodes, time.Now())
|
||||
var whys map[string]string
|
||||
records.RootFree, whys = rootVerdictsOf(judgeRootFree(ctx, newRootReader(ctx, open.inventory, bus.Conn),
|
||||
records.Nodes, time.Now()))
|
||||
where := broker.PlacementsOf(records, records.Interchangeable)
|
||||
// What the judgement kept off is said, with its reason (novox/hq issue 930); the self-check gives it back.
|
||||
for _, line := range droppedVerifiedSenders(records, where, whys) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
// **Every declared state's bucket, before the memberships that name it** (novox/hq ADR 0201). The
|
||||
// raise at start asserts them too, but a module registered and assigned since would otherwise have
|
||||
// its bucket only after the control plane next restarts — found the first time a module declared
|
||||
|
||||
@@ -80,30 +80,6 @@ func listQueue(ctx context.Context, args []string, w io.Writer) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// keepingTheQueueInOrder runs the build queue's passes while this controller acts (novox/hq issue 558):
|
||||
// the one process that puts the queue in order, so no two passes pause and resume it over each other.
|
||||
// The build seat is read from the catalogue at most every half minute.
|
||||
func keepingTheQueueInOrder(ctx context.Context, open *stores, js *broker.JetStream) {
|
||||
if js == nil {
|
||||
return
|
||||
}
|
||||
seat, readAt := link.TheBuildMachine, time.Time{}
|
||||
seatNow := func() string {
|
||||
if time.Since(readAt) < watchEvery {
|
||||
return seat
|
||||
}
|
||||
if entries, err := open.inventory.Catalogued(ctx); err == nil {
|
||||
seat, readAt = buildSeatAmong(entries), time.Now()
|
||||
} else {
|
||||
fmt.Printf("the build seat is read as %s: the catalogue cannot be read: %v\n", seat, err)
|
||||
}
|
||||
return seat
|
||||
}
|
||||
link.KeepingInOrder(ctx, js, seatNow, link.Holding, func(format string, args ...any) {
|
||||
fmt.Printf(format+"\n", args...)
|
||||
})
|
||||
}
|
||||
|
||||
// queueAnswer is the queue as data (novox/hq issue 455): what a caller reads to say how far back an ask
|
||||
// waits — "waiting in the build queue behind 2" — which the table alone could not tell it. Text is the
|
||||
// table, written from the same read: the verb answers it as its output and the rest as its answer, so the
|
||||
@@ -303,16 +279,8 @@ func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat str
|
||||
fmt.Fprintf(os.Stderr, "could not withdraw the cancel of %s: %v — a holder taking it ends it as cancelled\n", ask.ID, err)
|
||||
}
|
||||
}
|
||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
|
||||
// **Where it waits now** (novox/hq issue 558): moved into order since it was read, a waiting ask has a
|
||||
// later sequence, and the worker handing out what was behind its old one is not it being taken.
|
||||
if ask.State == link.AskWaiting {
|
||||
if q, err := link.ReadQueue(ctx, js, seat); err == nil {
|
||||
if now, ok := q.Find(ask.ID); ok && now.State == link.AskWaiting {
|
||||
ask.Seq = now.Seq
|
||||
}
|
||||
}
|
||||
}
|
||||
// The worker of the queue it waits in: the seat's own, or one of the two check queues' (novox/hq ADR 0406 §3).
|
||||
worker := link.WorkerOf(seat, ask)
|
||||
switch ask.State {
|
||||
case link.AskWaiting:
|
||||
if taken, err := takenSince(js, worker, ask.Seq); err != nil {
|
||||
@@ -334,7 +302,6 @@ func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat str
|
||||
"ends it there", ask.ID, started, ask.ID)
|
||||
}
|
||||
}
|
||||
// Found by its id when it was moved into order since it was read (novox/hq issue 558).
|
||||
removed, err := link.DeleteAsk(ctx, js, seat, ask)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s is marked cancelled and could not be deleted from the queue (seq %d): %w — a "+
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A channel's verified sender is composed into a membership only while the machines are judged root-free at
|
||||
// that moment (novox/hq ADR 0259 §8), and the judgement fails closed when a read fails (novox/hq issue 930):
|
||||
// a push composed during a failed read issued the router a membership without it, and nothing composed it
|
||||
// again, so every approval was refused until somebody pushed. Two things follow.
|
||||
//
|
||||
// What a composition drops it says (droppedVerifiedSenders), with the reason the judgement gave.
|
||||
//
|
||||
// And the self-check issues it again (reissueVerifiedSenders): each time it judges the machines, a membership
|
||||
// that lacks a verified sender the composition would now give it is issued again with only its kinds changed.
|
||||
// **Bound: one self-check, doctorEvery (five minutes), after the judgement passes.** It only ever adds: a
|
||||
// machine that stops passing keeps what it was issued, as at a push, because the router asks the root-free
|
||||
// verb again before it honours an approval, and that is the check that holds.
|
||||
|
||||
// droppedVerifiedSenders says, one line each, every kind that declares `verified-sender` and was composed
|
||||
// without it, naming the kind, its machine and why. whys is each machine's verdict reason (rootVerdict.Why).
|
||||
func droppedVerifiedSenders(r broker.Records, where broker.Placements, whys map[string]string) []string {
|
||||
var routerNodes []string
|
||||
for node, declared := range r.Assigned {
|
||||
for _, d := range declared {
|
||||
for _, s := range d.Holds {
|
||||
if s.Name == routerSeat && !slices.Contains(routerNodes, node) {
|
||||
routerNodes = append(routerNodes, node)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(routerNodes)
|
||||
var out []string
|
||||
for _, k := range where.Kinds {
|
||||
declared := false
|
||||
for _, d := range r.Assigned[k.Node] {
|
||||
for _, s := range d.Holds {
|
||||
if d.Module == k.Module && s.Name == k.Seat && s.Kind == k.Kind && slices.Contains(s.Capabilities, "verified-sender") {
|
||||
declared = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !declared || slices.Contains(k.Capabilities, "verified-sender") {
|
||||
continue
|
||||
}
|
||||
var why string
|
||||
switch {
|
||||
case !r.RootFree[k.Node]:
|
||||
why = k.Node + " is not judged root-free: " + whys[k.Node]
|
||||
case len(routerNodes) != 1:
|
||||
why = fmt.Sprintf("the router is placed on %d machines, not one", len(routerNodes))
|
||||
case !r.RootFree[routerNodes[0]]:
|
||||
why = "the router's machine " + routerNodes[0] + " is not judged root-free: " + whys[routerNodes[0]]
|
||||
default:
|
||||
why = k.Module + " does not run as an account of its own"
|
||||
}
|
||||
out = append(out, fmt.Sprintf(" %s kind %s of %s on %s is composed without its verified sender: %s",
|
||||
k.Seat, k.Kind, k.Module, k.Node, why))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// withVerifiedSenders is a membership as issued with its seat traffic's kinds replaced by the composed ones,
|
||||
// and whether that changed it: only when a composed kind carries a verified sender the issued one lacks. Pure.
|
||||
func withVerifiedSenders(issued []byte, composed *broker.SeatTraffic) ([]byte, bool, error) {
|
||||
if composed == nil {
|
||||
return issued, false, nil
|
||||
}
|
||||
var fields map[string]json.RawMessage
|
||||
if err := json.Unmarshal(issued, &fields); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
var traffic map[string]json.RawMessage
|
||||
var had []broker.KindHeld
|
||||
if raw, ok := fields["seat-traffic"]; ok {
|
||||
if err := json.Unmarshal(raw, &traffic); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if raw, ok := traffic["kinds"]; ok {
|
||||
if err := json.Unmarshal(raw, &had); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
}
|
||||
}
|
||||
gains := false
|
||||
for _, k := range composed.Kinds {
|
||||
if !slices.Contains(k.Capabilities, "verified-sender") {
|
||||
continue
|
||||
}
|
||||
held := false
|
||||
for _, h := range had {
|
||||
if h.Seat == k.Seat && h.Kind == k.Kind && h.Module == k.Module && h.Node == k.Node &&
|
||||
slices.Contains(h.Capabilities, "verified-sender") {
|
||||
held = true
|
||||
}
|
||||
}
|
||||
gains = gains || !held
|
||||
}
|
||||
if !gains {
|
||||
return issued, false, nil
|
||||
}
|
||||
if traffic == nil {
|
||||
traffic = map[string]json.RawMessage{}
|
||||
}
|
||||
raw, err := json.Marshal(composed.Kinds)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
traffic["kinds"] = raw
|
||||
if fields["seat-traffic"], err = json.Marshal(traffic); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
body, err := json.Marshal(fields)
|
||||
return body, true, err
|
||||
}
|
||||
|
||||
// reissueVerifiedSenders issues again each membership that lacks a verified sender the records now give it:
|
||||
// read as last issued, its kinds replaced, published only if no newer one was issued since (a push runs beside
|
||||
// the self-check, and its newer membership must not be overwritten by the older fields read here). One never issued is left to its own send. Failures are
|
||||
// said and passed over; the next self-check tries again.
|
||||
func reissueVerifiedSenders(r broker.Records, where broker.Placements,
|
||||
last func(node, module string) ([]byte, uint64, error), publish func(node, module string, body []byte, seq uint64) error,
|
||||
say func(format string, args ...any)) int {
|
||||
issued := 0
|
||||
for node, declared := range r.Assigned {
|
||||
for _, d := range declared {
|
||||
composed := broker.MembershipFor(node, d, where).SeatTraffic
|
||||
if composed == nil || !slices.ContainsFunc(composed.Kinds, func(k broker.KindHeld) bool {
|
||||
return slices.Contains(k.Capabilities, "verified-sender")
|
||||
}) {
|
||||
continue
|
||||
}
|
||||
was, seq, err := last(node, d.Module)
|
||||
if err != nil {
|
||||
if !errors.Is(err, nats.ErrMsgNotFound) {
|
||||
say(" %s on %s: its membership could not be read to give its verified sender back: %v", d.Module, node, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
body, changed, err := withVerifiedSenders(was, composed)
|
||||
if err != nil {
|
||||
say(" %s on %s: its membership could not be read to give its verified sender back: %v", d.Module, node, err)
|
||||
continue
|
||||
}
|
||||
if !changed {
|
||||
continue
|
||||
}
|
||||
if err := publish(node, d.Module, body, seq); err != nil {
|
||||
// A push issued another since it was read: that one is newer, and the next self-check looks again.
|
||||
if strings.Contains(err.Error(), "wrong last sequence") {
|
||||
say(" %s on %s: a newer membership was issued meanwhile; its verified sender is looked at again next time",
|
||||
d.Module, node)
|
||||
continue
|
||||
}
|
||||
say(" %s on %s: its verified sender could not be issued again: %v", d.Module, node, err)
|
||||
continue
|
||||
}
|
||||
say(" %s on %s: issued its membership again, with the verified sender its machines now pass for", d.Module, node)
|
||||
issued++
|
||||
}
|
||||
}
|
||||
return issued
|
||||
}
|
||||
|
||||
// rootVerdictsOf is each machine's verdict reason, for saying why one was not free.
|
||||
func rootVerdictsOf(verdicts []rootVerdict) (free map[string]bool, whys map[string]string) {
|
||||
free, whys = map[string]bool{}, map[string]string{}
|
||||
for _, v := range verdicts {
|
||||
if v.Free {
|
||||
free[v.Machine] = true
|
||||
}
|
||||
whys[v.Machine] = strings.TrimSpace(v.Why)
|
||||
}
|
||||
return free, whys
|
||||
}
|
||||
|
||||
// giveVerifiedSendersBack is the self-check's step: every machine judged, the memberships that lost a verified
|
||||
// sender to a failed judgement issued again. Never fails the probe: what could not be done is said.
|
||||
func giveVerifiedSendersBack(ctx context.Context, d *doctor) {
|
||||
if d.js == nil || d.open == nil || d.open.inventory == nil {
|
||||
return
|
||||
}
|
||||
records, err := d.open.inventory.BusRecords(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf(" the memberships could not be read to give a verified sender back: %v\n", err)
|
||||
return
|
||||
}
|
||||
// Judged for every machine the records name, as a push does, not only those the self-check judged.
|
||||
r := newRootReader(ctx, d.open.inventory, d.js.Conn())
|
||||
all := judgeRootFree(ctx, r, records.Nodes, time.Now())
|
||||
records.RootFree, _ = rootVerdictsOf(all)
|
||||
where := broker.PlacementsOf(records, records.Interchangeable)
|
||||
reissueVerifiedSendersOn(ctx, link.OverNATS{Conn: d.js.Conn(), JS: d.js.Context()}, records, where)
|
||||
}
|
||||
|
||||
// reissueVerifiedSendersOn is reissueVerifiedSenders on the bus: memberships read from the AssignmentsStream and
|
||||
// published to it.
|
||||
func reissueVerifiedSendersOn(ctx context.Context, bus link.OverNATS, records broker.Records, where broker.Placements) int {
|
||||
return reissueVerifiedSenders(records, where,
|
||||
func(node, module string) ([]byte, uint64, error) {
|
||||
m, err := bus.JS.GetLastMsg(broker.AssignmentsStream, broker.MembershipSubject(node, module))
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return m.Data, m.Sequence, nil
|
||||
},
|
||||
func(node, module string, body []byte, seq uint64) error {
|
||||
return bus.PublishMembershipAfter(ctx, node, module, body, seq)
|
||||
},
|
||||
func(format string, args ...any) { fmt.Printf(format+"\n", args...) })
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
func verifiedSenderRecords(free map[string]bool) (broker.Records, broker.Declared) {
|
||||
tg := broker.Seat{Name: "channel", Scope: "mesh", Accepts: []string{"show"}, Kinded: true, Kind: "telegram",
|
||||
DeclaredBy: "messenger", Capabilities: []string{"choice", "verified-sender"}}
|
||||
router := broker.Declared{Module: "messenger", RunsAs: "messenger", Holds: []broker.Seat{{Name: "operator-channel",
|
||||
Scope: "mesh", Accepts: []string{"ask"}}}, Uses: []broker.Seat{{Name: "channel", Scope: "mesh", Kinded: true,
|
||||
DeclaredBy: "messenger"}}}
|
||||
return broker.Records{Nodes: []string{"novox"}, RootFree: free, Assigned: map[string][]broker.Declared{
|
||||
"novox": {router, {Module: "telegram", RunsAs: "telegram", Holds: []broker.Seat{tg}}}}}, router
|
||||
}
|
||||
|
||||
type fakeBus struct {
|
||||
last map[string][]byte
|
||||
published map[string][]byte
|
||||
seq map[string]uint64
|
||||
// between runs once, after a read and before the write: a push issuing meanwhile.
|
||||
between func()
|
||||
}
|
||||
|
||||
func (f *fakeBus) get(node, module string) ([]byte, uint64, error) {
|
||||
if b, ok := f.last[node+"/"+module]; ok {
|
||||
return b, f.seq[node+"/"+module], nil
|
||||
}
|
||||
return nil, 0, nats.ErrMsgNotFound
|
||||
}
|
||||
func (f *fakeBus) put(node, module string, body []byte, seq uint64) error {
|
||||
if f.between != nil {
|
||||
f.between()
|
||||
f.between = nil
|
||||
}
|
||||
if seq != f.seq[node+"/"+module] {
|
||||
return errors.New("nats: wrong last sequence")
|
||||
}
|
||||
f.published[node+"/"+module] = body
|
||||
f.last[node+"/"+module] = body
|
||||
f.seq[node+"/"+module]++
|
||||
return nil
|
||||
}
|
||||
|
||||
func hasVerified(t *testing.T, membership []byte) bool {
|
||||
t.Helper()
|
||||
var m broker.Membership
|
||||
if err := json.Unmarshal(membership, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, k := range m.SeatTraffic.Kinds {
|
||||
if k.Kind == "telegram" {
|
||||
return strings.Contains(strings.Join(k.Capabilities, ","), "verified-sender")
|
||||
}
|
||||
}
|
||||
t.Fatal("telegram not in the router's membership")
|
||||
return false
|
||||
}
|
||||
|
||||
// A membership composed while the judgement failed is issued again once it passes, with the verified sender
|
||||
// (novox/hq issue 930); while it fails, nothing is given back (ADR 0259 §8).
|
||||
func TestAMembershipThatLostItsVerifiedSenderGetsItBackWhenTheJudgementPasses(t *testing.T) {
|
||||
failed, router := verifiedSenderRecords(nil)
|
||||
whereFailed := broker.PlacementsOf(failed, nil)
|
||||
bus := &fakeBus{last: map[string][]byte{}, published: map[string][]byte{}, seq: map[string]uint64{}}
|
||||
body, _ := json.Marshal(broker.MembershipFor("novox", router, whereFailed))
|
||||
bus.last["novox/messenger"] = body
|
||||
if hasVerified(t, body) {
|
||||
t.Fatal("composed with a failed judgement, and the verified sender is kept")
|
||||
}
|
||||
say := func(string, ...any) {}
|
||||
|
||||
if n := reissueVerifiedSenders(failed, whereFailed, bus.get, bus.put, say); n != 0 || len(bus.published) != 0 {
|
||||
t.Fatalf("issued %d while the judgement still fails", n)
|
||||
}
|
||||
passing, _ := verifiedSenderRecords(map[string]bool{"novox": true})
|
||||
wherePassing := broker.PlacementsOf(passing, nil)
|
||||
if n := reissueVerifiedSenders(passing, wherePassing, bus.get, bus.put, say); n != 1 {
|
||||
t.Fatalf("issued %d memberships after the judgement passed, want 1", n)
|
||||
}
|
||||
if !hasVerified(t, bus.published["novox/messenger"]) {
|
||||
t.Fatal("the membership issued again lacks the verified sender")
|
||||
}
|
||||
if n := reissueVerifiedSenders(passing, wherePassing, bus.get, bus.put, say); n != 0 {
|
||||
t.Fatalf("issued %d again with nothing changed", n)
|
||||
}
|
||||
}
|
||||
|
||||
// What a composition drops it says, naming the kind, the machine and the judgement's reason.
|
||||
func TestADroppedVerifiedSenderIsSaidWithItsReason(t *testing.T) {
|
||||
records, _ := verifiedSenderRecords(nil)
|
||||
where := broker.PlacementsOf(records, nil)
|
||||
lines := droppedVerifiedSenders(records, where, map[string]string{"novox": "not measured: no verb heard"})
|
||||
if len(lines) != 1 || !strings.Contains(lines[0], "telegram") || !strings.Contains(lines[0], "novox") ||
|
||||
!strings.Contains(lines[0], "not measured: no verb heard") {
|
||||
t.Fatalf("said %q", lines)
|
||||
}
|
||||
records, _ = verifiedSenderRecords(map[string]bool{"novox": true})
|
||||
if lines := droppedVerifiedSenders(records, broker.PlacementsOf(records, nil), nil); len(lines) != 0 {
|
||||
t.Fatalf("said %q of a kind that kept its verified sender", lines)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMembershipNeverIssuedIsLeftToItsOwnSend(t *testing.T) {
|
||||
records, _ := verifiedSenderRecords(map[string]bool{"novox": true})
|
||||
bus := &fakeBus{last: map[string][]byte{}, published: map[string][]byte{}, seq: map[string]uint64{}}
|
||||
var said []string
|
||||
n := reissueVerifiedSenders(records, broker.PlacementsOf(records, nil), bus.get, bus.put,
|
||||
func(f string, a ...any) { said = append(said, f) })
|
||||
if n != 0 || len(said) != 0 {
|
||||
t.Fatalf("issued %d, said %v", n, said)
|
||||
}
|
||||
}
|
||||
|
||||
// **On a real bus** (novox/hq issue 930): a router's membership issued without the verified sender because the
|
||||
// judgement failed is, once the judgement passes, published again to the AssignmentsStream with it in its seat
|
||||
// traffic's kinds, and a second pass publishes nothing.
|
||||
func TestNatsAMembershipIssuedWithoutItsVerifiedSenderIsIssuedAgainWhenTheJudgementPasses(t *testing.T) {
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bus := link.OverNATS{Conn: js.Conn(), JS: js.Context()}
|
||||
|
||||
failed, router := verifiedSenderRecords(nil)
|
||||
body, err := json.Marshal(broker.MembershipFor("novox", router, broker.PlacementsOf(failed, nil)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := bus.PublishMembership(t.Context(), "novox", "messenger", body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
issued := func() []byte {
|
||||
m, err := js.Context().GetLastMsg(broker.AssignmentsStream, broker.MembershipSubject("novox", "messenger"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m.Data
|
||||
}
|
||||
if hasVerified(t, issued()) {
|
||||
t.Fatal("issued with a failed judgement, and the verified sender is kept: the test proves nothing")
|
||||
}
|
||||
if n := reissueVerifiedSendersOn(t.Context(), bus, failed, broker.PlacementsOf(failed, nil)); n != 0 {
|
||||
t.Fatalf("issued %d memberships while the judgement still fails", n)
|
||||
}
|
||||
passing, _ := verifiedSenderRecords(map[string]bool{"novox": true})
|
||||
where := broker.PlacementsOf(passing, nil)
|
||||
if n := reissueVerifiedSendersOn(t.Context(), bus, passing, where); n != 1 {
|
||||
t.Fatalf("issued %d memberships after the judgement passed, want 1", n)
|
||||
}
|
||||
if !hasVerified(t, issued()) {
|
||||
t.Fatal("the membership on the bus lacks the verified sender after the judgement passed")
|
||||
}
|
||||
if n := reissueVerifiedSendersOn(t.Context(), bus, passing, where); n != 0 {
|
||||
t.Fatalf("a second pass issued %d memberships", n)
|
||||
}
|
||||
}
|
||||
|
||||
// A push issuing a newer membership between the self-check's read and its write is not overwritten (issue 930).
|
||||
func TestANewerMembershipIssuedMeanwhileIsNotOverwritten(t *testing.T) {
|
||||
failed, router := verifiedSenderRecords(nil)
|
||||
bus := &fakeBus{last: map[string][]byte{}, published: map[string][]byte{}, seq: map[string]uint64{}}
|
||||
old, _ := json.Marshal(broker.MembershipFor("novox", router, broker.PlacementsOf(failed, nil)))
|
||||
bus.last["novox/messenger"], bus.seq["novox/messenger"] = old, 1
|
||||
newer := []byte(`{"node":"novox","module":"messenger","newer":true}`)
|
||||
bus.between = func() { bus.last["novox/messenger"], bus.seq["novox/messenger"] = newer, 2 }
|
||||
passing, _ := verifiedSenderRecords(map[string]bool{"novox": true})
|
||||
if n := reissueVerifiedSenders(passing, broker.PlacementsOf(passing, nil), bus.get, bus.put, func(string, ...any) {}); n != 0 {
|
||||
t.Fatalf("issued %d over a newer membership", n)
|
||||
}
|
||||
if string(bus.last["novox/messenger"]) != string(newer) || len(bus.published) != 0 {
|
||||
t.Fatalf("the newer membership was overwritten: %s", bus.last["novox/messenger"])
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,10 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
@@ -11,7 +13,9 @@ import (
|
||||
// related when they share a repository, a module, a delivery group's head, or an edge of the module graph
|
||||
// between their modules. A module is every one the walk builds or sends: the modules its merges moved and the
|
||||
// modules that depend on them, not only the ones the merges touched. Unrelated changes may walk beside each
|
||||
// other (ADR 0304 decision 2, ADR 0282 decision 3); nothing walks so yet, and nothing but tests reads this.
|
||||
// other (ADR 0304 decision 2, ADR 0282 decision 3); a closed batch is cut beside an open walk it is
|
||||
// not related to, and a walk about to ask its first tier waits only for a started walk it is related to (issue 973,
|
||||
// ADR 0407 decision 4).
|
||||
|
||||
// footprint is what one change reaches: its repositories (lower case), its modules, and its delivery group
|
||||
// heads (the head branch names of its merges). unknown says a merge of it was kept without the modules it
|
||||
@@ -109,3 +113,83 @@ func firstShared(a, b map[string]bool) string {
|
||||
sort.Strings(both)
|
||||
return both[0]
|
||||
}
|
||||
|
||||
// openWalk is a walk open beside a change: its id, what it reaches, and whether it is on the controller's own path.
|
||||
type openWalk struct {
|
||||
id string
|
||||
fp footprint
|
||||
own bool
|
||||
}
|
||||
|
||||
// relatedOpen is the first open walk a change is related to, with the reason; empty when it is related to none
|
||||
// and may walk beside them all (issue 973, ADR 0407 decision 4, ADR 0306 decision 3).
|
||||
//
|
||||
// **The controller's own path is related to everything** (decided for issue 973): a walk or batch that replaces
|
||||
// the controller, the node-engine, the tool runner, the bus or mesh-delivery never runs beside another, and
|
||||
// nothing runs beside it, whatever the footprints say.
|
||||
func relatedOpen(cand footprint, candOwn bool, open []openWalk, edges []inventory.Edge) (behind, why string) {
|
||||
for _, w := range open {
|
||||
if candOwn {
|
||||
return w.id, "it is on the controller's own path"
|
||||
}
|
||||
if w.own {
|
||||
return w.id, w.id + " is on the controller's own path"
|
||||
}
|
||||
if ok, why := related(cand, w.fp, edges); ok {
|
||||
return w.id, why
|
||||
}
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
|
||||
// ownWalk says a walk is on the controller's own path: its batch says so, or it plans a module of that path
|
||||
// (issue 973).
|
||||
func ownWalk(p inventory.Plan) bool {
|
||||
if p.OwnPath() {
|
||||
return true
|
||||
}
|
||||
for name := range p.Modules {
|
||||
if _, own := onTheControllersPath[name]; own {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// startedWalks are the open walks started beside the walk id, made at created: a merge's walk past its wait, not
|
||||
// the backlog's, with a tier asked, or on its way (let go, or waiting for nobody) before this one.
|
||||
func startedWalks(plans []inventory.Plan, id string, created time.Time) []inventory.Plan {
|
||||
var out []inventory.Plan
|
||||
for _, q := range plans {
|
||||
if q.ID == id || q.Release != nil || q.Waiting() {
|
||||
continue
|
||||
}
|
||||
if q.Tier > 0 || askedAny(q) || q.Created.Before(created) {
|
||||
out = append(out, q)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// openWalkOf is an open walk as a change is judged against it: what it reaches, from its planned modules and the
|
||||
// merges the record holds for it.
|
||||
func openWalkOf(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) (openWalk, error) {
|
||||
kept, err := inv.MergesOf(ctx, p.ID)
|
||||
if err != nil {
|
||||
return openWalk{}, err
|
||||
}
|
||||
return openWalk{id: p.ID, fp: footprintOfWalk(p, kept), own: ownWalk(p)}, nil
|
||||
}
|
||||
|
||||
// openWalksOf are openWalkOf for each of plans.
|
||||
func openWalksOf(ctx context.Context, inv *inventory.Inventory, plans []inventory.Plan) ([]openWalk, error) {
|
||||
var out []openWalk
|
||||
for _, p := range plans {
|
||||
w, err := openWalkOf(ctx, inv, p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, w)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -2,8 +2,11 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
@@ -108,3 +111,125 @@ func TestABatchWhoseMovesAreNotKnownIsRelatedToEverything(t *testing.T) {
|
||||
t.Fatalf("a known, unrelated batch read as related: %q", why)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRelatedOpenNamesTheFirstOpenWalkAChangeIsRelatedTo: a change that shares nothing with any open walk is
|
||||
// held behind none (issue 973, ADR 0407 decision 4); one that shares a repository, a module, a graph edge or a
|
||||
// group head is held behind the first walk it shares it with, with the reason; one whose moves are not known
|
||||
// is held behind the first; a walk or change on the controller's own path is related to every other.
|
||||
func TestRelatedOpenNamesTheFirstOpenWalkAChangeIsRelatedTo(t *testing.T) {
|
||||
edges := []inventory.Edge{{From: "nzbget", To: "base-image", Kind: inventory.EdgeStandsOn}}
|
||||
walk := func(id, repo string, own bool, modules ...string) openWalk {
|
||||
return openWalk{id: id, fp: footprintOfWalk(walkOf(repo, modules...), nil), own: own}
|
||||
}
|
||||
one, two := walk("plan-1", "novox/one", false, "notes"), walk("plan-2", "novox/two", false, "nzbget")
|
||||
cand := func(repo, head string, moves ...string) footprint {
|
||||
return footprintOfBatch([]inventory.BatchedMerge{heardMerge(repo, head, moves...)}, edges)
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
fp footprint
|
||||
own bool
|
||||
open []openWalk
|
||||
behind string
|
||||
why string
|
||||
}{
|
||||
{"nothing open", cand("novox/x", "fix/a", "app"), false, nil, "", ""},
|
||||
{"unrelated to every open walk", cand("novox/x", "fix/a", "app"), false, []openWalk{one, two}, "", ""},
|
||||
{"a repository", cand("novox/ONE", "fix/a", "app"), false, []openWalk{two, one}, "plan-1", "both change novox/one"},
|
||||
{"a module through its dependents", cand("novox/x", "fix/a", "base-image"), false, []openWalk{one, two},
|
||||
"plan-2", "both move nzbget"},
|
||||
{"a graph edge", cand("novox/x", "fix/a", "nzbget"),
|
||||
false, []openWalk{one, walk("plan-3", "novox/y", false, "base-image")}, "plan-3",
|
||||
"nzbget depends on base-image (stands-on)"},
|
||||
{"moves not known", cand("novox/x", "fix/a"), false, []openWalk{one, two}, "plan-1",
|
||||
"the modules a merge of novox/x moves are not known until its batch is cut"},
|
||||
{"the candidate on the own path", cand("novox/x", "fix/a", "app"), true, []openWalk{one}, "plan-1",
|
||||
"it is on the controller's own path"},
|
||||
{"an open walk on the own path", cand("novox/x", "fix/a", "app"), false,
|
||||
[]openWalk{one, walk("plan-4", "novox/mesh-controller", true, "mesh-controller")}, "plan-4",
|
||||
"plan-4 is on the controller's own path"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
behind, why := relatedOpen(c.fp, c.own, c.open, edges)
|
||||
if behind != c.behind || why != c.why {
|
||||
t.Errorf("relatedOpen = %q, %q; want %q, %q", behind, why, c.behind, c.why)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A walk is on the controller's own path when its batch says so or it plans a module of that path (issue 973).
|
||||
func TestAWalkOnTheControllersPathIsKnownByItsFlagOrItsModules(t *testing.T) {
|
||||
if ownWalk(walkOf("novox/one", "notes")) {
|
||||
t.Error("an ordinary walk read as on the controller's own path")
|
||||
}
|
||||
flagged := walkOf("novox/one", "notes")
|
||||
flagged.Delivery = &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: true}}
|
||||
if !ownWalk(flagged) {
|
||||
t.Error("a walk whose batch is own-path read as ordinary")
|
||||
}
|
||||
var module string
|
||||
for m := range onTheControllersPath {
|
||||
module = m
|
||||
}
|
||||
if !ownWalk(walkOf("novox/one", "notes", module)) {
|
||||
t.Errorf("a walk planning %s read as ordinary", module)
|
||||
}
|
||||
}
|
||||
|
||||
// A walk about to ask its first tier waits only for a started walk it is related to (issue 973): started is a
|
||||
// tier asked, or a walk made before it; a waiting walk, the backlog's and itself are not.
|
||||
func TestWalksStartedBesideAreTheOpenOnesBeforeIt(t *testing.T) {
|
||||
at := time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC)
|
||||
mk := func(id string, created time.Time, tier int) inventory.Plan {
|
||||
p := walkOf("novox/"+id, id)
|
||||
p.ID, p.Created, p.Tier, p.State = id, created, tier, inventory.PlanBuilding
|
||||
return p
|
||||
}
|
||||
older, newer, ticking := mk("older", at, 0), mk("newer", at.Add(time.Minute), 0), mk("ticking", at.Add(2*time.Minute), 1)
|
||||
backlog := mk("backlog", at, 1)
|
||||
backlog.Release = &inventory.PlanRelease{}
|
||||
got := startedWalks([]inventory.Plan{older, newer, ticking, backlog, mk("me", at.Add(time.Minute), 0)}, "me", at.Add(time.Minute))
|
||||
var ids []string
|
||||
for _, p := range got {
|
||||
ids = append(ids, p.ID)
|
||||
}
|
||||
if strings.Join(ids, ",") != "older,ticking" {
|
||||
t.Errorf("started beside me = %v; want older, ticking", ids)
|
||||
}
|
||||
}
|
||||
|
||||
// A retry is refused beside an open walk it is related to and allowed beside one it shares nothing with
|
||||
// (issue 973); a retry on the controller's own path is refused beside any open walk.
|
||||
func TestARetryWaitsOnlyForAnOpenWalkItIsRelatedTo(t *testing.T) {
|
||||
failed := walkOf("novox/one", "notes")
|
||||
failed.ID, failed.State = "plan-1", inventory.PlanFailed
|
||||
open := func(id, repo string, modules ...string) inventory.Plan {
|
||||
p := walkOf(repo, modules...)
|
||||
p.ID, p.State = id, inventory.PlanBuilding
|
||||
return p
|
||||
}
|
||||
if err := oneWalkAtATime(failed, []inventory.Plan{open("plan-2", "novox/two", "app")}); err != nil {
|
||||
t.Errorf("refused beside an unrelated walk: %v", err)
|
||||
}
|
||||
err := oneWalkAtATime(failed, []inventory.Plan{open("plan-3", "novox/three", "notes")})
|
||||
if err == nil || !strings.Contains(err.Error(), "plan-3") || !strings.Contains(err.Error(), "both move notes") {
|
||||
t.Errorf("a retry beside a walk moving the same module: %v", err)
|
||||
}
|
||||
failed.Delivery = &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: true}}
|
||||
if err := oneWalkAtATime(failed, []inventory.Plan{open("plan-2", "novox/two", "app")}); err == nil {
|
||||
t.Error("an own-path retry went beside an open walk")
|
||||
}
|
||||
}
|
||||
|
||||
// Plans another controller is working are "not now", not a failure: the merge is held and handed over again
|
||||
// (issue 921).
|
||||
func TestPlansHeldByAnotherControllerAreNotNow(t *testing.T) {
|
||||
if err := notNow(fmt.Errorf("x: %w", inventory.ErrPlansBusy)); !errors.Is(err, link.ErrTryAgain) {
|
||||
t.Errorf("busy plans read as %v; want ErrTryAgain", err)
|
||||
}
|
||||
if err := notNow(errors.New("no such module")); errors.Is(err, link.ErrTryAgain) {
|
||||
t.Errorf("another failure read as ErrTryAgain: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1023,12 +1023,12 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time.
|
||||
outcome, reported = provisionedOutcome(node, sent.Only(carried...), byNode[node], reported, provLookup(ctx))
|
||||
}
|
||||
if !reported {
|
||||
if now.Sub(*g.Since) > inventory.ApplySilentAfter {
|
||||
if now.Sub(*g.Since) > applySilentAfter {
|
||||
g.Silent = true
|
||||
r.Silent = append(r.Silent, node)
|
||||
changed = true
|
||||
notes = append(notes, fmt.Sprintf("%s said nothing on its send within %s: left silent", node,
|
||||
inventory.ApplySilentAfter))
|
||||
applySilentAfter))
|
||||
continue
|
||||
}
|
||||
judging = append(judging, node+": not yet reported on its send")
|
||||
|
||||
@@ -24,9 +24,9 @@ import (
|
||||
//
|
||||
// The handler that hears the merge computes the plan from the catalogue's one dependency relation,
|
||||
// writes it to the store, asks the first tier and returns — the receive loop is never held by a
|
||||
// build. Every outcome taken in advances the plan it belongs to; a ticker advances what outcomes
|
||||
// alone cannot (a tier waiting for machines to report); a controller replaced mid-plan finds the
|
||||
// plan where it left it.
|
||||
// build. Every outcome taken in advances the plan it belongs to; a look at the plans, asked for by each event that can
|
||||
// move one (plan_looks.go, ADR 0406 §5), advances what outcomes alone cannot (a tier waiting for machines to report);
|
||||
// a controller replaced mid-plan finds the plan where it left it and looks first.
|
||||
|
||||
// gateFirstMachine sends a module that rolls out to one machine first and judges it there (ADR 0236). Off since
|
||||
// novox/hq ADR 0360: the gate's code is kept for the walks a controller before it started, and its tests turn it on.
|
||||
@@ -618,23 +618,32 @@ func planBuiltSaying(ctx context.Context, open *stores, module, commit, failed s
|
||||
advanceHeld(ctx, open)
|
||||
}
|
||||
|
||||
// startedBeside is the id of a started walk open beside this one — a merge's walk past its wait, not the
|
||||
// backlog's — or empty: one walk at a time (novox/hq ADR 0276).
|
||||
func startedBeside(ctx context.Context, inv *inventory.Inventory, id string, created time.Time) (string, error) {
|
||||
// startedBeside is the id of a started walk open beside this one that it is related to — a merge's walk past
|
||||
// its wait, not the backlog's — with why, or empty: a walk waits only for a started walk it shares something with
|
||||
// (novox/hq ADR 0276 decision 4, ADR 0407 decision 4, issue 973).
|
||||
func startedBeside(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) (behind, why string, err error) {
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", "", err
|
||||
}
|
||||
for _, q := range plans {
|
||||
if q.ID == id || q.Release != nil || q.Waiting() {
|
||||
continue
|
||||
}
|
||||
// Started: a tier asked, or on its way (let go, or waiting for nobody) before this one.
|
||||
if q.Tier > 0 || askedAny(q) || q.Created.Before(created) {
|
||||
return q.ID, nil
|
||||
}
|
||||
started := startedWalks(plans, p.ID, p.Created)
|
||||
if len(started) == 0 {
|
||||
return "", "", nil
|
||||
}
|
||||
return "", nil
|
||||
edges, err := inv.Dependencies(ctx)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
open, err := openWalksOf(ctx, inv, started)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
mine, err := openWalkOf(ctx, inv, p)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
behind, why = relatedOpen(mine.fp, mine.own, open, edges)
|
||||
return behind, why, nil
|
||||
}
|
||||
|
||||
// deferredNote begins the note of a walk deferred behind another.
|
||||
@@ -659,29 +668,6 @@ func askedAny(p inventory.Plan) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// advancePlans moves every open plan as far as the facts allow: a tier whose modules are all built
|
||||
// and whose gates are applied gives way to the next; the last tier done is the plan done. Called
|
||||
// after every outcome and on a timer, so a plan waiting on a machine's report moves when it comes.
|
||||
//
|
||||
// **One controller at a time** (novox/hq issue 213). A plan is read, changed and saved whole; two
|
||||
// controllers — the old and the new while a machine hands its controller over — would each ask a
|
||||
// tier the other had just asked. Taken without waiting: whoever holds the plans is moving them.
|
||||
func advancePlans(ctx context.Context, open *stores) {
|
||||
release, err := open.inventory.HoldPlans(ctx, false)
|
||||
if err != nil {
|
||||
if !errors.Is(err, inventory.ErrPlansBusy) {
|
||||
fmt.Printf("plans: cannot hold them: %v\n", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
// What waits for a gate, released one machine first, then the rest at once (ADR 0236, ADR 0353).
|
||||
if _, err := releaseBacklog(ctx, open, ""); err != nil {
|
||||
fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err)
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
}
|
||||
|
||||
// keepWalkPhases keeps where the walks that ended lately spent their time (novox/hq ADR 0282), outside the hold
|
||||
// on the plans so it never lengthens it: measured, never acted on, and an error only said.
|
||||
func keepWalkPhases(ctx context.Context, open *stores) {
|
||||
@@ -690,10 +676,15 @@ func keepWalkPhases(ctx context.Context, open *stores) {
|
||||
}
|
||||
}
|
||||
|
||||
// advanceHeld is advancePlans for a caller already holding the plans.
|
||||
func advanceHeld(ctx context.Context, open *stores) {
|
||||
// advanceHeld moves every open plan as far as the facts allow, for a caller holding the plans: a tier whose modules
|
||||
// are all built and whose gates are applied gives way to the next; the last tier done is the plan done. It says
|
||||
// whether a step failed, which the look at the plans (lookAtPlans) retries.
|
||||
//
|
||||
// **One controller at a time** (novox/hq issue 213). A plan is read, changed and saved whole; two controllers — the
|
||||
// old and the new while a machine hands its controller over — would each ask a tier the other had just asked.
|
||||
func advanceHeld(ctx context.Context, open *stores) (failed bool) {
|
||||
inv := open.inventory
|
||||
// A walk that ended lets the next batch be cut at once, not at the cutter's next look (novox/hq ADR 0276).
|
||||
// A walk that ended lets the next batch be cut at once, not at a window's next expiry (novox/hq ADR 0276, ADR 0407).
|
||||
defer func() {
|
||||
if err := cutBatchesHeld(ctx, open, time.Now().UTC()); err != nil {
|
||||
fmt.Printf("batches: %v\n", err)
|
||||
@@ -702,15 +693,15 @@ func advanceHeld(ctx context.Context, open *stores) {
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("plans: cannot read them: %v\n", err)
|
||||
return
|
||||
return true
|
||||
}
|
||||
if len(plans) == 0 {
|
||||
return
|
||||
return false
|
||||
}
|
||||
edges, err := inv.Dependencies(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("plans: cannot read the dependencies: %v\n", err)
|
||||
return
|
||||
return true
|
||||
}
|
||||
rollsOut := func(module string) bool {
|
||||
u, err := inv.UpgradeOf(ctx, module)
|
||||
@@ -724,10 +715,12 @@ func advanceHeld(ctx context.Context, open *stores) {
|
||||
// stood still for half an hour was written every few seconds, its revision moved, `plan-moved`
|
||||
// said on the bus, and its last save read as when it began building.
|
||||
before := planSnapshot(*p)
|
||||
moved, err := advanceOnce(ctx, open, p, edges, rollsOut)
|
||||
sends := &sendsMade{}
|
||||
moved, err := advanceOnce(withSendsMade(ctx, sends), open, p, edges, rollsOut)
|
||||
if err != nil {
|
||||
failed = true
|
||||
// Kept in the plan, so `plans` says why it has not moved rather than the log alone;
|
||||
// the state is left as it was and the step is tried again on the next tick. Said and
|
||||
// the state is left as it was and the step is tried again by the next look. Said and
|
||||
// kept when it is new: the same refusal on every tick is one fact, not one per tick.
|
||||
p.Note = "tier " + fmt.Sprint(p.Tier) + ": " + err.Error() + " — tried again"
|
||||
// A refusal for the bus's planned step is a person's to end: S17 says it from its first moment.
|
||||
@@ -748,14 +741,18 @@ func advanceHeld(ctx context.Context, open *stores) {
|
||||
if planSnapshot(*p) != before {
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
|
||||
failed = true
|
||||
break
|
||||
}
|
||||
// Said once the send is kept (ADR 0406 §5, issue 974): what an outsider hears of a tier sent.
|
||||
saySends(ctx, *p, sends)
|
||||
}
|
||||
if !moved {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return failed
|
||||
}
|
||||
|
||||
// planSnapshot is a plan as a save would write it, to compare before and after a step: what the store
|
||||
@@ -793,13 +790,14 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
tier := p.Tiers[p.Tier]
|
||||
// Not yet asked: ask — the whole tier, or what of it is still unasked (novox/hq issue 754).
|
||||
if unasked := unaskedOf(*p); len(unasked) > 0 {
|
||||
// **One walk at a time** (novox/hq ADR 0276): a walk about to ask its first tier while another started
|
||||
// walk is open waits for that one to end, let go or not, and says so.
|
||||
// **One walk at a time beside a related one** (novox/hq ADR 0276, ADR 0407 decision 4, issue 973): a walk
|
||||
// about to ask its first tier while a started walk it is related to is open waits for that one to end,
|
||||
// let go or not, and says so and why. A started walk it shares nothing with does not hold it.
|
||||
if p.Tier == 0 && !askedInTier(*p) {
|
||||
if behind, err := startedBeside(ctx, inv, p.ID, p.Created); err != nil {
|
||||
if behind, why, err := startedBeside(ctx, inv, *p); err != nil {
|
||||
return false, err
|
||||
} else if behind != "" {
|
||||
note := fmt.Sprintf("%s%s ended — one walk at a time", deferredNote, behind)
|
||||
note := fmt.Sprintf("%s%s ended — %s", deferredNote, behind, why)
|
||||
changed := p.Note != note
|
||||
p.Note = note
|
||||
return changed, nil
|
||||
@@ -1056,6 +1054,16 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
p.Modules[m].SentAt = &now
|
||||
p.Modules[m].Rest = restOf(restTo[m], sent, sentWhat)
|
||||
}
|
||||
// One `sent` for each machine, with what it was sent (ADR 0406 §5, issue 974).
|
||||
for _, n := range sent {
|
||||
var carried []string
|
||||
for _, m := range rest {
|
||||
if slices.Contains(restTo[m], n) {
|
||||
carried = append(carried, m)
|
||||
}
|
||||
}
|
||||
noteSends(ctx, p.Tier, []string{n}, carried, now)
|
||||
}
|
||||
fmt.Printf("%s: tier %d built; sent %s to %s, one send each\n", p.ID, p.Tier, strings.Join(rest, ", "),
|
||||
strings.Join(sent, ", "))
|
||||
return true, nil
|
||||
@@ -1166,7 +1174,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
said = append(said, "failed or refused its send (apply-failing says it): "+strings.Join(failed, ", "))
|
||||
}
|
||||
if len(silent) > 0 {
|
||||
said = append(said, fmt.Sprintf("said nothing on its send within %s: %s", inventory.ApplySilentAfter,
|
||||
said = append(said, fmt.Sprintf("said nothing on its send within %s: %s", applySilentAfter,
|
||||
strings.Join(silent, ", ")))
|
||||
}
|
||||
said = append(said, recordedSaid(*p)...)
|
||||
@@ -1231,7 +1239,7 @@ func walkReports(p inventory.Plan, reports []inventory.Reported, now time.Time,
|
||||
case strings.HasPrefix(outcome, inventory.OutcomeProvisioningFailed):
|
||||
failed = append(failed, node+" ("+outcome+")")
|
||||
}
|
||||
case !at.IsZero() && now.Sub(at) > inventory.ApplySilentAfter:
|
||||
case !at.IsZero() && now.Sub(at) > applySilentAfter:
|
||||
seen[node] = true
|
||||
silent = append(silent, node)
|
||||
default:
|
||||
@@ -1328,6 +1336,7 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
||||
s.GatedBy = lead
|
||||
}
|
||||
}
|
||||
noteSends(ctx, p.Tier, sent, modules, now)
|
||||
p.State = inventory.PlanRolling
|
||||
p.Note = fmt.Sprintf("tier %d built; sent %s to %s first, in one send", p.Tier, strings.Join(modules, ", "),
|
||||
strings.Join(sent, ", "))
|
||||
@@ -1642,23 +1651,6 @@ func sayUnsent(p *inventory.Plan, rollsOut func(string) bool) {
|
||||
p.Note += "; built and never sent: " + strings.Join(unsent, ", ") + " — `push --behind` sends them"
|
||||
}
|
||||
|
||||
// planTicker advances open plans on a timer, for the steps outcomes alone cannot take.
|
||||
func planTicker(ctx context.Context, open *stores) {
|
||||
advancePlans(ctx, open)
|
||||
keepWalkPhases(ctx, open)
|
||||
tick := time.NewTicker(30 * time.Second)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
advancePlans(ctx, open)
|
||||
keepWalkPhases(ctx, open)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// planLine is one plan as `status` says it, given the least bound of a tier.
|
||||
func planLine(p inventory.Plan, now time.Time) string {
|
||||
return planLineWith(p, now, pauseView{}, tierAtLeast)
|
||||
@@ -1722,7 +1714,7 @@ func planLineSaid(p inventory.Plan, now time.Time, pause pauseView, bound time.D
|
||||
return fmt.Sprintf("%s · %s, %s, for %s", name, where, waitingSaid(p, waits[p.ID]), since)
|
||||
}
|
||||
if deferred(p) {
|
||||
// Nor is waiting for the walk before it to end (one walk at a time, novox/hq ADR 0276).
|
||||
// Nor is waiting for the walk before it to end (a related walk, novox/hq ADR 0276, ADR 0407).
|
||||
return fmt.Sprintf("%s · %s, %s, for %s", name, where, p.Note, since)
|
||||
}
|
||||
if waiting, paused := pausedWaiting(p, pause, now); paused {
|
||||
@@ -2024,7 +2016,7 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s (%s at %s) is let go by hand; its first tier is asked at the next pass\n", p.ID, p.Repository,
|
||||
fmt.Printf("%s (%s at %s) is let go by hand; its first tier is asked when the controller next looks at the plans, which it does as this ends\n", p.ID, p.Repository,
|
||||
short(p.Commit))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,345 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// Acceptance of the step (novox/hq ADR 0406 §5, §6, acceptance 7; issue 974), live, with the store and a real bus and no
|
||||
// ticker: a rollout of two tiers advances on the events alone — the plan kept, the machines' reports, a silent key's
|
||||
// expiry — a machine that reports in time is never named silent and one that never reports is named silent when its
|
||||
// key expires, and a controller replaced in the middle of it resumes with no tier asked twice and none skipped.
|
||||
func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
|
||||
// A quarter of an hour of silence is two seconds here.
|
||||
wasSilent := applySilentAfter
|
||||
applySilentAfter = 2 * time.Second
|
||||
t.Cleanup(func() { applySilentAfter = wasSilent })
|
||||
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
legacy, err := js.Conn().JetStream()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy}
|
||||
t.Cleanup(func() { checkEvents = nil })
|
||||
sentHeard := make(chan *nats.Msg, 32)
|
||||
sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeySent), sentHeard)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
|
||||
// Every save of a plan is kept, said and looked at by the serving controller's own hook.
|
||||
saved := &movesSaid{revs: map[string][]int64{}}
|
||||
wasSaved := inventory.PlanSaved
|
||||
serving := servingPlanSaved(ctx, checkEvents, open)
|
||||
inventory.PlanSaved = func(i *inventory.Inventory, p inventory.Plan) { saved.hear(i, p); serving(i, p) }
|
||||
t.Cleanup(func() { inventory.PlanSaved = wasSaved })
|
||||
|
||||
// The build machine and the machines, as far as the plan meets them: an ask is counted, a send is recorded and kept.
|
||||
var mu sync.Mutex
|
||||
var asks []string
|
||||
var sends [][]string
|
||||
wasAsk := askABuild
|
||||
askABuild = func(_ context.Context, source buildSource, _, _ string) (string, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
id := link.NewBuildID(time.Now().Add(time.Duration(len(asks)) * time.Millisecond))
|
||||
asks = append(asks, source.Repository+"#"+id)
|
||||
return id, nil
|
||||
}
|
||||
t.Cleanup(func() { askABuild = wasAsk })
|
||||
digests := 0
|
||||
wasSend := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
sends = append(sends, append([]string(nil), names...))
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
builds := map[string]string{}
|
||||
for _, m := range []string{"a", "b"} {
|
||||
builds[m] = current[m].Commit
|
||||
}
|
||||
for _, node := range names {
|
||||
digests++
|
||||
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), fmt.Sprintf("d-%s-%d", node, digests), builds); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = wasSend })
|
||||
|
||||
register := func(m, commit string, asked time.Time) {
|
||||
t.Helper()
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: commit},
|
||||
inventory.Source{Repository: "novox/" + m, Seat: "git", Path: "modules/" + m, BuiltFrom: commit, Head: commit,
|
||||
Asked: asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
record := func(id, m, commit string, at time.Time) {
|
||||
t.Helper()
|
||||
b := inventory.Build{ID: id, Module: m, Commit: commit, Repository: "novox/" + m, Path: "modules/" + m, Asked: at, At: at}
|
||||
b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: m, Version: commit})
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
old := time.Now().Add(-2 * time.Hour)
|
||||
for _, m := range []string{"a", "b"} {
|
||||
record("build-"+m+"-1", m, "c1", old)
|
||||
register(m, "c1", old)
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.Assign(ctx, n, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"a": "c1", "b": "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
record("build-a-2", "a", "c2", time.Now().Add(-time.Minute))
|
||||
register("a", "c2", time.Now().Add(-time.Minute))
|
||||
|
||||
// A machine's report on what it was sent, heard as the controller hears it.
|
||||
listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}}
|
||||
seq := int64(0)
|
||||
reports := func(node string) {
|
||||
t.Helper()
|
||||
last, found, err := inv.SentTo(ctx, node)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("%s was sent nothing: %v", node, err)
|
||||
}
|
||||
seq++
|
||||
body, _ := json.Marshal(map[string]any{"node": node, "applied": []string{"x.server"}, "declared": last.Digest,
|
||||
"report_sequence": seq})
|
||||
var report link.Report
|
||||
if err := json.Unmarshal(body, &report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := listener.Heard(ctx, report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
heldKeys := func() map[silentWait]time.Time {
|
||||
c := silentClockOf()
|
||||
if c == nil {
|
||||
return nil
|
||||
}
|
||||
held, err := c.Held(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return held
|
||||
}
|
||||
plan := func() inventory.Plan {
|
||||
t.Helper()
|
||||
p, err := inv.PlanByID(ctx, "plan-974")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// The first controller. Its plan has tier 0 built and tier 1 not asked: nothing but the save asks anything.
|
||||
first, stopFirst := context.WithCancel(ctx)
|
||||
defer stopFirst()
|
||||
startRolloutEvents(first, open, js)
|
||||
built := time.Now().UTC()
|
||||
p := inventory.Plan{ID: "plan-974", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", Created: built,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Modules: map[string]*inventory.PlanModule{
|
||||
"a": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-a-2"},
|
||||
"b": {},
|
||||
}}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
eventually(t, "tier 0 sent to both machines and tier 1 asked, on the save alone", func() bool {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return len(sends) == 1 && len(asks) == 1 && plan().Tier == 1
|
||||
})
|
||||
eventually(t, "a key on the bus for each machine tier 0 was sent to", func() bool { return len(heldKeys()) == 2 })
|
||||
|
||||
// The controller is replaced: its looks and its watcher end. anchor reports meanwhile, to nobody.
|
||||
stopFirst()
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
reports("anchor")
|
||||
|
||||
second, stopSecond := context.WithCancel(ctx)
|
||||
defer stopSecond()
|
||||
startRolloutEvents(second, open, js)
|
||||
// Its first look finds anchor reported, so its key goes; laptop's stays, and is the new controller's to hear expire.
|
||||
eventually(t, "the new controller's first look deletes the key of the machine that reported", func() bool {
|
||||
h := heldKeys()
|
||||
_, anchor := h[silentWait{Plan: "plan-974", Tier: 0, Node: "anchor"}]
|
||||
return !anchor
|
||||
})
|
||||
|
||||
// tier 1's build comes in, as the build machine's outcome is taken in.
|
||||
b := plan().Modules["b"]
|
||||
if b == nil || b.Build == "" {
|
||||
t.Fatalf("tier 1 was not asked: %+v", b)
|
||||
}
|
||||
askedAt, _ := link.BuildAskedAt(b.Build)
|
||||
record(b.Build, "b", "c2", askedAt)
|
||||
register("b", "c2", askedAt)
|
||||
planBuilt(ctx, open, "b", "c2", "", askedAt, b.Build)
|
||||
eventually(t, "tier 1 sent to both machines", func() bool {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return len(sends) == 2
|
||||
})
|
||||
reports("anchor")
|
||||
|
||||
// laptop never reports: its key expires and the new controller names it silent, which ends the rollout.
|
||||
eventually(t, "the rollout ended, naming laptop silent and not anchor", func() bool {
|
||||
p := plan()
|
||||
return p.State == inventory.PlanDone
|
||||
})
|
||||
p = plan()
|
||||
if !strings.Contains(p.Note, "laptop") || strings.Contains(p.Note, "anchor") || !strings.Contains(p.Note, "said nothing") {
|
||||
t.Errorf("the plan ended with the note %q; want laptop named silent and not anchor", p.Note)
|
||||
}
|
||||
eventually(t, "no key left on the bus", func() bool { return len(heldKeys()) == 0 })
|
||||
|
||||
// No tier asked twice and none skipped: one ask for tier 1 (tier 0 was built when the plan was kept), each tier sent
|
||||
// once to both machines, and said as sent once for each.
|
||||
mu.Lock()
|
||||
if len(asks) != 1 || !strings.HasPrefix(asks[0], "novox/b#") {
|
||||
t.Errorf("asks = %v; want tier 1's module asked once", asks)
|
||||
}
|
||||
if len(sends) != 2 || strings.Join(sends[0], ",") != "anchor,laptop" || strings.Join(sends[1], ",") != "anchor,laptop" {
|
||||
t.Errorf("sends = %v; want both machines sent each tier once", sends)
|
||||
}
|
||||
mu.Unlock()
|
||||
said := map[string]int{}
|
||||
for deadline := time.After(5 * time.Second); len(said) < 4; {
|
||||
select {
|
||||
case m := <-sentHeard:
|
||||
var body planSentBody
|
||||
if err := json.Unmarshal(m.Data, &body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said[fmt.Sprintf("%d/%s", body.Tier, body.Node)]++
|
||||
case <-deadline:
|
||||
t.Fatalf("sent was said for %v; want tiers 0 and 1 to both machines", said)
|
||||
}
|
||||
}
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
for len(sentHeard) > 0 {
|
||||
var body planSentBody
|
||||
_ = json.Unmarshal((<-sentHeard).Data, &body)
|
||||
said[fmt.Sprintf("%d/%s", body.Tier, body.Node)]++
|
||||
}
|
||||
for k, n := range said {
|
||||
if n != 1 {
|
||||
t.Errorf("sent for %s was said %d times", k, n)
|
||||
}
|
||||
}
|
||||
// And every move of the plan was said as plan-moved.
|
||||
saved.noUnsaid(t, open)
|
||||
}
|
||||
|
||||
// Acceptance of the verb path (issue 974, review 1), live, with the store and a bus: a walk the delivery's owner lets
|
||||
// go by the verb `deliver` after the operator's approval is asked its first tier with no other event. The verb runs as a
|
||||
// child process, whose saves the serving controller does not hear; the handler asks the look as the verb ends.
|
||||
func TestAWalkLetGoByTheDeliverVerbIsAskedWithNoOtherEvent(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
legacy, err := js.Conn().JetStream()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy}
|
||||
t.Cleanup(func() { checkEvents = nil })
|
||||
asked := asksRecorded(t)
|
||||
register(t, open, catalogue.Manifest{Module: "app", Version: "1"})
|
||||
if _, err := assign(ctx, open, "anchor", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cut := time.Now().UTC()
|
||||
waiting := inventory.Plan{ID: "plan-974-deliver", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c8c8c8c8",
|
||||
Created: cut, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
|
||||
Modules: map[string]*inventory.PlanModule{"app": {}},
|
||||
Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat},
|
||||
Times: &inventory.PlanTimes{Cut: &cut, Class: inventory.ClassLeaf}}
|
||||
if err := open.inventory.SavePlan(ctx, &waiting); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The serving controller, as serve starts it.
|
||||
wasSaved := inventory.PlanSaved
|
||||
inventory.PlanSaved = servingPlanSaved(ctx, checkEvents, open)
|
||||
t.Cleanup(func() { inventory.PlanSaved = wasSaved })
|
||||
started, stop := context.WithCancel(ctx)
|
||||
defer stop()
|
||||
startRolloutEvents(started, open, js)
|
||||
time.Sleep(500 * time.Millisecond) // the first look: the walk waits, and is asked nothing
|
||||
|
||||
// The verb as its child process runs it: the delivery command, whose saves are not the serving controller's.
|
||||
wasRun := runTheVerb
|
||||
runTheVerb = func(ctx context.Context, argv []string) (verbAnswer, error) {
|
||||
was := inventory.PlanSaved
|
||||
inventory.PlanSaved = func(*inventory.Inventory, inventory.Plan) {}
|
||||
defer func() { inventory.PlanSaved = was }()
|
||||
return verbAnswer{}, deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"})
|
||||
}
|
||||
t.Cleanup(func() { runTheVerb = wasRun })
|
||||
handlers, _, err := seatToolHandlers()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := handlers["deliver"](ctx, []byte(`{"plan":"`+waiting.ID+`","why":"its turn"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
eventually(t, "the walk's first tier asked, with no event but the verb's end", func() bool {
|
||||
p, err := open.inventory.PlanByID(ctx, waiting.ID)
|
||||
return err == nil && p.Modules["app"] != nil && p.Modules["app"].Build != "" && len(*asked) == 1
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// A rollout's wait for a machine is a key on the bus (novox/hq ADR 0406 §5, issue 974): each send to a machine puts
|
||||
// a key `<plan>.<tier>.<node>` that lives as long as the machine may stay silent (ApplySilentAfter); the machine's
|
||||
// report on that send deletes it, so only a missing report lets it expire; and the expiry, heard by a watcher,
|
||||
// ends the wait for that machine, which the plan then names silent. No timer looks every so often whether a
|
||||
// machine has reported.
|
||||
|
||||
// silentWait is one machine's wait on one send of a plan's tier.
|
||||
type silentWait struct {
|
||||
Plan string
|
||||
Tier int
|
||||
Node string
|
||||
}
|
||||
|
||||
// silentKey is a wait's key. A plan id may hold a dot, so it is read from the right (parseSilentKey).
|
||||
func silentKey(w silentWait) string { return w.Plan + "." + strconv.Itoa(w.Tier) + "." + w.Node }
|
||||
|
||||
// parseSilentKey reads a key back: the node is the last token, the tier the one before, the plan what is left.
|
||||
func parseSilentKey(key string) (silentWait, bool) {
|
||||
last := strings.LastIndexByte(key, '.')
|
||||
if last < 1 || last == len(key)-1 {
|
||||
return silentWait{}, false
|
||||
}
|
||||
rest, node := key[:last], key[last+1:]
|
||||
mid := strings.LastIndexByte(rest, '.')
|
||||
if mid < 1 || mid == len(rest)-1 {
|
||||
return silentWait{}, false
|
||||
}
|
||||
tier, err := strconv.Atoi(rest[mid+1:])
|
||||
if err != nil || tier < 0 {
|
||||
return silentWait{}, false
|
||||
}
|
||||
return silentWait{Plan: rest[:mid], Tier: tier, Node: node}, true
|
||||
}
|
||||
|
||||
// silentClock is the bus's side of the wait for a machine's report.
|
||||
type silentClock interface {
|
||||
// Arm puts a wait's key, to live ttl: made again if the machine is sent again. The key holds when the send was.
|
||||
Arm(ctx context.Context, w silentWait, ttl time.Duration, sent time.Time) error
|
||||
// Held is every wait that has a key now, with when its send was.
|
||||
Held(ctx context.Context) (map[silentWait]time.Time, error)
|
||||
// Reported takes a wait's key away: the machine reported on the send, and nothing is to wake the controller.
|
||||
Reported(ctx context.Context, w silentWait) error
|
||||
}
|
||||
|
||||
// busSilent is the clock on the controller's bucket.
|
||||
type busSilent struct {
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
|
||||
// newBusSilent opens the bucket and starts the watcher, which names each wait whose key expired until the context
|
||||
// ends. Only expiry wakes it: the marker of an expiry is a purge, and a key a report deleted, or one Arm made again,
|
||||
// leaves a delete (the vendored client reads the server's MaxAge marker as a purge, kv.go:971).
|
||||
func newBusSilent(ctx context.Context, api jetstream.JetStream, silent func(silentWait)) (*busSilent, error) {
|
||||
kv, err := api.KeyValue(ctx, broker.RolloutSilentBucket)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the rollout's silent keys' bucket: %w", err)
|
||||
}
|
||||
w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("watching the rollout's silent keys: %w", err)
|
||||
}
|
||||
go func() {
|
||||
defer w.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if e == nil || e.Operation() != jetstream.KeyValuePurge {
|
||||
continue // a put, or a delete: a report's, or a send made again
|
||||
}
|
||||
if wait, ok := parseSilentKey(e.Key()); ok && silent != nil {
|
||||
silent(wait)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return &busSilent{kv: kv}, nil
|
||||
}
|
||||
|
||||
func (s *busSilent) Arm(ctx context.Context, w silentWait, ttl time.Duration, sent time.Time) error {
|
||||
key := silentKey(w)
|
||||
// Made again, not changed: a key's time to live is set when it is made.
|
||||
if err := s.kv.Delete(ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return err
|
||||
}
|
||||
_, err := s.kv.Create(ctx, key, []byte(sent.UTC().Format(time.RFC3339Nano)), jetstream.KeyTTL(leastSecond(ttl)))
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *busSilent) Reported(ctx context.Context, w silentWait) error {
|
||||
if err := s.kv.Delete(ctx, silentKey(w)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// leastSecond rounds a time to live up to a whole second, at least one: the server rounds a key's time down to
|
||||
// whole seconds, so a fraction would close the key before the time it stands for.
|
||||
func leastSecond(d time.Duration) time.Duration {
|
||||
if d < time.Second {
|
||||
return time.Second
|
||||
}
|
||||
return (d + time.Second - 1) / time.Second * time.Second
|
||||
}
|
||||
|
||||
func (s *busSilent) Held(ctx context.Context) (map[silentWait]time.Time, error) {
|
||||
keys, err := s.kv.Keys(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
held := map[silentWait]time.Time{}
|
||||
for _, key := range keys {
|
||||
w, ok := parseSilentKey(key)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
e, err := s.kv.Get(ctx, key)
|
||||
if err != nil {
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
continue
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
at, _ := time.Parse(time.RFC3339Nano, string(e.Value()))
|
||||
held[w] = at
|
||||
}
|
||||
return held, nil
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// silentOnABus is the rollout's silent keys on a real bus, and the waits its watcher heard expire.
|
||||
func silentOnABus(t *testing.T) (*busSilent, <-chan silentWait) {
|
||||
t.Helper()
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conn, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(conn.Close)
|
||||
api, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
heard := make(chan silentWait, 16)
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
t.Cleanup(stop)
|
||||
s, err := newBusSilent(ctx, api, func(w silentWait) { heard <- w })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return s, heard
|
||||
}
|
||||
|
||||
func hearsSilent(t *testing.T, heard <-chan silentWait, within time.Duration) (silentWait, bool) {
|
||||
t.Helper()
|
||||
select {
|
||||
case w := <-heard:
|
||||
return w, true
|
||||
case <-time.After(within):
|
||||
return silentWait{}, false
|
||||
}
|
||||
}
|
||||
|
||||
// A machine that never reports on its send is named silent when its key expires (issue 974, novox/hq ADR 0406 §5):
|
||||
// the watcher names the plan, the tier and the machine, which a plan id or a machine name with a dot does not blur.
|
||||
func TestAKeyLeftAloneExpiresAndTheWatcherNamesTheMachineSilent(t *testing.T) {
|
||||
s, heard := silentOnABus(t)
|
||||
want := silentWait{Plan: "plan.7", Tier: 2, Node: "ace"}
|
||||
if err := s.Arm(t.Context(), want, time.Second, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, ok := hearsSilent(t, heard, 10*time.Second); !ok || got != want {
|
||||
t.Fatalf("heard %+v (%v); want %+v", got, ok, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine that reports in time is never named silent: the report deletes the key, and a delete is not an expiry.
|
||||
func TestAReportDeletesTheKeyAndNoOneIsNamedSilent(t *testing.T) {
|
||||
s, heard := silentOnABus(t)
|
||||
ctx := t.Context()
|
||||
w := silentWait{Plan: "plan-1", Tier: 0, Node: "g14"}
|
||||
if err := s.Arm(ctx, w, time.Second, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Reported(ctx, w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, ok := hearsSilent(t, heard, 2500*time.Millisecond); ok {
|
||||
t.Fatalf("%+v was named silent after it reported", got)
|
||||
}
|
||||
// Reporting twice, or on a send that put no key, is no error.
|
||||
if err := s.Reported(ctx, w); err != nil {
|
||||
t.Fatalf("a second report: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A send made again to a machine puts its key again with the full time, and only the last send's key can expire.
|
||||
func TestASendMadeAgainPutsTheKeyAgain(t *testing.T) {
|
||||
s, heard := silentOnABus(t)
|
||||
ctx := t.Context()
|
||||
w := silentWait{Plan: "plan-3", Tier: 1, Node: "novox"}
|
||||
if err := s.Arm(ctx, w, 2*time.Second, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(1200 * time.Millisecond)
|
||||
if err := s.Arm(ctx, w, 2*time.Second, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, ok := hearsSilent(t, heard, 1500*time.Millisecond); ok {
|
||||
t.Fatalf("%+v expired at the first send's time, not the second's", got)
|
||||
}
|
||||
if got, ok := hearsSilent(t, heard, 5*time.Second); !ok || got != w {
|
||||
t.Fatalf("heard %+v (%v); want %+v after the second send's time", got, ok, w)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASilentKeyIsParsedFromTheRightOfTheNode(t *testing.T) {
|
||||
for _, w := range []silentWait{{"plan-1", 0, "ace"}, {"a.b", 12, "g14"}} {
|
||||
got, ok := parseSilentKey(silentKey(w))
|
||||
if !ok || got != w {
|
||||
t.Errorf("%+v became %+v (%v)", w, got, ok)
|
||||
}
|
||||
}
|
||||
for _, bad := range []string{"", "plan", "plan.x.ace", "plan..ace", ".1.ace"} {
|
||||
if _, ok := parseSilentKey(bad); ok {
|
||||
t.Errorf("%q was read as a wait", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -413,10 +413,15 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
if err := need("members"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argv := []string{"delivery", "check", "--members", str("members")}
|
||||
if g := str("group"); g != "" {
|
||||
return []string{"delivery", "check", "--group", g, "--members", str("members")}, nil
|
||||
argv = []string{"delivery", "check", "--group", g, "--members", str("members")}
|
||||
}
|
||||
return []string{"delivery", "check", "--members", str("members")}, nil
|
||||
// Marked ready (novox/hq ADR 0406 §3): the check is queued before every other check.
|
||||
if r := strings.ToLower(str("ready")); r != "" && r != "false" && r != "no" && r != "0" {
|
||||
argv = append(argv, "--ready")
|
||||
}
|
||||
return argv, nil
|
||||
case "deliver":
|
||||
if err := need("plan"); err != nil {
|
||||
return nil, err
|
||||
@@ -1398,12 +1403,24 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
||||
// reloading its user list forgets the answer it was about to permit (novox/hq issue 265).
|
||||
link.Acknowledge(ctx)
|
||||
}
|
||||
return runVerb(ctx, argv)
|
||||
answer, err := runTheVerb(ctx, argv)
|
||||
// **A plan the verb moved is looked at as it ends** (novox/hq ADR 0406 §5, issue 974): the verb ran as a
|
||||
// child process, whose saves this process does not hear, so `deliver` after an approval, a stop, a retry
|
||||
// and a send from a verb would otherwise wait for an unrelated event. Even when it failed: it may have
|
||||
// kept a plan before it did.
|
||||
if !readingVerbs[verb] && !(verb == "plans" && !actsOnAPlan(args)) {
|
||||
askToLook()
|
||||
}
|
||||
return answer, err
|
||||
}
|
||||
}
|
||||
return handlers, behind, nil
|
||||
}
|
||||
|
||||
// runTheVerb is how the serving controller runs a verb: as a command of its own. A variable so a test of what follows a
|
||||
// verb needs no child process.
|
||||
var runTheVerb = runVerb
|
||||
|
||||
// actsOnAPlan is `plans` asked to stop, close or retry one rather than to show them.
|
||||
func actsOnAPlan(args map[string]any) bool {
|
||||
for _, act := range []string{"stop", "close", "retry"} {
|
||||
|
||||
@@ -126,11 +126,6 @@ var signalsTable = []signalRow{
|
||||
Kind: "controller-deaf", Severity: conditions.Urgent, Phase: 1,
|
||||
needs: func(f *signalFacts) error { return f.loopErr }, watch: watchLoop,
|
||||
newest: func(f *signalFacts) time.Time { return f.loop.took }},
|
||||
{Row: "S5", Signal: "a merge announced becomes a plan, or nothing reads it", Emitter: "announcer → controller",
|
||||
Trigger: "each merge", Bound: "10 min (the catch-up pass of issue 266)",
|
||||
Kind: "merge-not-acted", Severity: conditions.Urgent, Phase: 1,
|
||||
needs: func(f *signalFacts) error { return f.mergesErr }, watch: watchMerges,
|
||||
newest: func(f *signalFacts) time.Time { return f.mergesPassed }},
|
||||
{Row: "S6", Signal: "a build asked → its outcome", Emitter: "build seat", Trigger: "each ask",
|
||||
Bound: "max(20 min, 3 × the p90 of measured builds), 1 h while nothing is measured, provisional — the " +
|
||||
"build seat declares no timeout; and an ask the queue gave up on (dead) at once",
|
||||
@@ -459,18 +454,6 @@ func watchLoop(f *signalFacts) []conditions.Observation {
|
||||
Said: fmt.Sprintf("%d pending (%s), last taken %s", f.loop.pending, f.loop.where, since.UTC().Format(time.RFC3339))}}
|
||||
}
|
||||
|
||||
func watchMerges(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, m := range f.merges {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMerge, ID: m.Repo + "." + short(m.Commit),
|
||||
Kind: "merge-not-acted", Token: "not-acted", Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("%s/%s merged into %s (%s) was never handed to the controller by the bus; "+
|
||||
"acted on late by the catch-up", m.Owner, m.Repo, m.Base, short(m.Commit)),
|
||||
Said: fmt.Sprintf("announced %s, %s behind it", m.At.UTC().Format(time.RFC3339), readableList(m.Modules))})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func watchAsks(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, a := range f.asks {
|
||||
@@ -686,8 +669,10 @@ func watchDeadLetters(f *signalFacts) []conditions.Observation {
|
||||
// controller, or a seat's holders.
|
||||
func consumerWho(stream, consumer string) string {
|
||||
switch {
|
||||
case consumer == broker.ControllerName:
|
||||
case consumer == broker.ControllerName || consumer == broker.MergesConsumerName:
|
||||
return "the controller"
|
||||
case broker.IsCheckStream(stream):
|
||||
return "the holder of " + broker.TheBuildSeat
|
||||
case strings.HasPrefix(stream, "SEAT_") && strings.HasSuffix(consumer, "_worker"):
|
||||
seat := strings.ToLower(strings.ReplaceAll(strings.TrimSuffix(strings.TrimPrefix(consumer, "SEAT_"), "_worker"), "_", "-"))
|
||||
return "the holder of " + seat
|
||||
|
||||
@@ -33,7 +33,6 @@ func calm(now time.Time) *signalFacts {
|
||||
plans: []planFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee00", tier: 0, tiers: 2,
|
||||
entered: now.Add(-time.Minute), bound: 30 * time.Minute, waiting: "building"}},
|
||||
loop: loopFacts{took: now.Add(-time.Second), pending: 1},
|
||||
mergesPassed: now.Add(-time.Minute),
|
||||
selfCheck: selfCheckFacts{last: now.Add(-time.Minute), every: 5 * time.Minute},
|
||||
lostConsumers: map[string]bool{}, epochs: map[int64]inventory.Epoch{},
|
||||
lease: leaseFacts{held: true, epoch: 57, renewed: now.Add(-2 * time.Second)},
|
||||
@@ -72,12 +71,6 @@ var suppressions = map[string]suppression{
|
||||
inside: func(f *signalFacts) { f.loop.took = f.now.Add(-119 * time.Second) },
|
||||
past: func(f *signalFacts) { f.loop.took = f.now.Add(-121 * time.Second) },
|
||||
},
|
||||
"S5": {
|
||||
inside: func(f *signalFacts) {},
|
||||
past: func(f *signalFacts) {
|
||||
f.merges = []missedMerge{{Owner: "novox", Repo: "app", Base: "main", Commit: "c0ffee0011", At: f.now.Add(-11 * time.Minute)}}
|
||||
},
|
||||
},
|
||||
"S6": {
|
||||
inside: func(f *signalFacts) {
|
||||
f.asks = []askFacts{{id: "build-1", seat: "node-build-agent", what: "novox/app", state: link.AskInFlight,
|
||||
|
||||
@@ -210,6 +210,11 @@ func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, e
|
||||
if news {
|
||||
l.summary.nudge()
|
||||
}
|
||||
// A machine's account of an apply is looked at at once (novox/hq ADR 0406 §5, issue 974): it ends the wait for the
|
||||
// machine, and the look is what deletes its key, knowing the report was on that send.
|
||||
if err == nil && report.Declared != "" && report.Superseded == "" && report.Rekey == nil {
|
||||
askToLook()
|
||||
}
|
||||
// What it says of its long-running resources (novox/hq ADR 0240): kept, and its modules' conditions
|
||||
// raised or cleared from it. Only from an account of the machine the store took.
|
||||
if err == nil && report.Health != nil && report.Superseded == "" && report.Rekey == nil && report.Node != "" &&
|
||||
|
||||
@@ -1,157 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A new push to a pull request ends the check of its old head (novox/hq issue 726).
|
||||
//
|
||||
// A check judges one head; once the pull request has another, its verdict answers nobody and its run holds a build
|
||||
// machine for minutes the new head waits behind. So when a check is asked for pull request N of a repository at a
|
||||
// head, **every check of that pull request at another head ends**: a waiting one is dropped from the queue as
|
||||
// `cancel` drops it, one in flight is ended on its machine as `kill` ends it. Each is said in the log.
|
||||
//
|
||||
// **Never anything else**: a try (`try`, asked for no pull request), a walk's build (no check), a delivery group's
|
||||
// composed check (it answers several pull requests), another pull request's check, a dead ask (left for `clear
|
||||
// --dead`), and a check of the same head — a recheck of that head is not another head.
|
||||
|
||||
// **Only for the pull request's current head.** Announcements of a pull request are not ordered, one is heard again
|
||||
// after a restart, and a recheck asks for the head it recorded: a check asked for a head the pull request no longer
|
||||
// has must never end the check of the head it has. So the forge is asked first which head the pull request has; only
|
||||
// when it names this one does anything end, and when it cannot be asked, or names another, nothing ends and that is
|
||||
// said.
|
||||
|
||||
// forgePullHead asks the forge, through its module's tools on the bus, which commit a pull request's head is. A
|
||||
// variable so a test needs no forge.
|
||||
var forgePullHead = func(ctx context.Context, js *broker.JetStream, owner, repo string, number int) (string, error) {
|
||||
raw, _ := json.Marshal(map[string]any{"path": fmt.Sprintf("/repos/%s/%s/pulls/%d", owner, repo, number)})
|
||||
answer, err := link.Ask(ctx, link.OverNATS{Conn: js.Conn()}, "gitea", "gitea_api", raw, 30*time.Second)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return "", errors.New(answer.Error)
|
||||
}
|
||||
var pull struct {
|
||||
Result struct {
|
||||
Head struct {
|
||||
SHA string `json:"sha"`
|
||||
} `json:"head"`
|
||||
} `json:"result"`
|
||||
}
|
||||
if err := json.Unmarshal(answer.Result, &pull); err != nil {
|
||||
return "", fmt.Errorf("the forge's answer about the pull request cannot be read: %w", err)
|
||||
}
|
||||
if pull.Result.Head.SHA == "" {
|
||||
return "", errors.New("the forge named no head for the pull request")
|
||||
}
|
||||
return pull.Result.Head.SHA, nil
|
||||
}
|
||||
|
||||
// supersededChecks are the checks in the queue that a check of owner/repo#number at head ends: of the same pull
|
||||
// request, at another head, waiting or in flight, and not the ask keep.
|
||||
func supersededChecks(q link.Queue, owner, repo string, number int, head, keep string) []link.QueuedAsk {
|
||||
if number <= 0 || head == "" {
|
||||
return nil
|
||||
}
|
||||
var out []link.QueuedAsk
|
||||
for _, a := range q.Asks {
|
||||
c := a.Request.Check
|
||||
switch {
|
||||
case c == nil, link.IsTry(a.Request), c.Group != "", c.Number != number, a.ID == keep,
|
||||
!strings.EqualFold(c.Owner, owner), !strings.EqualFold(c.Repo, repo),
|
||||
a.State != link.AskWaiting && a.State != link.AskInFlight:
|
||||
continue
|
||||
}
|
||||
ref := a.Request.Ref
|
||||
if ref == "" {
|
||||
ref = a.Ref
|
||||
}
|
||||
if ref == "" || ref == head || link.SameHead(ref, head) {
|
||||
continue
|
||||
}
|
||||
out = append(out, a)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// endSupersededChecks ends the checks a check of p at its head supersedes, keep being the new check's own ask, and
|
||||
// says each in the log. A waiting one is cancelled — nothing recorded: a check builds nothing, and the forge's status
|
||||
// of the old head is no longer the pull request's — and one a machine builds is killed there.
|
||||
func endSupersededChecks(ctx context.Context, js *broker.JetStream, seat string, p link.PullUpdated, keep string) []string {
|
||||
if p.Number <= 0 || p.Commit == "" {
|
||||
return nil
|
||||
}
|
||||
current, err := forgePullHead(ctx, js, p.Owner, p.Repo, p.Number)
|
||||
if err != nil {
|
||||
return []string{fmt.Sprintf("%s/%s#%d (%.8s): nothing ended — the forge could not say the pull request's "+
|
||||
"current head, so checks of its other heads keep running: %v", p.Owner, p.Repo, p.Number, p.Commit, err)}
|
||||
}
|
||||
if current != p.Commit && !link.SameHead(current, p.Commit) {
|
||||
return []string{fmt.Sprintf("%s/%s#%d (%.8s): nothing ended — the pull request's head is %.12s now, not "+
|
||||
"this one", p.Owner, p.Repo, p.Number, p.Commit, current)}
|
||||
}
|
||||
q, err := link.ReadQueue(ctx, js, seat)
|
||||
if err != nil {
|
||||
return []string{fmt.Sprintf("%s/%s#%d (%.8s): could not read the %s queue for checks of its earlier heads: %v",
|
||||
p.Owner, p.Repo, p.Number, p.Commit, seat, err)}
|
||||
}
|
||||
var said []string
|
||||
for _, a := range supersededChecks(q, p.Owner, p.Repo, p.Number, p.Commit, keep) {
|
||||
old := a.Request.Ref
|
||||
if old == "" {
|
||||
old = a.Ref
|
||||
}
|
||||
what := fmt.Sprintf("check of %s/%s#%d at %.12s", p.Owner, p.Repo, p.Number, old)
|
||||
var err error
|
||||
if a.State == link.AskInFlight && a.On != "" {
|
||||
var answer link.Answer
|
||||
answer, err = link.AskSeatTool(ctx, js.Conn(), seat, "kill", a.On, map[string]string{"id": a.ID}, killAnswer)
|
||||
if err == nil && answer.Error != "" {
|
||||
err = fmt.Errorf("%s refused: %s", a.On, answer.Error)
|
||||
}
|
||||
} else {
|
||||
_, err = cancelAsk(ctx, js, nil, seat, a)
|
||||
}
|
||||
if err != nil {
|
||||
said = append(said, fmt.Sprintf("not ended: %s (%s) — superseded by %.12s, but: %v", what, a.ID, p.Commit, err))
|
||||
continue
|
||||
}
|
||||
said = append(said, fmt.Sprintf("ended: %s — superseded by %.12s (%s, %s)", what, p.Commit, a.ID, a.State))
|
||||
}
|
||||
return said
|
||||
}
|
||||
|
||||
// endingSupersededChecks ends them beside the announcement, so a kill waiting on its machine never holds the
|
||||
// handler: on its own connection to the bus, within twice a kill's wait. A one-shot process (oneShot) ends them
|
||||
// within the call instead, as its exit would cut them off.
|
||||
func endingSupersededChecks(ctx context.Context, seat string, p link.PullUpdated, keep string, oneShot bool) {
|
||||
end := func() {
|
||||
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*killAnswer)
|
||||
defer cancel()
|
||||
js, err := dialTheBus()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s/%s#%d (%.8s): could not reach the bus to end checks of its earlier heads: %v\n",
|
||||
p.Owner, p.Repo, p.Number, p.Commit, err)
|
||||
return
|
||||
}
|
||||
defer js.Close()
|
||||
for _, line := range endSupersededChecks(ctx, js, seat, p, keep) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
}
|
||||
if oneShot {
|
||||
end()
|
||||
return
|
||||
}
|
||||
go end()
|
||||
}
|
||||
@@ -1,192 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A new push to a pull request ends the check of its old head (novox/hq issue 726): the waiting one is dropped from
|
||||
// the queue, the one in flight is killed on its machine, and each is said. A try, a walk's build, another pull
|
||||
// request's check, a group's check and a recheck of the new head are left alone. On the bus alone: no store.
|
||||
func TestANewHeadEndsTheChecksOfTheOldOnes(t *testing.T) {
|
||||
js, _ := aBuildQueue(t)
|
||||
ctx := t.Context()
|
||||
const (
|
||||
oldHead = "1111111111111111111111111111111111111111"
|
||||
olderHead = "2222222222222222222222222222222222222222"
|
||||
newHead = "3333333333333333333333333333333333333333"
|
||||
)
|
||||
publish := func(r link.BuildRequest) {
|
||||
t.Helper()
|
||||
body, _ := json.Marshal(r)
|
||||
if _, err := js.Context().Publish(link.BuildWork(), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
check := func(id string, number int, head string) link.BuildRequest {
|
||||
return link.BuildRequest{ID: id, Repository: "https://forge.example/novox/a.git", Ref: head,
|
||||
Check: &link.CheckRequest{Owner: "novox", Repo: "a", Number: number}}
|
||||
}
|
||||
// In flight first: taken by a holder, started on ace.
|
||||
publish(check("check-running", 7, olderHead))
|
||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
|
||||
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
started, _ := json.Marshal(link.BuildStart{ID: "check-running", On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
|
||||
if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
publish(check("check-old", 7, oldHead))
|
||||
publish(check("check-other-pull", 8, oldHead))
|
||||
publish(check("check-recheck", 7, newHead))
|
||||
publish(check("check-new", 7, newHead))
|
||||
try := check("try-old", 7, oldHead)
|
||||
try.Check.Number, try.Check.Try = 0, true
|
||||
publish(try)
|
||||
group := check("group-old", 7, oldHead)
|
||||
group.Check.Group = "g1"
|
||||
publish(group)
|
||||
publish(link.BuildRequest{ID: "walk-build", Repository: "https://forge.example/novox/a.git", Ref: oldHead})
|
||||
|
||||
var mu sync.Mutex
|
||||
var killed []string
|
||||
stop, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(link.TheBuildMachine, "ace", map[string]link.ToolHandler{
|
||||
"kill": func(_ context.Context, raw json.RawMessage) (any, error) {
|
||||
var args struct{ ID string }
|
||||
_ = json.Unmarshal(raw, &args)
|
||||
mu.Lock()
|
||||
killed = append(killed, args.ID)
|
||||
mu.Unlock()
|
||||
return map[string]any{"said": "killed " + args.ID}, nil
|
||||
}}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
|
||||
forgeSays(t, newHead, nil)
|
||||
said := endSupersededChecks(ctx, js, link.TheBuildMachine,
|
||||
link.PullUpdated{Owner: "novox", Repo: "a", Number: 7, Commit: newHead}, "check-new")
|
||||
|
||||
mu.Lock()
|
||||
if len(killed) != 1 || killed[0] != "check-running" {
|
||||
t.Errorf("the holder was asked to kill %v, want only the in-flight check of the older head", killed)
|
||||
}
|
||||
mu.Unlock()
|
||||
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, found := q.Find("check-old"); found {
|
||||
t.Error("the waiting check of the old head is still in the queue")
|
||||
}
|
||||
for _, id := range []string{"check-other-pull", "check-recheck", "check-new", "try-old", "group-old", "walk-build"} {
|
||||
if _, found := q.Find(id); !found {
|
||||
t.Errorf("%s was ended; only checks of the same pull request at another head end", id)
|
||||
}
|
||||
}
|
||||
text := strings.Join(said, "\n")
|
||||
for _, want := range []string{
|
||||
"ended: check of novox/a#7 at " + oldHead[:12] + " — superseded by " + newHead[:12],
|
||||
"ended: check of novox/a#7 at " + olderHead[:12] + " — superseded by " + newHead[:12],
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Errorf("the log does not say %q:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
if len(said) != 2 {
|
||||
t.Errorf("said %d lines, want 2:\n%s", len(said), text)
|
||||
}
|
||||
}
|
||||
|
||||
// Which asks a check supersedes, read from the queue alone.
|
||||
func TestSupersededChecksAreOnlyThePullRequestsOtherHeads(t *testing.T) {
|
||||
ask := func(id, state, ref string, c *link.CheckRequest) link.QueuedAsk {
|
||||
return link.QueuedAsk{ID: id, State: state, Ref: ref, Request: link.BuildRequest{ID: id, Ref: ref, Check: c}}
|
||||
}
|
||||
pr := func(n int) *link.CheckRequest { return &link.CheckRequest{Owner: "Novox", Repo: "A", Number: n} }
|
||||
head := "abcdefabcdefabcdef0123456789abcdef012345"
|
||||
q := link.Queue{Asks: []link.QueuedAsk{
|
||||
ask("old", link.AskWaiting, "0123456789ab0123456789ab0123456789ab0123", pr(7)),
|
||||
ask("short-same", link.AskWaiting, head[:12], pr(7)),
|
||||
ask("dead", link.AskDead, "0123456789ab0123456789ab0123456789ab0123", pr(7)),
|
||||
ask("other", link.AskWaiting, "0123456789ab0123456789ab0123456789ab0123", pr(8)),
|
||||
ask("build", link.AskWaiting, "0123456789ab0123456789ab0123456789ab0123", nil),
|
||||
ask("keep", link.AskWaiting, "0123456789ab0123456789ab0123456789ab0123", pr(7)),
|
||||
}}
|
||||
got := supersededChecks(q, "novox", "a", 7, head, "keep")
|
||||
if len(got) != 1 || got[0].ID != "old" {
|
||||
t.Fatalf("superseded %+v, want only old", got)
|
||||
}
|
||||
if got := supersededChecks(q, "novox", "a", 0, head, ""); got != nil {
|
||||
t.Errorf("a check of no pull request superseded %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// forgeSays makes the forge name head as every pull request's current head, or fail with err.
|
||||
func forgeSays(t *testing.T, head string, err error) {
|
||||
t.Helper()
|
||||
was := forgePullHead
|
||||
forgePullHead = func(context.Context, *broker.JetStream, string, string, int) (string, error) { return head, err }
|
||||
t.Cleanup(func() { forgePullHead = was })
|
||||
}
|
||||
|
||||
// An older head heard after a newer one — an announcement out of order, one heard again after a restart, a recheck
|
||||
// of a recorded head — ends nothing: the forge says the pull request's head is the newer one. Nor does a check whose
|
||||
// pull request's head the forge cannot say.
|
||||
func TestAnOlderHeadHeardLateEndsNothing(t *testing.T) {
|
||||
js, _ := aBuildQueue(t)
|
||||
ctx := t.Context()
|
||||
const (
|
||||
oldHead = "1111111111111111111111111111111111111111"
|
||||
newHead = "3333333333333333333333333333333333333333"
|
||||
)
|
||||
for _, id := range []string{"check-new", "check-old-late"} {
|
||||
head := newHead
|
||||
if id == "check-old-late" {
|
||||
head = oldHead
|
||||
}
|
||||
body, _ := json.Marshal(link.BuildRequest{ID: id, Repository: "https://forge.example/novox/a.git", Ref: head,
|
||||
Check: &link.CheckRequest{Owner: "novox", Repo: "a", Number: 7}})
|
||||
if _, err := js.Context().Publish(link.BuildWork(), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
late := link.PullUpdated{Owner: "novox", Repo: "a", Number: 7, Commit: oldHead}
|
||||
forgeSays(t, newHead, nil)
|
||||
said := endSupersededChecks(ctx, js, link.TheBuildMachine, late, "check-old-late")
|
||||
if len(said) != 1 || !strings.Contains(said[0], "nothing ended") {
|
||||
t.Errorf("an older head heard late said %q", said)
|
||||
}
|
||||
forgeSays(t, "", errors.New("nothing serves gitea.gitea_api"))
|
||||
said = endSupersededChecks(ctx, js, link.TheBuildMachine,
|
||||
link.PullUpdated{Owner: "novox", Repo: "a", Number: 7, Commit: newHead}, "check-new")
|
||||
if len(said) != 1 || !strings.Contains(said[0], "nothing ended") {
|
||||
t.Errorf("a head the forge could not confirm said %q", said)
|
||||
}
|
||||
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, id := range []string{"check-new", "check-old-late"} {
|
||||
if _, found := q.Find(id); !found {
|
||||
t.Errorf("%s was ended", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -294,10 +294,10 @@ func (f following) Announceable(ctx context.Context) ([]link.Announcement, error
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// notNow marks a store that could not be read right now, so the announcement is held rather than
|
||||
// lost; anything else is returned as it was.
|
||||
// notNow marks a store that could not be read right now, or plans another controller is working (issue 921), so
|
||||
// the announcement is held rather than lost; anything else is returned as it was.
|
||||
func notNow(err error) error {
|
||||
if inventory.Unreachable(err) {
|
||||
if inventory.Unreachable(err) || errors.Is(err, inventory.ErrPlansBusy) {
|
||||
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
|
||||
}
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A plan moved by a seat verb is looked at as the verb ends (issue 974, review 1): the verb runs as a child process
|
||||
// whose saves the serving controller does not hear, so `deliver`, which lets a walk go after the operator's approval,
|
||||
// would leave its first tier waiting for an unrelated event.
|
||||
func TestAVerbThatMayMoveAPlanHasTheServingControllerLookWhenItEnds(t *testing.T) {
|
||||
var looks, failures atomic.Int32
|
||||
l := lookerCounting(&failures, &looks, nil)
|
||||
lookerMu.Lock()
|
||||
theLooker = l
|
||||
lookerMu.Unlock()
|
||||
t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() })
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
go l.run(ctx)
|
||||
|
||||
was := runTheVerb
|
||||
runTheVerb = func(context.Context, []string) (verbAnswer, error) { return verbAnswer{}, nil }
|
||||
t.Cleanup(func() { runTheVerb = was })
|
||||
handlers, _, err := seatToolHandlers()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args string
|
||||
look bool
|
||||
}{
|
||||
{"deliver", `{"plan":"p"}`, true},
|
||||
{"plans", `{"stop":"plan-1","why":"w"}`, true},
|
||||
{"plans", `{}`, false},
|
||||
{"nodes", `{}`, false},
|
||||
} {
|
||||
before := looks.Load()
|
||||
h := handlers[c.verb]
|
||||
if h == nil {
|
||||
t.Fatalf("no verb %s", c.verb)
|
||||
}
|
||||
if _, err := h(ctx, []byte(c.args)); err != nil {
|
||||
t.Fatalf("%s %s: %v", c.verb, c.args, err)
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if got := looks.Load() > before; got != c.look {
|
||||
t.Errorf("%s %s: look asked = %v; want %v", c.verb, c.args, got, c.look)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -15,7 +15,6 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
@@ -64,24 +63,8 @@ func warrantedAct(asker, ask, caller, state string, w *asks.Warrant) (link.HandA
|
||||
// readRouterRecord reads the router's record of one asker's ask: its state and warrant, or "" when there is none.
|
||||
// The controller's grant reaches the JetStream API whole (`$JS.API.>`), so it reads any asker's record.
|
||||
func readRouterRecord(ctx context.Context, conn *nats.Conn, bucket, asker, ask string) (string, *asks.Warrant, error) {
|
||||
reply, err := conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET.KV_"+bucket+".$KV."+bucket+"."+asker+"."+ask, nil)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if status := reply.Header.Get("Status"); status != "" {
|
||||
if status == "404" {
|
||||
return "", nil, nil
|
||||
}
|
||||
return "", nil, fmt.Errorf("the router's record could not be read: %s %s", status, reply.Header.Get("Description"))
|
||||
}
|
||||
var rec struct {
|
||||
State string `json:"state"`
|
||||
Warrant *asks.Warrant `json:"warrant"`
|
||||
}
|
||||
if err := json.Unmarshal(reply.Data, &rec); err != nil {
|
||||
return "", nil, fmt.Errorf("the router's record of %s's ask %s cannot be read: %w", asker, ask, err)
|
||||
}
|
||||
return rec.State, rec.Warrant, nil
|
||||
state, _, w, err := readRouterEntry(ctx, conn, bucket, asker, ask)
|
||||
return state, w, err
|
||||
}
|
||||
|
||||
func handActWarrantCommand(ctx context.Context, args []string) error {
|
||||
|
||||
@@ -70,10 +70,6 @@ type signalFacts struct {
|
||||
loop loopFacts
|
||||
loopErr error
|
||||
|
||||
merges []missedMerge
|
||||
mergesPassed time.Time
|
||||
mergesErr error
|
||||
|
||||
asks []askFacts
|
||||
asksErr error
|
||||
|
||||
@@ -358,10 +354,6 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
|
||||
f.batches, f.plansErr = gatherBatches(ctx, inv, now)
|
||||
}
|
||||
f.loop, f.loopErr = w.gatherLoop()
|
||||
f.mergesPassed, f.merges, f.mergesErr = watchedMerges.last()
|
||||
if f.mergesErr == nil && !f.mergesPassed.IsZero() && now.Sub(f.mergesPassed) > 3*mergeCatchUpEvery {
|
||||
f.mergesErr = fmt.Errorf("the catch-up of merges has not passed since %s", f.mergesPassed.UTC().Format(time.RFC3339))
|
||||
}
|
||||
f.asks, f.asksErr = w.gatherAsks(ctx, inv)
|
||||
if open, err := w.keeper.Open(ctx); err != nil {
|
||||
f.standingsErr = err
|
||||
@@ -767,9 +759,6 @@ func watchTheMesh(ctx context.Context, open *stores, server *link.Server, bus li
|
||||
// and the answer chosen is performed on its warrant.
|
||||
startAsking(watching, open, server, bus.Conn, keeper)
|
||||
go forgettingOldHeals(watching, open.inventory)
|
||||
// And the build queue's order (novox/hq issue 558): a check that has waited its while goes ahead of the
|
||||
// deliveries asked after it even when nothing new is asked.
|
||||
go keepingTheQueueInOrder(watching, open, server.JetStream())
|
||||
fmt.Printf("watching the mesh: %d signal(s) every %s, %d probe(s) every %s; what is wrong is kept in %s "+
|
||||
"and said as %s events\n", len(watchedRows()), watchEvery, len(runnableProbes()), doctorEvery,
|
||||
broker.ConditionsBucket, conditions.Seat)
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
# The route proxy, as a module ships one.
|
||||
#
|
||||
# Static and FROM scratch like the control plane's image, and for the same reason: it is fetched
|
||||
# by digest and run on a machine, so everything in it is something a person would have to audit.
|
||||
# The Go it builds with is the one the manifest pins (build.on GO_BASE), passed by the Makefile and the
|
||||
# build machine alike; the default only serves a hand build, and matches go.mod.
|
||||
ARG GO_BASE=golang:1.26-alpine
|
||||
FROM ${GO_BASE} AS build
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN CGO_ENABLED=0 go build -trimpath -buildvcs=false -ldflags '-s -w' -o /mesh-route-proxy ./examples/route-proxy
|
||||
|
||||
FROM scratch
|
||||
COPY --from=build /mesh-route-proxy /mesh-route-proxy
|
||||
ENTRYPOINT ["/mesh-route-proxy"]
|
||||
@@ -36,4 +36,4 @@ require (
|
||||
// committed. Every build (the build agent's `go build`, the Dockerfile) compiles from vendor/ and
|
||||
// fetches nothing; go refuses to build when vendor/ and this file disagree, so a pin moved without
|
||||
// `go mod vendor` fails loudly, at once, everywhere.
|
||||
replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261015211328-5a7ec3775b98
|
||||
replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261017162006-6469e8238443
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261015211328-5a7ec3775b98 h1:27nPn3XO9yfrHoAOuTuuJPVZTA3Bdnbqlc2wRI7S/LY=
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261015211328-5a7ec3775b98/go.mod h1:5QOm3cIfu/k9fN6QqgDbKGCcDst7Ul1UlcfsiMVm9j4=
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261017162006-6469e8238443 h1:zUtpGbiH9DLjCPY+KJsm+PZ25yW6k/NmGVS3haGFfMM=
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261017162006-6469e8238443/go.mod h1:5QOm3cIfu/k9fN6QqgDbKGCcDst7Ul1UlcfsiMVm9j4=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19 h1:mAMt3KFmrHzDiBv4MLL94opWcG9y73ISMrBUyisapQE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
package broker
|
||||
|
||||
import "strings"
|
||||
|
||||
// The pull requests' checks have work queues of their own (novox/hq ADR 0406 §3, issue 974).
|
||||
//
|
||||
// A delivery build goes on the build seat's own queue, `mesh.seat.node-build-agent.accept.build`. A check of a pull
|
||||
// request marked ready goes on a second work-queue stream and every other check on a third. A builder takes from the
|
||||
// first, then the second, then the third; the others are taken in the order asked. **Each check stream has one subject
|
||||
// per pull request and keeps only the newest request** (a limit of one message a subject, the older discarded), so a
|
||||
// newer request replaces one still waiting and a pull request has at most one waiting check: the stream's rule, not a
|
||||
// pass of the controller that moves asks about. Nothing pauses the worker or deletes and republishes a request.
|
||||
|
||||
const (
|
||||
// ChecksReadyStream holds the checks of pull requests marked ready; ChecksOtherStream every other check.
|
||||
ChecksReadyStream = "CHECKS_READY"
|
||||
ChecksOtherStream = "CHECKS_OTHER"
|
||||
// ChecksReadyWorker and ChecksOtherWorker are the durable consumers the build seat's holders share on them, as
|
||||
// they share the seat's worker on its own queue (novox/hq ADR 0190).
|
||||
ChecksReadyWorker = "CHECKS_READY_worker"
|
||||
ChecksOtherWorker = "CHECKS_OTHER_worker"
|
||||
// TheBuildSeat is the role whose checks these streams queue. The retired build role's checks stay on its own queue.
|
||||
TheBuildSeat = "node-build-agent"
|
||||
)
|
||||
|
||||
// PullsBucket is mesh-delivery's key per pull request: its head, the trunk commit its current check is asked against and
|
||||
// its verdict (ADR 0406 §4). A running check watches its key.
|
||||
var PullsBucket = BucketName("mesh-delivery", "pulls")
|
||||
|
||||
// CheckSubjectPrefix is where a seat's checks of one kind are asked: `mesh.seat.<seat>.check.ready.` or `.other.`. The
|
||||
// `check` token keeps the subjects out of the seat's own queue, which takes `accept.>`.
|
||||
func CheckSubjectPrefix(seat string, ready bool) string {
|
||||
kind := "other"
|
||||
if ready {
|
||||
kind = "ready"
|
||||
}
|
||||
return "mesh.seat." + seat + ".check." + kind + "."
|
||||
}
|
||||
|
||||
// CheckStreams are the two check queues, in the order a builder takes from them.
|
||||
func CheckStreams() []Stream {
|
||||
return []Stream{
|
||||
{
|
||||
Name: ChecksReadyStream,
|
||||
Subjects: []string{CheckSubjectPrefix(TheBuildSeat, true) + ">"},
|
||||
Retention: RetentionWorkQueue,
|
||||
MaxAge: 7 * 24 * 60 * 60,
|
||||
MaxMsgsPerSubject: 1,
|
||||
Why: "the checks of pull requests marked ready, taken after a delivery build and before any other check " +
|
||||
"(novox/hq ADR 0406 §3); one subject a pull request, the newest request kept, so a newer one replaces a " +
|
||||
"request still waiting",
|
||||
},
|
||||
{
|
||||
Name: ChecksOtherStream,
|
||||
Subjects: []string{CheckSubjectPrefix(TheBuildSeat, false) + ">"},
|
||||
Retention: RetentionWorkQueue,
|
||||
MaxAge: 7 * 24 * 60 * 60,
|
||||
MaxMsgsPerSubject: 1,
|
||||
Why: "every other check, taken last and in the order asked (novox/hq ADR 0406 §3); one subject a pull " +
|
||||
"request, the newest request kept; one left behind for a head already judged is dropped when taken",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// CheckWorkers are the consumers the build seat's holders pull the check queues through: as the seat's own worker is,
|
||||
// one for every holder, one ask at a time, acknowledged when the check is done.
|
||||
func CheckWorkers() []Consumer {
|
||||
worker := func(name, stream string, ready bool) Consumer {
|
||||
return Consumer{
|
||||
Name: name,
|
||||
Stream: stream,
|
||||
Filters: []string{CheckSubjectPrefix(TheBuildSeat, ready) + ">"},
|
||||
AckWaitSeconds: 60,
|
||||
MaxDeliver: 5,
|
||||
Why: "every holder of " + TheBuildSeat + " pulls one check at a time from this worker and acknowledges after " +
|
||||
"the check is done, so a crash mid-check redelivers rather than loses (novox/hq ADR 0406 §3)",
|
||||
}
|
||||
}
|
||||
return []Consumer{
|
||||
worker(ChecksReadyWorker, ChecksReadyStream, true),
|
||||
worker(ChecksOtherWorker, ChecksOtherStream, false),
|
||||
}
|
||||
}
|
||||
|
||||
// holderCheckGrants are what a holder of the build seat is granted to take checks: ask about the two workers, pull from
|
||||
// them and acknowledge, and nothing of the subjects themselves.
|
||||
func holderCheckGrants() []string {
|
||||
var pub []string
|
||||
for _, w := range CheckWorkers() {
|
||||
pub = append(pub,
|
||||
"$JS.API.CONSUMER.INFO."+w.Stream+"."+w.Name,
|
||||
"$JS.API.CONSUMER.MSG.NEXT."+w.Stream+"."+w.Name,
|
||||
"$JS.ACK."+w.Stream+"."+w.Name+".>")
|
||||
}
|
||||
// And the bucket of the checks they are running: each writes the key of its own check, reads another's, and the bucket
|
||||
// is a controller's own bucket the build seat's holders write by design (ADR 0406 §3).
|
||||
pub = append(pub,
|
||||
"$KV."+CheckRunningBucket+".>",
|
||||
"$JS.API.STREAM.INFO.KV_"+CheckRunningBucket,
|
||||
"$JS.API.DIRECT.GET.KV_"+CheckRunningBucket+".>")
|
||||
return pub
|
||||
}
|
||||
|
||||
// IsCheckStream says a stream is one of the check queues.
|
||||
func IsCheckStream(stream string) bool {
|
||||
return stream == ChecksReadyStream || stream == ChecksOtherStream
|
||||
}
|
||||
|
||||
// isCheckAsk says a message of a check queue is an ask the controller itself makes.
|
||||
func isCheckAsk(stream, subject string) bool {
|
||||
switch stream {
|
||||
case ChecksReadyStream:
|
||||
return strings.HasPrefix(subject, CheckSubjectPrefix(TheBuildSeat, true))
|
||||
case ChecksOtherStream:
|
||||
return strings.HasPrefix(subject, CheckSubjectPrefix(TheBuildSeat, false))
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package broker
|
||||
|
||||
import "testing"
|
||||
|
||||
// The build agent's credential is the module principal `<node>.build-agent` (Username, KindModule): its manifest's
|
||||
// `reads: ["mesh-delivery.pulls"]` reaches that principal as a read and a watch of mesh-delivery's bucket of pull requests'
|
||||
// keys — the direct get of a key, the consumer a watch makes, the stream's info — and as no write and no direct
|
||||
// subscription of the bucket's subjects (novox/hq ADR 0406 §4, issue 974).
|
||||
func TestTheBuildAgentReadsAndWatchesThePullRequestsKeys(t *testing.T) {
|
||||
seat := Seat{Name: TheBuildSeat, Scope: "node", Accepts: []string{"build"}, Emits: []string{"built", "started"}}
|
||||
p := Principal{Kind: KindModule, Node: "one", Module: "build-agent", Holds: []Seat{seat},
|
||||
Reads: []string{"mesh-delivery.pulls"}, PasswordHash: "x"}
|
||||
if got := p.Username(); got != "one.build-agent" {
|
||||
t.Fatalf("the build agent's credential is %q", got)
|
||||
}
|
||||
perms, err := PermissionsFor(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, s := range []string{
|
||||
"$JS.API.STREAM.INFO.KV_mesh-delivery_pulls",
|
||||
"$JS.API.DIRECT.GET.KV_mesh-delivery_pulls.>",
|
||||
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery_pulls.>",
|
||||
"$JS.API.CONSUMER.DELETE.KV_mesh-delivery_pulls.>",
|
||||
// And the two check workers it takes from.
|
||||
"$JS.API.CONSUMER.MSG.NEXT.CHECKS_READY.CHECKS_READY_worker",
|
||||
"$JS.API.CONSUMER.MSG.NEXT.CHECKS_OTHER.CHECKS_OTHER_worker",
|
||||
"$JS.ACK.CHECKS_OTHER.CHECKS_OTHER_worker.>",
|
||||
// And the bucket of the checks running, which it writes and reads (ADR 0406 §3).
|
||||
"$KV." + CheckRunningBucket + ".>",
|
||||
"$JS.API.DIRECT.GET.KV_" + CheckRunningBucket + ".>",
|
||||
} {
|
||||
has(t, perms.Publish, s)
|
||||
}
|
||||
hasNot(t, perms.Publish, "$KV."+PullsBucket+".>")
|
||||
hasNot(t, perms.Publish, "mesh.seat.node-build-agent.check.>")
|
||||
for _, s := range perms.Subscribe {
|
||||
if s == "$KV."+PullsBucket+".>" {
|
||||
t.Fatalf("the build agent subscribes the bucket's subjects directly: %v", perms.Subscribe)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -38,8 +38,35 @@ var (
|
||||
// each ask by its id, its options and the actions they stand for, how it ended and whether the
|
||||
// controller acted on its warrant — so a restart neither asks twice nor acts twice.
|
||||
AskedBucket = BucketName(ControllerSeat, "asked")
|
||||
// MergeWindowBucket holds the clock of every open batch's merge window (novox/hq ADR 0407 decision 1): per
|
||||
// batch a key `<batch>.quiet`, made again by each merge into it and living the window, and a key
|
||||
// `<batch>.most`, made once when it opens and living the window's most. The first to expire closes the
|
||||
// batch, and the controller's watcher hears the delete marker. Written by the lease holder alone.
|
||||
MergeWindowBucket = BucketName(ControllerSeat, "merge-window")
|
||||
// RolloutSilentBucket holds the wait of every machine a rollout has sent a tier to (novox/hq ADR 0406 §5, issue
|
||||
// 974): per send a key `<plan>.<tier>.<node>` that lives ApplySilentAfter, made when the controller sends that
|
||||
// machine and deleted by the machine's report on that send. Only a missing report lets it expire, and the
|
||||
// controller's watcher hearing the expiry ends the wait and names the machine silent. Written by the lease
|
||||
// holder alone.
|
||||
RolloutSilentBucket = BucketName(ControllerSeat, "rollout-silent")
|
||||
// CheckRunningBucket holds the pull request checks the builders are running now (novox/hq ADR 0406 §3, issue 974): per
|
||||
// pull request a key `<owner>.<repo>.<number>` (or `group.<name>`) holding the head and trunk commit being checked, the
|
||||
// builder and its build. A builder puts it when it starts a check and again on each heartbeat, and deletes it when the
|
||||
// check ends; a key not renewed within the bucket's age is a builder that died, and is gone. A builder that takes a
|
||||
// waiting request for a head and trunk commit a key names drops it. **Written by the build seat's holders, several by
|
||||
// design** — each the key of the check it runs.
|
||||
CheckRunningBucket = BucketName(ControllerSeat, "checks-running")
|
||||
// DeliveryMergesBucket holds the merges mesh-delivery announced it made, each with the ask whose Approve it merged
|
||||
// on (novox/hq ADR 0420 §7, issue 1025): per merge a key `<owner>/<repo>/<number>/<head sha>` holding the
|
||||
// announcement, kept DeliveryMergesKeptFor. What lets the build of a moved module register it from its new
|
||||
// repository: the controller reads the ask's warrant from the router and accepts only the operator's Approve of
|
||||
// those heads. Written by the controller alone, from the announcement, when it holds the lease.
|
||||
DeliveryMergesBucket = BucketName(ControllerSeat, "delivery-merges")
|
||||
)
|
||||
|
||||
// DeliveryMergesKeptFor is how long a merge mesh-delivery announced is kept: a week, as the events are.
|
||||
const DeliveryMergesKeptFor = 7 * 24 * time.Hour
|
||||
|
||||
// The controller's record mirrors (novox/hq ADR 0289, design 54, design 57 package F3): a bucket per kind of
|
||||
// record, one key per record, written from the store by the lease holder only when the record changed, with
|
||||
// one event on the controller's seat per write. Read by whatever names them in `reads` as
|
||||
@@ -79,6 +106,22 @@ const (
|
||||
MirrorMarkerTTL = 24 * time.Hour
|
||||
)
|
||||
|
||||
// MergeWindowMarkerTTL is how long the marker an expired window key leaves is kept: long enough for a watcher
|
||||
// that is up to hear it, short enough to leave nothing behind.
|
||||
const MergeWindowMarkerTTL = time.Minute
|
||||
|
||||
// RolloutSilentMarkerTTL is how long the marker an expired silent key leaves is kept, for the same reason.
|
||||
const RolloutSilentMarkerTTL = time.Minute
|
||||
|
||||
// RolloutSilentKeptFor is the age of the silent keys' bucket: every report deletes a key and the delete leaves a marker, which
|
||||
// with one value per key would stay for ever and fill the bucket. A key lives a quarter of an hour at most, so an hour
|
||||
// takes nothing live.
|
||||
const RolloutSilentKeptFor = time.Hour
|
||||
|
||||
// CheckRunningKeptFor is the age of a running check's key: a few heartbeats (ten seconds each) of a builder, so a builder that
|
||||
// died is forgotten within two minutes and the check it ran is taken again then.
|
||||
const CheckRunningKeptFor = 2 * time.Minute
|
||||
|
||||
// AskedKeptFor is how long an ask is kept after it was made: a month, as the router keeps its own.
|
||||
const AskedKeptFor = 30 * 24 * time.Hour
|
||||
|
||||
@@ -106,13 +149,16 @@ const (
|
||||
func IsControllerBucket(bucket string) bool {
|
||||
return bucket == CallsBucket || bucket == HandActsBucket || bucket == ConditionsBucket ||
|
||||
bucket == ConditionHistoryBucket || bucket == LeaseBucket || bucket == AskedBucket ||
|
||||
bucket == AssignmentsBucket || bucket == DeclaredConfigBucket || bucket == PredefinedBucket
|
||||
bucket == AssignmentsBucket || bucket == DeclaredConfigBucket || bucket == PredefinedBucket ||
|
||||
bucket == MergeWindowBucket || bucket == RolloutSilentBucket || bucket == CheckRunningBucket ||
|
||||
bucket == DeliveryMergesBucket
|
||||
}
|
||||
|
||||
// ControllerBuckets are the controller's own buckets, in the order they are asserted.
|
||||
func ControllerBuckets() []string {
|
||||
return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket, AskedBucket,
|
||||
AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket}
|
||||
AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket, MergeWindowBucket, RolloutSilentBucket, CheckRunningBucket,
|
||||
DeliveryMergesBucket}
|
||||
}
|
||||
|
||||
// ControllerBucketsAsserter is what raising the controller's buckets needs of a connection.
|
||||
@@ -209,6 +255,66 @@ func (j *JetStream) EnsureControllerBuckets() error {
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", AskedBucket, err)
|
||||
}
|
||||
// **The merge window: keys that expire.** A key's own time to live is the window; the marker an expired key
|
||||
// leaves is what a watcher hears, kept a minute (ADR 0407 decision 1).
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: MergeWindowBucket,
|
||||
Description: "the clock of every open batch's merge window (novox/hq ADR 0407): `<batch>.quiet` lives the " +
|
||||
"window and is made again by each merge, `<batch>.most` lives the most; the first to expire closes the " +
|
||||
"batch; written by the controller alone",
|
||||
History: 1,
|
||||
MaxValueSize: 4 << 10,
|
||||
MaxBytes: 8 << 20,
|
||||
Storage: jetstream.FileStorage,
|
||||
LimitMarkerTTL: MergeWindowMarkerTTL,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", MergeWindowBucket, err)
|
||||
}
|
||||
// **The rollout's silent keys: keys that expire** (ADR 0406 §5, issue 974). A key lives as long as the machine
|
||||
// may stay silent; the machine's report deletes it, and a marker of expiry is what the watcher hears.
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: RolloutSilentBucket,
|
||||
Description: "the wait of every machine a rollout sent a tier to (novox/hq ADR 0406): `<plan>.<tier>.<node>` " +
|
||||
"lives the time a machine may stay silent and is deleted by its report; one that expires names the " +
|
||||
"machine silent; written by the controller alone",
|
||||
History: 1,
|
||||
TTL: RolloutSilentKeptFor,
|
||||
MaxValueSize: 4 << 10,
|
||||
MaxBytes: 8 << 20,
|
||||
Storage: jetstream.FileStorage,
|
||||
LimitMarkerTTL: RolloutSilentMarkerTTL,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", RolloutSilentBucket, err)
|
||||
}
|
||||
// **The checks running: a key that ages** (ADR 0406 §3, issue 974). One message a key, each renewal a new one, so the
|
||||
// age of the bucket is how long a key outlives the builder that stopped renewing it.
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: CheckRunningBucket,
|
||||
Description: "the pull request checks the builders are running now (novox/hq ADR 0406): `<pull request>` holds the " +
|
||||
"head and trunk commit, the builder and its build; put by the builder when it starts and at each heartbeat, " +
|
||||
"deleted when the check ends, gone within two minutes of a builder that died; written by the build seat's holders",
|
||||
History: 1,
|
||||
TTL: CheckRunningKeptFor,
|
||||
MaxValueSize: 4 << 10,
|
||||
MaxBytes: 8 << 20,
|
||||
Storage: jetstream.FileStorage,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", CheckRunningBucket, err)
|
||||
}
|
||||
// **The merges mesh-delivery announced: keys that age** (ADR 0420 §7, issue 1025). One message a key; a week.
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: DeliveryMergesBucket,
|
||||
Description: "the merges mesh-delivery announced it made, each with the ask whose Approve it merged on (novox/hq " +
|
||||
"ADR 0420): `<owner>/<repo>/<number>/<head sha>` holds the announcement, kept a week; what lets the build " +
|
||||
"of a moved module register it from its new repository; written by the controller alone",
|
||||
History: 1,
|
||||
TTL: DeliveryMergesKeptFor,
|
||||
MaxValueSize: 16 << 10,
|
||||
MaxBytes: 16 << 20,
|
||||
Storage: jetstream.FileStorage,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", DeliveryMergesBucket, err)
|
||||
}
|
||||
// **The record mirrors: no age** (design 54 §5). A key is removed when its record is, by a purge whose
|
||||
// marker lives a day; a key that expired would be a record the bus forgot while the store still holds it.
|
||||
for _, m := range []struct{ bucket, why string }{
|
||||
|
||||
@@ -2,6 +2,7 @@ package broker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/nats-io/nats.go"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -268,3 +269,236 @@ func TestTheControllerMayAskMeshConfigInItsOwnName(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **The merge window is a bucket whose keys expire** (novox/hq ADR 0407 decision 1, issue 973): a key made with a
|
||||
// time to live goes when it is up, and a watcher on the bucket hears it go. The controller may write it, and it is
|
||||
// no undeclared state.
|
||||
func TestTheMergeWindowsKeysExpireAndAWatcherHearsIt(t *testing.T) {
|
||||
controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Contains(controller.Publish, "$KV."+MergeWindowBucket+".>") {
|
||||
t.Error("the controller may not write the merge window's bucket")
|
||||
}
|
||||
if !IsControllerBucket(MergeWindowBucket) || !slices.Contains(ControllerBuckets(), MergeWindowBucket) {
|
||||
t.Error("the merge window's bucket is not asserted as the controller's own")
|
||||
}
|
||||
js, err := Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer js.Close()
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, MergeWindowBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w, err := kv.WatchAll(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer w.Stop()
|
||||
if _, err := kv.Create(ctx, "plan-1.quiet", []byte("x"), jetstream.KeyTTL(time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case e := <-w.Updates():
|
||||
if e != nil && e.Key() == "plan-1.quiet" && e.Operation() != jetstream.KeyValuePut {
|
||||
return
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("no watcher heard the key expire")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **A rollout's wait for a machine is a bucket whose keys expire** (novox/hq ADR 0406 §5, issue 974): a key made with a
|
||||
// time to live goes when it is up, and a watcher hears it go as a purge; a key a report deleted is a delete. The
|
||||
// controller may write it, and it is no undeclared state.
|
||||
func TestTheRolloutsSilentKeysExpireAsAPurgeAndAReportsDeleteIsNot(t *testing.T) {
|
||||
controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Contains(controller.Publish, "$KV."+RolloutSilentBucket+".>") {
|
||||
t.Error("the controller may not write the rollout's silent keys' bucket")
|
||||
}
|
||||
if !IsControllerBucket(RolloutSilentBucket) || !slices.Contains(ControllerBuckets(), RolloutSilentBucket) {
|
||||
t.Error("the rollout's silent keys' bucket is not asserted as the controller's own")
|
||||
}
|
||||
js, err := Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer js.Close()
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, RolloutSilentBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer w.Stop()
|
||||
if _, err := kv.Create(ctx, "plan-1.0.ace", []byte("x"), jetstream.KeyTTL(time.Hour)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := kv.Delete(ctx, "plan-1.0.ace"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := kv.Create(ctx, "plan-1.0.g14", []byte("x"), jetstream.KeyTTL(time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case e := <-w.Updates():
|
||||
if e == nil || e.Operation() == jetstream.KeyValuePut {
|
||||
continue
|
||||
}
|
||||
if e.Key() == "plan-1.0.ace" && e.Operation() != jetstream.KeyValueDelete {
|
||||
t.Fatalf("a report's delete was heard as %v", e.Operation())
|
||||
}
|
||||
if e.Key() == "plan-1.0.g14" {
|
||||
if e.Operation() != jetstream.KeyValuePurge {
|
||||
t.Fatalf("an expiry was heard as %v, not a purge", e.Operation())
|
||||
}
|
||||
return
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("no watcher heard the key expire")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **The silent keys' bucket ages**, or its delete markers would fill it (issue 974, review 1): every report deletes a key,
|
||||
// the delete leaves a marker, and with one value per key and no age the markers stay for ever. A bucket made before the
|
||||
// age was set is brought to it by the next start, which the server accepts as an update of an existing bucket.
|
||||
func TestTheSilentKeysBucketAgesAndAnOlderOneIsBroughtToIt(t *testing.T) {
|
||||
js, err := Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer js.Close()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// As the first build made it: no age.
|
||||
if _, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: RolloutSilentBucket, History: 1,
|
||||
MaxValueSize: 4 << 10, MaxBytes: 8 << 20, Storage: jetstream.FileStorage, LimitMarkerTTL: RolloutSilentMarkerTTL}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatalf("a bucket made without the age is not brought to it: %v", err)
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, RolloutSilentBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
status, err := kv.Status(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if status.TTL() != RolloutSilentKeptFor {
|
||||
t.Errorf("the bucket ages at %v; want %v", status.TTL(), RolloutSilentKeptFor)
|
||||
}
|
||||
}
|
||||
|
||||
// **The merges have a consumer of their own, which a reset of the events consumer leaves alone** (issue 973, novox/hq
|
||||
// ADR 0407 decision 3): healer H4 re-makes the events consumer from now, dropping what it holds pending; a merge
|
||||
// waiting in the merges consumer is not that consumer's, and nothing would catch it up.
|
||||
func TestAResetOfTheEventsConsumerLeavesTheMergesAndTheirConsumerAlone(t *testing.T) {
|
||||
js, err := Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer js.Close()
|
||||
if err := AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := AssertMeshConsumers(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range MeshConsumers() {
|
||||
if c.Stream == EventsStream && c.Name == MergesConsumerName {
|
||||
if c.Resettable != "" || c.MaxDeliver != 0 || len(c.Filters) != 1 || c.Filters[0] != ControllerFollows[3] {
|
||||
t.Errorf("the merges consumer is %+v; want one filter, no max-deliver, not resettable", c)
|
||||
}
|
||||
}
|
||||
if c.Stream == EventsStream && c.Name == ControllerName {
|
||||
for _, f := range c.Filters {
|
||||
if f == ControllerFollows[3] {
|
||||
t.Errorf("the events consumer still follows the merges, which the merges consumer takes")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if _, err := js.Context().Publish(ControllerFollows[3], []byte(`{"commit":"c1"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.Context().Publish(ControllerFollows[0], []byte(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
merges, err := js.Context().ConsumerInfo(EventsStream, MergesConsumerName)
|
||||
if err != nil || merges.NumPending != 1 {
|
||||
t.Fatalf("the merges consumer holds %v (%v); want the merge", merges, err)
|
||||
}
|
||||
if _, _, err := js.ResetConsumer(EventsStream, ControllerName); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
merges, err = js.Context().ConsumerInfo(EventsStream, MergesConsumerName)
|
||||
if err != nil || merges.NumPending != 1 || merges.Delivered.Stream != 0 {
|
||||
t.Fatalf("after the reset the merges consumer holds %+v (%v); want its merge left alone", merges, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **Delivering this change loses no merge** (review of issue 973): a merge published under the old filter of the
|
||||
// `controller` consumer and never acknowledged by it is delivered by the merges consumer once the assertion has
|
||||
// made it, because it starts where `controller` has acknowledged up to, not from now.
|
||||
func TestAMergeTheOldConsumerHadNotAcknowledgedIsTheMergesConsumers(t *testing.T) {
|
||||
js, err := Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer js.Close()
|
||||
if err := AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old := &nats.ConsumerConfig{Durable: ControllerName, AckPolicy: nats.AckExplicitPolicy, AckWait: 30 * time.Second,
|
||||
MaxDeliver: 5, MaxAckPending: 1, DeliverSubject: "_DELIVER.controller.EVENTS", DeliverPolicy: nats.DeliverNewPolicy,
|
||||
FilterSubjects: append(append([]string(nil), ControllerFollows...), AgainFilter(ControllerName))}
|
||||
if _, err := js.Context().AddConsumer(EventsStream, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.Context().Publish(ControllerFollows[3], []byte(`{"commit":"c1"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := AssertMeshConsumers(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
merges, err := js.Context().ConsumerInfo(EventsStream, MergesConsumerName)
|
||||
if err != nil || merges.NumPending != 1 {
|
||||
t.Fatalf("the merges consumer holds %v (%v); want the merge the old consumer never acknowledged", merges, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,6 +36,12 @@ type Consumer struct {
|
||||
// them over, and a loop that acts on one message at a time is the property the store window
|
||||
// depends on. A queue group implies this, because a group has nothing to pull from.
|
||||
Push bool
|
||||
// TakesOverFrom names a consumer of the same stream whose filter this one takes over (issue 973): made while
|
||||
// that consumer still filters this one's subject, it starts after what that consumer has acknowledged, so a
|
||||
// message the old one held unacknowledged is not lost to the change. Otherwise it starts as FromNow says.
|
||||
TakesOverFrom string
|
||||
// DeliverSubject, when set, is where a push consumer delivers instead of DeliverSubjectFor's.
|
||||
DeliverSubject string
|
||||
// AckWaitSeconds before an unacknowledged delivery is redelivered.
|
||||
AckWaitSeconds int
|
||||
// MaxDeliver is how often a message is handed over before the consumer gives it up; zero for no
|
||||
|
||||
@@ -343,6 +343,9 @@ func (j *JetStream) EnsureConsumer(c Consumer) error {
|
||||
if c.FromNow {
|
||||
want.DeliverPolicy = nats.DeliverNewPolicy
|
||||
}
|
||||
if from := j.ackFloorOfTheConsumerItTakesOver(c); from > 0 {
|
||||
want.DeliverPolicy, want.OptStartSeq = nats.DeliverByStartSequencePolicy, from
|
||||
}
|
||||
if _, err := j.js.AddConsumer(c.Stream, want); err != nil {
|
||||
return fmt.Errorf("creating consumer %s on %s: %w", c.Name, c.Stream, err)
|
||||
}
|
||||
@@ -352,6 +355,26 @@ func (j *JetStream) EnsureConsumer(c Consumer) error {
|
||||
}
|
||||
}
|
||||
|
||||
// ackFloorOfTheConsumerItTakesOver is the stream sequence a consumer made now starts at so as to lose nothing the
|
||||
// consumer it takes over from (Consumer.TakesOverFrom) has not acknowledged: that consumer's ack floor plus one, when
|
||||
// it exists and still filters this one's subject; zero otherwise.
|
||||
func (j *JetStream) ackFloorOfTheConsumerItTakesOver(c Consumer) uint64 {
|
||||
if c.TakesOverFrom == "" || len(c.Filters) != 1 {
|
||||
return 0
|
||||
}
|
||||
have, err := j.js.ConsumerInfo(c.Stream, c.TakesOverFrom)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
filters := append([]string{have.Config.FilterSubject}, have.Config.FilterSubjects...)
|
||||
for _, f := range filters {
|
||||
if f == c.Filters[0] {
|
||||
return have.AckFloor.Stream + 1
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// ConsumerState is what a reset says about a consumer, before and after.
|
||||
type ConsumerState struct {
|
||||
DeliverPolicy string
|
||||
|
||||
+49
-1
@@ -217,6 +217,9 @@ var seatsTheControllerAsks = []string{"node-build-agent", "mesh-build-machine"}
|
||||
// original — and no other can (novox/hq issue 334, ADR 0264's consequences: no grant over the seats'
|
||||
// queues).
|
||||
func TheControllersAsk(stream, subject string) bool {
|
||||
if isCheckAsk(stream, subject) {
|
||||
return true
|
||||
}
|
||||
for _, seat := range seatsTheControllerAsks {
|
||||
if stream == seatStreamName(seat) && strings.HasPrefix(subject, "mesh.seat."+seat+".accept.") {
|
||||
return true
|
||||
@@ -368,6 +371,42 @@ var DegradableGrants = []DegradableGrant{
|
||||
Subject: "mesh.seat." + VerbsTheControllerAsksTheOperatorChannel[0].Seat + ".tool." + VerbsTheControllerAsksTheOperatorChannel[0].Verb,
|
||||
Reason: "refused: the controller raises the secret-given condition as before (ADR 0380 §9)",
|
||||
},
|
||||
{
|
||||
Subject: "$JS.ACK.EVENTS." + MergesConsumerName + ".>",
|
||||
Reason: "carried only at genesis: the controller composes its own permissions and pushes them to the bus at " +
|
||||
"its first push, which is when it acknowledges a merge (ADR 0407 decision 3, issue 973)",
|
||||
},
|
||||
{
|
||||
Subject: "$KV." + MergeWindowBucket + ".>",
|
||||
Reason: "refused: the controller closes a batch's window by a timer of its own for that batch, as the " +
|
||||
"window setting says (ADR 0407 decision 1, issue 973)",
|
||||
},
|
||||
{
|
||||
Subject: "mesh.seat." + TheBuildSeat + ".check.>",
|
||||
Reason: "carried only at genesis: the controller composes its own permissions and pushes them to the bus at its " +
|
||||
"first push, and a check asked before that is asked again by the next announcement of its head (ADR 0406 §3, " +
|
||||
"issue 974)",
|
||||
},
|
||||
{
|
||||
Subject: "$KV." + CheckRunningBucket + ".>",
|
||||
Reason: "refused: the controller asserts this bucket through the API and writes none of its keys — the build seat's " +
|
||||
"holders do (ADR 0406 §3, issue 974)",
|
||||
},
|
||||
{
|
||||
Subject: "mesh.seat." + ControllerSeat + ".event.sent",
|
||||
Reason: "refused: the controller says a tier sent to a machine only for outsiders, and a send that could not " +
|
||||
"be said is printed and goes on (ADR 0406 §5, issue 974)",
|
||||
},
|
||||
{
|
||||
Subject: "$KV." + DeliveryMergesBucket + ".>",
|
||||
Reason: "refused: the controller then keeps no merge mesh-delivery announced, so the build of a moved module is " +
|
||||
"refused as it is without the operator's Approve — failing closed (ADR 0420 §7, issue 1025)",
|
||||
},
|
||||
{
|
||||
Subject: "$KV." + RolloutSilentBucket + ".>",
|
||||
Reason: "refused: the controller looks at the plans when a wait for a machine's report is up, by a timer of its own " +
|
||||
"for that wait (ADR 0406 §5, issue 974)",
|
||||
},
|
||||
}
|
||||
|
||||
// perMachineEvents are a node-scoped seat's events about the holder itself, whose last token is the
|
||||
@@ -555,6 +594,11 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// the role, and whichever machine holding it is idle takes it.
|
||||
for _, seat := range seatsTheControllerAsks {
|
||||
pub = append(pub, "mesh.seat."+seat+".accept.>")
|
||||
// **And its checks, on the two queues of their own** (novox/hq ADR 0406 §3, issue 974): one subject a pull
|
||||
// request on each, under `check`; the retired role's checks stay on its accepts.
|
||||
if seat == TheBuildSeat {
|
||||
pub = append(pub, "mesh.seat."+seat+".check.>")
|
||||
}
|
||||
// **And its holders' verbs, on every machine** (novox/hq ADR 0219): what a holder is
|
||||
// building, kill it, pause it, resume it. The queue is the controller's to show and to
|
||||
// change, and what one machine is doing with an ask it took only that machine can say.
|
||||
@@ -635,7 +679,7 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// delivery from EVENTS — a consumer that cannot ack has every message redelivered for
|
||||
// ever, refused by the list it already has.
|
||||
sub = append(sub, ControllerFollows...)
|
||||
pub = append(pub, "$JS.ACK.EVENTS."+ControllerName+".>")
|
||||
pub = append(pub, "$JS.ACK.EVENTS."+ControllerName+".>", "$JS.ACK.EVENTS."+MergesConsumerName+".>")
|
||||
|
||||
// **Where an enrolment's answer goes**, and `allow_responses` does not cover it. That
|
||||
// permits one reply to the reply subject of a message the user received — and a message a
|
||||
@@ -863,6 +907,10 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
"$JS.API.CONSUMER.INFO."+stream+"."+worker,
|
||||
"$JS.API.CONSUMER.MSG.NEXT."+stream+"."+worker,
|
||||
"$JS.ACK."+stream+"."+worker+".>")
|
||||
// And the checks' two workers, for the build role they are queued for (novox/hq ADR 0406 §3).
|
||||
if s.Name == TheBuildSeat {
|
||||
pub = append(pub, holderCheckGrants()...)
|
||||
}
|
||||
// **And whether an ask it took was cancelled** (novox/hq ADR 0219): one key of the
|
||||
// seat's cancelled set, read directly by its id, so a holder that fetched an ask in the
|
||||
// moment the controller cancelled it ends it instead of building it. Read, never written:
|
||||
|
||||
@@ -34,6 +34,8 @@ func TestTheFactsTheGrantPermitsAreTheFactsTheMeshStates(t *testing.T) {
|
||||
states = append(states, link.KeyChecked)
|
||||
// And a walk kept in a new state, for the delivery it walks (novox/hq ADR 0239).
|
||||
states = append(states, link.KeyPlanMoved)
|
||||
// And a tier sent to a machine, once the send is kept (novox/hq ADR 0406 §5, issue 974).
|
||||
states = append(states, link.KeySent)
|
||||
// And a key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3; design 57 F3).
|
||||
states = append(states, link.KeyAssignmentChanged, link.KeyDeclaredConfigChanged, link.KeyPredefinedChanged)
|
||||
for _, event := range states {
|
||||
|
||||
+66
-11
@@ -147,7 +147,7 @@ func OriginalOfAgain(subject string) (string, bool) {
|
||||
// every tool invocation in the mesh, and a tool call must never be persisted (design 25 §3 keeps
|
||||
// tools on core NATS, where a lost call is a timeout the caller already handles).
|
||||
func MeshStreams() []Stream {
|
||||
return []Stream{
|
||||
return append([]Stream{
|
||||
{
|
||||
Name: "CONTROL",
|
||||
// A build's outcome is no longer here: it is the build-machine seat's own event, so one
|
||||
@@ -221,7 +221,7 @@ func MeshStreams() []Stream {
|
||||
"failed with the reason (novox/hq ADR 0349 §4): what a walk's end and a gate wait for (ADR 0362, " +
|
||||
"ADR 0349 §15), read by subject; a week, as the events are",
|
||||
},
|
||||
}
|
||||
}, CheckStreams()...)
|
||||
}
|
||||
|
||||
// ProvAppliedStream keeps what each machine's provisioner client last said of each module's files, newest per
|
||||
@@ -233,6 +233,18 @@ const ProvAppliedStream = "PROV_APPLIED"
|
||||
// all (ProvisionerAppliedSubject).
|
||||
const ProvAppliedSubjects = "prov.applied.>"
|
||||
|
||||
// eventsTheControllerFollows are the events the controller's events consumer takes: all it follows but the forge's
|
||||
// merges, which the merges consumer takes (MergesConsumerName).
|
||||
func eventsTheControllerFollows() []string {
|
||||
var out []string
|
||||
for _, subject := range ControllerFollows {
|
||||
if subject != ControllerFollows[3] {
|
||||
out = append(out, subject)
|
||||
}
|
||||
}
|
||||
return append(out, ControllerHears...)
|
||||
}
|
||||
|
||||
// DeliverSubjectFor is where a push consumer's messages land.
|
||||
//
|
||||
// **Per consumer, which means per stream as well as per name** (novox/hq 04-ISSUES/146). A push
|
||||
@@ -248,6 +260,9 @@ const ProvAppliedSubjects = "prov.applied.>"
|
||||
// durable's name to its stream, and this subject was the one place that scoping was dropped. It
|
||||
// stays inside what a controller may already subscribe (`_DELIVER.controller.>`).
|
||||
func DeliverSubjectFor(c Consumer) string {
|
||||
if c.DeliverSubject != "" {
|
||||
return c.DeliverSubject
|
||||
}
|
||||
return "_DELIVER." + c.Name + "." + c.Stream
|
||||
}
|
||||
|
||||
@@ -317,6 +332,13 @@ func Overlaps() []string {
|
||||
// ack subject is derived from (nats.go: `$JS.ACK.<stream>.controller.>`).
|
||||
const ControllerName = "controller"
|
||||
|
||||
// MergesConsumerName is the controller's durable consumer of the forge's merged event on EVENTS, apart from the
|
||||
// events consumer (novox/hq ADR 0407 decision 3, issue 973). A merge is never dropped, so this consumer has no
|
||||
// max-deliver, which would send a merge held five times to DEAD_LETTERS to wait for a person, and no healer
|
||||
// resets it, which would drop the merges it holds. A consumer of its own because both of those are properties of
|
||||
// a consumer, not of a subject.
|
||||
const MergesConsumerName = "controller-merges"
|
||||
|
||||
// ControllerSeat is the role the control plane holds, and ControllerStates are the facts it states
|
||||
// under it (novox/hq ADR 0134).
|
||||
//
|
||||
@@ -346,6 +368,8 @@ var ControllerStates = []string{"applied", "refused", "built-before",
|
||||
"checked",
|
||||
// And a walk kept in a new state (novox/hq ADR 0239): what the delivery it walks reads its steps from.
|
||||
"plan-moved",
|
||||
// And a tier sent to a machine, once the send is kept (novox/hq ADR 0406 §5, issue 974).
|
||||
"sent",
|
||||
// And a key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3): the assignments and
|
||||
// each registered module's declared config (design 57 F3).
|
||||
"assignment-changed", "declared-config-changed", "predefined-changed"}
|
||||
@@ -404,6 +428,19 @@ var ControllerFollows = []string{
|
||||
DecidedSubject,
|
||||
}
|
||||
|
||||
// ControllerHears are events the controller's events consumer takes beside the ones it follows, and that need no
|
||||
// subscribe grant of their own. **A push consumer delivers onto its deliver subject** (`_DELIVER.controller.>`, already
|
||||
// granted), so the subject a consumer filters on is never one the controller subscribes: listing this one in
|
||||
// ControllerFollows would add a subscribe grant the installer's carried user list, which lives in mesh-host, would
|
||||
// have to carry too (TestTheInstallersFirstUserListIsWhatTheControllerWouldCompose), for nothing the consumer uses.
|
||||
//
|
||||
// A merge mesh-delivery made, announced (novox/hq ADR 0420 §7, issue 1025): with the ask whose Approve it merged on and
|
||||
// the heads that Approve covered, kept a week to read the warrant a moved module's registration needs.
|
||||
var ControllerHears = []string{DeliveryMergedSubject}
|
||||
|
||||
// DeliveryMergedSubject is where mesh-delivery says it merged a pull request.
|
||||
var DeliveryMergedSubject = moduleEventSubject("mesh-delivery", "merged")
|
||||
|
||||
// AsksSeat is the seat an ask is made on and its warrant heard from (novox/hq ADR 0259): the router's.
|
||||
const AsksSeat = "operator-channel"
|
||||
|
||||
@@ -441,7 +478,7 @@ const EventsStream = "EVENTS"
|
||||
// held through a store restart — the exact message the stream exists to protect — some minutes
|
||||
// before the controller had finished deciding about it.
|
||||
func MeshConsumers() []Consumer {
|
||||
return []Consumer{
|
||||
return append([]Consumer{
|
||||
{
|
||||
Name: ControllerName,
|
||||
Stream: "CONTROL",
|
||||
@@ -450,10 +487,27 @@ func MeshConsumers() []Consumer {
|
||||
Why: "the controller is the single consumer of what nodes say; explicit ack and no " +
|
||||
"max-deliver, because the store window's bound is the controller's own",
|
||||
},
|
||||
// The forge's merges (novox/hq ADR 0407 decision 3, issue 973): never dropped, so no max-deliver and no
|
||||
// reset; a merge the controller cannot record yet is held with a growing delay until it can. **Asserted before
|
||||
// the events consumer**, which stops filtering merges: the order of this list is the order of assertion.
|
||||
{
|
||||
Name: MergesConsumerName,
|
||||
Stream: "EVENTS",
|
||||
// Within what the controller may subscribe already (`_DELIVER.controller.>`).
|
||||
DeliverSubject: "_DELIVER." + ControllerName + ".merges",
|
||||
Filters: []string{ControllerFollows[3]},
|
||||
Push: true,
|
||||
AckWaitSeconds: 30,
|
||||
MaxAckPending: 1,
|
||||
FromNow: true,
|
||||
TakesOverFrom: ControllerName,
|
||||
Why: "the forge's merges, one at a time; no max-deliver and never reset, because a merge is never " +
|
||||
"dropped and no person releases one",
|
||||
},
|
||||
{
|
||||
Name: ControllerName,
|
||||
Stream: "EVENTS",
|
||||
Filters: append(append([]string(nil), ControllerFollows...), AgainFilter(ControllerName)),
|
||||
Filters: append(eventsTheControllerFollows(), AgainFilter(ControllerName)),
|
||||
Push: true,
|
||||
AckWaitSeconds: 30,
|
||||
MaxDeliver: 5,
|
||||
@@ -463,12 +517,13 @@ func MeshConsumers() []Consumer {
|
||||
MaxAckPending: 1,
|
||||
FromNow: true,
|
||||
// **The one consumer the mesh resets by itself** (healer H4, issue 248): it fell a week
|
||||
// behind once and held every merge after it. What a reset drops is caught up elsewhere —
|
||||
// a merge by the catch-up pass that reads the forge (issue 266), a build's outcome from the
|
||||
// build records a plan settles from (issue 214), a provider's failing word by the provider
|
||||
// saying it again every quarter of an hour (ADR 0224).
|
||||
Resettable: "what it drops is caught up: merges by the catch-up pass (issue 266), build outcomes " +
|
||||
"from the build records (issue 214), a provider's failing word said again (ADR 0224)",
|
||||
// behind once and held every merge after it. What a reset drops is caught up elsewhere — a
|
||||
// build's outcome from the build records a plan settles from (issue 214), a provider's failing
|
||||
// word by the provider saying it again every quarter of an hour (ADR 0224). **Not the merges**:
|
||||
// they are the merges consumer's below, which a reset leaves alone (ADR 0407 decision 3).
|
||||
Resettable: "what it drops is caught up: build outcomes from the build records (issue 214), a " +
|
||||
"provider's failing word said again (ADR 0224); the forge's merges are not its, they have a " +
|
||||
"consumer of their own that no reset touches",
|
||||
Why: "the events the mesh's own controller reacts to, one at a time; after " +
|
||||
"max-deliver it gives the event up, and the controller keeps it in DEAD_LETTERS until " +
|
||||
"a person delivers it again or drops it",
|
||||
@@ -483,7 +538,7 @@ func MeshConsumers() []Consumer {
|
||||
Why: "the controller copies each message a consumer gave up on into DEAD_LETTERS; no max-deliver, " +
|
||||
"because a notice it gave up on would lose the message it is about",
|
||||
},
|
||||
}
|
||||
}, CheckWorkers()...)
|
||||
}
|
||||
|
||||
// NoticesConsumer is the controller's consumer on DEAD_LETTER_NOTICES (novox/hq issue 330).
|
||||
|
||||
@@ -134,6 +134,9 @@ func TestEachStreamCarriesTheRetentionItsShapeNeeds(t *testing.T) {
|
||||
// What each machine's provisioner client last said of each module's files (design 57 P6): the newest
|
||||
// word per module, what a walk's end reads.
|
||||
"PROV_APPLIED": RetentionLastPerSubject,
|
||||
// The checks of pull requests, in two work queues of their own (novox/hq ADR 0406 §3, issue 974).
|
||||
"CHECKS_READY": RetentionWorkQueue,
|
||||
"CHECKS_OTHER": RetentionWorkQueue,
|
||||
}
|
||||
got := map[string]Retention{}
|
||||
for _, s := range MeshStreams() {
|
||||
|
||||
+1
-1
@@ -24,7 +24,7 @@ accounts {
|
||||
jetstream: enabled
|
||||
users = [
|
||||
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_predefined.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
|
||||
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
|
||||
@@ -41,6 +41,14 @@ func ownMachine(p Principal, pattern string) bool {
|
||||
return p.Kind == KindNode && len(tokens) > 2 && tokens[2] == p.Node
|
||||
}
|
||||
|
||||
// holdsTheBuildSeat is a principal holding the build seat, or the controller, which asserts the bucket the holders write.
|
||||
func holdsTheBuildSeat(p Principal, _ string) bool {
|
||||
if p.Kind == KindController {
|
||||
return true
|
||||
}
|
||||
return holdsSeatOf(p, "mesh.seat."+TheBuildSeat+".x")
|
||||
}
|
||||
|
||||
// holdsSeatOf is a principal holding the seat a `mesh.seat.<seat>.…` pattern names: its module's own
|
||||
// principal, or the node tools carrying a module that holds it (ADR 0175, the runtime is its modules).
|
||||
func holdsSeatOf(p Principal, pattern string) bool {
|
||||
@@ -117,8 +125,16 @@ var WritersTable = []WriterRow{
|
||||
Subjects: []string{"mesh.control.*.report", "mesh.control.*.health", "mesh.control.*.cli"}, Writes: ownMachine},
|
||||
{State: "the controller lease", Writer: "the controller instance holding it", KeptIn: "key-value " + LeaseBucket,
|
||||
Others: "a candidate waits", Subjects: kvOf(LeaseBucket), Writes: isController},
|
||||
// And the clock of an open batch's merge window, a plan before it is one (novox/hq ADR 0407 decision 1), and the
|
||||
// wait of a rollout for each machine it sent (ADR 0406 §5, issue 974): keys that expire in buckets of their own,
|
||||
// written by the lease holder.
|
||||
{State: "plans and their tiers", Writer: "controller (lease holder), compare-and-set on the plan's revision",
|
||||
KeptIn: "the controller's store", Others: "read through plans"},
|
||||
KeptIn: "the controller's store; a batch's merge window in key-value " + MergeWindowBucket +
|
||||
", a rollout's wait for a machine's report in key-value " + RolloutSilentBucket +
|
||||
", and the merges mesh-delivery announced, for a moved module's registration (ADR 0420 §7), in key-value " +
|
||||
DeliveryMergesBucket, Others: "read through plans",
|
||||
Subjects: append(append(kvOf(MergeWindowBucket), kvOf(RolloutSilentBucket)...), kvOf(DeliveryMergesBucket)...),
|
||||
Writes: isController},
|
||||
{State: "conditions", Writer: "controller", KeptIn: "key-value " + ConditionsBucket + " (and its history, " +
|
||||
ConditionHistoryBucket + "); how often each was observed, when last and its newest evidence in the " +
|
||||
"controller's store, so a look that changes nothing writes no bucket", Others: "raise or clear only through observations the controller reads",
|
||||
@@ -143,6 +159,15 @@ var WritersTable = []WriterRow{
|
||||
Subjects: []string{"mesh.seat.node-build-agent.event.built", "mesh.seat.mesh-build-machine.event.built"},
|
||||
Writes: holdsSeatOf,
|
||||
Shared: "every machine holding the build seat answers the asks it took; each outcome names its ask"},
|
||||
{State: "a pull request's check asked", Writer: "controller",
|
||||
KeptIn: "the work queues " + ChecksReadyStream + " and " + ChecksOtherStream + ", one subject a pull request, the newest kept",
|
||||
Others: "the build seat's holders take it",
|
||||
Subjects: []string{CheckSubjectPrefix(TheBuildSeat, true) + ">", CheckSubjectPrefix(TheBuildSeat, false) + ">"},
|
||||
Writes: isController},
|
||||
{State: "a pull request's check running", Writer: "the build seat's holders — several by design, each the key of the check it runs",
|
||||
KeptIn: "key-value " + CheckRunningBucket + ", aged out when not renewed", Others: "the build seat's holders read it to drop a duplicate request",
|
||||
Subjects: kvOf(CheckRunningBucket), Writes: holdsTheBuildSeat,
|
||||
Shared: "every machine holding the build seat puts the key of the check it runs and deletes it when the check ends"},
|
||||
{State: "a merge announced", Writer: "one announcer per forge (the hook, or the poll when the hook is absent — never both)",
|
||||
KeptIn: "the bus", Others: "—", Subjects: []string{"mesh.mod.*.event.pull.merged"}, Writes: ownModule},
|
||||
// A pull request's head, before it merges (novox/hq to-be 45 §9): the same one announcer.
|
||||
|
||||
@@ -27,6 +27,8 @@ var designRows = []string{
|
||||
"the healers' acts and their brake",
|
||||
"stream definitions and bus permissions",
|
||||
"builds and their outcomes",
|
||||
"a pull request's check asked",
|
||||
"a pull request's check running",
|
||||
"a merge announced",
|
||||
"a pull request's head announced",
|
||||
"a pull request's merge check",
|
||||
|
||||
@@ -111,7 +111,7 @@ func boundNeed(n Needed, bound Chosen, catalogue map[string]Manifest, node Node,
|
||||
}
|
||||
held.From, held.At, held.Module = node.Name, at, m.Module
|
||||
held.Serves, held.SharedOwn, held.Identity = servedByOne(m, n.Name), sharedByOne(m, n.Name), m.IdentityBoundOf(n.Name)
|
||||
held.CredentialForm = m.CredentialFormOf(n.Name)
|
||||
held.CredentialForm, held.CredentialTool = m.CredentialFormOf(n.Name), m.CredentialToolOf(n.Name)
|
||||
return held, true, ""
|
||||
}
|
||||
matching := bound.among(world.Offered[n.Name])
|
||||
@@ -125,7 +125,7 @@ func boundNeed(n Needed, bound Chosen, catalogue map[string]Manifest, node Node,
|
||||
identity := DefaultIdentityBound
|
||||
if pm, known := catalogue[p.Module]; known {
|
||||
held.SharedOwn, _ = pm.SharedCredentialOf(n.Name)
|
||||
held.CredentialForm = pm.CredentialFormOf(n.Name)
|
||||
held.CredentialForm, held.CredentialTool = pm.CredentialFormOf(n.Name), pm.CredentialToolOf(n.Name)
|
||||
identity = pm.IdentityBoundOf(n.Name)
|
||||
}
|
||||
held.From, held.At, held.Module, held.Serves, held.Identity = p.Node, p.At, p.Module, p.Serves, identity
|
||||
|
||||
@@ -3,6 +3,7 @@ package catalogue
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
@@ -16,9 +17,22 @@ import (
|
||||
// real parser (novox/hq ADR 0100):
|
||||
// the store and the broker say which of their ports the mesh guards on an adopted node, and the
|
||||
// filter module loads its table through a unit of its own whose stop deletes only that table.
|
||||
// moduleTree is where a module is read from: this repository's own modules/<module> when it lives whole here
|
||||
// (novox/hq ADR 0420 §1, issue 1019: build-agent and route-proxy), with this repository as the tree root;
|
||||
// otherwise the catalogue beside, with the catalogue as the root.
|
||||
func moduleTree(t *testing.T, module string) (root, dir string) {
|
||||
t.Helper()
|
||||
own := filepath.Join("..", "..", "modules", module)
|
||||
if _, err := os.Stat(filepath.Join(own, "module.json")); err == nil {
|
||||
return filepath.Join("..", ".."), own
|
||||
}
|
||||
return beside.Dir(t, "mesh-catalog"), filepath.Join(beside.Catalogue(t), module)
|
||||
}
|
||||
|
||||
func catalogueManifest(t *testing.T, module string) Manifest {
|
||||
t.Helper()
|
||||
m, err := ReadTreeManifest(beside.Dir(t, "mesh-catalog"), filepath.Join(beside.Catalogue(t), module))
|
||||
root, dir := moduleTree(t, module)
|
||||
m, err := ReadTreeManifest(root, dir)
|
||||
if err != nil {
|
||||
t.Fatalf("%s does not parse:\n%v", module, err)
|
||||
}
|
||||
@@ -327,8 +341,8 @@ func forgeBuilt(forge Manifest) []Built {
|
||||
// templates composes with it, and one that did not is resolved as it was.
|
||||
func catalogueResolved(t *testing.T, module string, built ...Built) Manifest {
|
||||
t.Helper()
|
||||
dir := filepath.Join(beside.Catalogue(t), module)
|
||||
m, err := ReadTreeManifestWritten(beside.Dir(t, "mesh-catalog"), dir)
|
||||
root, dir := moduleTree(t, module)
|
||||
m, err := ReadTreeManifestWritten(root, dir)
|
||||
if err != nil {
|
||||
t.Fatalf("%s does not parse:\n%v", module, err)
|
||||
}
|
||||
|
||||
@@ -104,6 +104,9 @@ type PlanProvision struct {
|
||||
Provider string `json:"provider"`
|
||||
ProviderModule string `json:"provider-module"`
|
||||
CredentialForm string `json:"credential-form,omitempty"`
|
||||
// CredentialTool is, for an offer whose credential the provider's software issues (issue 928), the provider's tool
|
||||
// the vault asks for each holder's value; empty for any other form.
|
||||
CredentialTool string `json:"credential-tool,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
// Contributes is what the consumer's manifest tells its provider for this provision (Manifest.Contributes: a
|
||||
// database's name, its extensions), so the provider's helper learns it from the bus (issue 776). Nil where the
|
||||
@@ -134,7 +137,7 @@ func provisionsOf(needs []Needed, module, as string, contributes map[string]map[
|
||||
continue
|
||||
}
|
||||
p := PlanProvision{Provision: n.Name, Local: n.Local, Provider: n.From, ProviderModule: n.Module,
|
||||
CredentialForm: n.CredentialForm, Contributes: maps.Clone(contributes[n.Name]),
|
||||
CredentialForm: n.CredentialForm, CredentialTool: n.CredentialTool, Contributes: maps.Clone(contributes[n.Name]),
|
||||
At: n.At, From: n.From, As: as}
|
||||
// **A credential only a template names** (issue 788): named like a file's, so the client fills it from the
|
||||
// vault's row for this holder.
|
||||
|
||||
@@ -123,3 +123,37 @@ func TestAHandedModulesLocalCredentialsOfTheVaultAreNamedByLocalInItsPlan(t *tes
|
||||
t.Fatalf("the plan names %v, want admin and internal-token", names)
|
||||
}
|
||||
}
|
||||
|
||||
// **Issue 969: a key its provider serves only at run time.** A moved module's file may name `${bound:…}` of a
|
||||
// provision whose provider tells it at run time (ssh-client's known_host of the forge). Where the machine takes
|
||||
// handed files the client fills it and the module composes; where it does not the controller must fill it from what
|
||||
// the provider serves statically, cannot, and refuses it with the very words an unmarked module is refused with
|
||||
// (ADR 0349 §17: the old way, byte for byte).
|
||||
func TestAHandedModuleNamingARunTimeBoundKeyComposesOnlyWhereItIsHanded(t *testing.T) {
|
||||
text := "Host ${bound:postgres-database:at}\n KnownHost ${bound:postgres-database:known_host}\n"
|
||||
moved := strings.Replace(botMoved(), `"own-secrets":{"token":"${dir:state}/token"}`,
|
||||
`"requires":["postgres-database"],"own-secrets":{"token":"${dir:state}/token"}`, 1)
|
||||
unmoved := strings.Replace(moved, `"moved":true,`, ``, 1)
|
||||
if unmoved == moved {
|
||||
t.Fatal("the twin is still marked moved")
|
||||
}
|
||||
|
||||
c, got := composeWith(t, withDatabase(t, moved, text, true), handingTo("store:5000"))
|
||||
if strings.Contains(got, "known_host") || len(c.Filled) == 0 {
|
||||
t.Fatalf("handed, the module is filled by the client and the controller names nothing: %v %s", c.Filled, got)
|
||||
}
|
||||
|
||||
for name, with := range map[string]Rendering{
|
||||
"an engine that does not take a filled file alone": {Provisioner: "mesh-provisioner-client", ArtifactStore: "store:5000"},
|
||||
"no provisioner client on the machine": {Hands: true, ArtifactStore: "store:5000"},
|
||||
} {
|
||||
_, err := withDatabase(t, moved, text, true).Compose(sealedForBot(with))
|
||||
if err == nil || !strings.Contains(err.Error(), `asks its postgres-database binding for "known_host"`) {
|
||||
t.Fatalf("%s: the key is not refused: %v", name, err)
|
||||
}
|
||||
_, before := withDatabase(t, unmoved, text, false).Compose(sealedForBot(with))
|
||||
if before == nil || before.Error() != err.Error() {
|
||||
t.Fatalf("%s: refused otherwise than the unmarked module: %v / %v", name, err, before)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -242,3 +242,33 @@ func TestAnOfferAnsweredByTheHelperIsReadAndHandsNoFile(t *testing.T) {
|
||||
t.Fatalf("the flag changed what is composed or handed:\nbefore: %s\nnow: %s\n%v %v", before, got, c2.Filled, c2.Plans)
|
||||
}
|
||||
}
|
||||
|
||||
// A plan entry names the tool of an offer whose credential the provider's software issues (issue 928), so the vault
|
||||
// asks that tool for each holder; an offer of any other form names none.
|
||||
func TestAHandedModulesPlanNamesTheCredentialToolOfASoftwareIssuedOffer(t *testing.T) {
|
||||
for credential, want := range map[string]string{
|
||||
``: "",
|
||||
`,"credential":{"own":"admin"}`: "",
|
||||
`,"credential":{"issued-by":"outside"}`: "",
|
||||
`,"credential":{"issued-by":"software","tool":"gitea_agent_token"}`: "gitea_agent_token",
|
||||
} {
|
||||
raw := `{"module":"store","version":"1","provides":[{"name":"postgres-database","scope":"mesh"` +
|
||||
credential + `}],"serves":{"postgres-database":{"port":5432}}`
|
||||
if strings.Contains(credential, `"own"`) {
|
||||
raw += `,"own-secrets":{"admin":{"path":"/var/lib/store/admin","taken":"at-start"}}`
|
||||
}
|
||||
node := workstation()
|
||||
node.At = "workstation.internal"
|
||||
consumer := notifierConsumer(t, true, false)
|
||||
r, err := Resolve(shelf(consumer, mustParse(t, raw+`}`)), []string{consumer.Module}, node, twoStores("anchor"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.Account = "op"
|
||||
c, _ := composeWith(t, r, handingTo("store:5000"))
|
||||
got := c.Plans["notify"].Provisions
|
||||
if len(got) != 1 || got[0].CredentialTool != want {
|
||||
t.Errorf("an offer with %q: the plan names %+v, want the tool %q", credential, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A jail whose pattern names <HOST> twice is refused by name: fail2ban would read it as a duplicate
|
||||
// capture group and refuse its whole configuration, leaving the machine with no bans at all — which
|
||||
// is what happened on the control node the day this was built (novox/hq ADR 0179).
|
||||
func TestAJailsPatternNamesTheHostOncePerShape(t *testing.T) {
|
||||
twice := Manifest{Module: "route-proxy", Jails: []Jail{{
|
||||
Name: "route-proxy",
|
||||
Failregex: `^.*(?:refused for <HOST>:\d+|asked from <HOST>:\d+)$`,
|
||||
Jail: "backend = systemd",
|
||||
}}}
|
||||
problems := twice.jailProblems()
|
||||
if len(problems) != 1 || !strings.Contains(problems[0], "names <HOST> 2 times") {
|
||||
t.Fatalf("a pattern naming <HOST> twice was not refused by name: %v", problems)
|
||||
}
|
||||
if !strings.Contains(problems[0], "no bans at all") {
|
||||
t.Errorf("the refusal does not say what it costs: %q", problems[0])
|
||||
}
|
||||
|
||||
// The same two shapes, one pattern each, is how it is written — and is accepted.
|
||||
split := Manifest{Module: "route-proxy", Jails: []Jail{{
|
||||
Name: "route-proxy",
|
||||
Failregex: "^.*refused for <HOST>:\\d+$\n ^.*asked from <HOST>:\\d+$",
|
||||
Jail: "backend = systemd",
|
||||
}}}
|
||||
if problems := split.jailProblems(); len(problems) != 0 {
|
||||
t.Fatalf("one <HOST> per pattern was refused: %v", problems)
|
||||
}
|
||||
}
|
||||
|
||||
// A jail with no name, no pattern, or a name another jail of the same module already took is refused.
|
||||
func TestAJailIsNamedOnceAndSaysWhatAFailureLooksLike(t *testing.T) {
|
||||
m := Manifest{Module: "mailu", Jails: []Jail{
|
||||
{Name: "", Failregex: "^.*rip=<HOST>$"},
|
||||
{Name: "mailu-front", Failregex: " "},
|
||||
{Name: "mailu-front", Failregex: "^.*rip=<HOST>$"},
|
||||
}}
|
||||
problems := strings.Join(m.jailProblems(), "\n")
|
||||
for _, want := range []string{"a jail with no name", "says nothing a failed attempt looks like", "two jails called"} {
|
||||
if !strings.Contains(problems, want) {
|
||||
t.Errorf("missing %q in:\n%s", want, problems)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -303,6 +303,13 @@ func (m Manifest) IssuedCredentialOf(provision string) (string, bool) {
|
||||
return "", false
|
||||
}
|
||||
|
||||
// CredentialToolOf is the tool a plan names for a provision, set only where the offer says the provider's software
|
||||
// issues its credential (issue 928); empty otherwise.
|
||||
func (m Manifest) CredentialToolOf(provision string) string {
|
||||
tool, _ := m.IssuedCredentialOf(provision)
|
||||
return tool
|
||||
}
|
||||
|
||||
// IssuedOutsideOf is whether this module's offer of a provision says only a person outside the mesh can give its
|
||||
// credential (novox/hq ADR 0337): the mesh never makes a value for it, and a consumer holds one only once a person
|
||||
// gave it with `secret accept`.
|
||||
|
||||
@@ -230,6 +230,8 @@ type Needed struct {
|
||||
// CredentialForm is the credential form the provider's offer declares (Manifest.CredentialFormOf, issue 768),
|
||||
// named in a handed consumer's plan for the vault; empty where the provider's manifest is not in hand.
|
||||
CredentialForm string
|
||||
// CredentialTool is the tool of the offer that the provider's software issues the credential through (issue 928).
|
||||
CredentialTool string
|
||||
// Manager is set when this holder is a refreshable-grant licence's MANAGER, delivered the refresh
|
||||
// token rather than an access token (novox/hq ADR 0050). It changes one thing downstream: an empty
|
||||
// Sealed is tolerated — the manager has not adopted a refresh token yet, which is a real waiting
|
||||
@@ -543,7 +545,7 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
|
||||
needs = append(needs, Needed{
|
||||
Name: want, From: node.Name, At: at, Module: by.Module,
|
||||
Serves: servedByOne(by, want), For: because[want],
|
||||
SharedOwn: sharedByOne(by, want), CredentialForm: by.CredentialFormOf(want),
|
||||
SharedOwn: sharedByOne(by, want), CredentialForm: by.CredentialFormOf(want), CredentialTool: by.CredentialToolOf(want),
|
||||
Identity: by.IdentityBoundOf(want)})
|
||||
} else if served := servedByOne(by, want); len(served) > 0 {
|
||||
// Answered here with no credential to mint, but the provider serves facts the
|
||||
@@ -565,7 +567,7 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
|
||||
}
|
||||
needs = append(needs, Needed{
|
||||
Name: want, From: node.Name, At: at, Serves: served, For: because[want], Module: by.Module,
|
||||
CredentialForm: by.CredentialFormOf(want)})
|
||||
CredentialForm: by.CredentialFormOf(want), CredentialTool: by.CredentialToolOf(want)})
|
||||
}
|
||||
continue
|
||||
}
|
||||
@@ -594,17 +596,17 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
|
||||
node.Name, want, p.Node, meshNetwork))
|
||||
return
|
||||
}
|
||||
shared, form := "", ""
|
||||
shared, form, tool := "", "", ""
|
||||
// A provider whose definition is not in hand is held to the tightest bound the
|
||||
// mesh knows, not to none: what it keeps is not known here (ADR 0225).
|
||||
bound := DefaultIdentityBound
|
||||
if pm, known := catalogue[p.Module]; known {
|
||||
shared, _ = pm.SharedCredentialOf(want)
|
||||
form = pm.CredentialFormOf(want)
|
||||
form, tool = pm.CredentialFormOf(want), pm.CredentialToolOf(want)
|
||||
bound = pm.IdentityBoundOf(want)
|
||||
}
|
||||
needs = append(needs, Needed{Name: want, From: p.Node, At: p.At, Module: p.Module,
|
||||
Serves: p.Serves, For: because[want], SharedOwn: shared, CredentialForm: form, Identity: bound,
|
||||
Serves: p.Serves, For: because[want], SharedOwn: shared, CredentialForm: form, CredentialTool: tool, Identity: bound,
|
||||
LeftOut: p.LeftOut})
|
||||
}
|
||||
switch {
|
||||
|
||||
@@ -108,6 +108,8 @@ var defaultSeats = append([]Seat{
|
||||
"checked",
|
||||
// A walk kept in a new state, for the delivery it walks (novox/hq ADR 0239).
|
||||
"plan-moved",
|
||||
// A tier sent to a machine, once the send is kept (novox/hq ADR 0406 §5, issue 974).
|
||||
"sent",
|
||||
// A key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3; design 57 F3).
|
||||
"assignment-changed", "declared-config-changed", "predefined-changed"},
|
||||
Serves: ControllerVerbs},
|
||||
|
||||
@@ -1,19 +1,66 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/beside"
|
||||
)
|
||||
|
||||
// boundKeys are every key the files of a module ask of ${bound:forge-account:<key>}, read from the module's own
|
||||
// tree on the shelf, so the test offers what ssh-client asks however often it asks one more (issue 985).
|
||||
func boundKeys(t *testing.T, module, provision string) map[string]bool {
|
||||
t.Helper()
|
||||
ask := regexp.MustCompile(`\$\{bound:` + regexp.QuoteMeta(provision) + `:([A-Za-z0-9_.-]+)\}`)
|
||||
keys := map[string]bool{}
|
||||
root := filepath.Join(beside.Catalogue(t), module)
|
||||
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() || strings.HasSuffix(path, "_test.go") {
|
||||
return err
|
||||
}
|
||||
body, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, m := range ask.FindAllStringSubmatch(string(body), -1) {
|
||||
keys[m[1]] = true
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("reading what %s asks of %s: %v", module, provision, err)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// The ssh-client module, composed as a machine receives it (novox/hq to-be 29, research 027/03): a
|
||||
// region at the START of the operator's ~/.ssh/config that includes ~/.ssh/config.d/* — first,
|
||||
// because ssh takes the first value it finds for each option — and the mesh's Host blocks as the
|
||||
// whole of ~/.ssh/config.d/00-mesh, every other node with its account. ~/.ssh and ~/.ssh/config.d
|
||||
// are created 0700 and owned by the account; the operator's own config below the region is kept.
|
||||
func TestSSHClientOwnsTheOperatorsSSHConfig(t *testing.T) {
|
||||
shelf := shelf(catalogueManifest(t, "ssh-client"))
|
||||
shelf := shelf(catalogueManifest(t, "ssh-client"), catalogueManifest(t, "gitea"))
|
||||
// ssh-client requires forge-account since mesh-catalog #340 (novox/hq issue 955), answered by the forge's
|
||||
// holder on another node; this test is about the operator's config, so the forge is offered from there
|
||||
// (issue 985).
|
||||
// Every key ssh-client's files ask is offered, read from them (boundKeys); a few have a realistic value.
|
||||
serves := map[string]any{}
|
||||
for key := range boundKeys(t, "ssh-client", "forge-account") {
|
||||
serves[key] = "sample-" + key
|
||||
}
|
||||
for key, value := range map[string]any{"ssh_host": "marge.internal", "ssh_port": 222,
|
||||
"known_host": "[marge.internal]:222 ssh-ed25519 AAAA", "ssh_base": "ssh://git@marge.internal:222/",
|
||||
"https_base": "https://git.example.org/"} {
|
||||
serves[key] = value
|
||||
}
|
||||
forge := World{Offered: map[string][]Provider{
|
||||
"forge-account": {{Node: "marge", At: "marge.internal", Module: "gitea", Serves: serves}}}}
|
||||
got, err := Resolve(shelf, []string{"ssh-client"},
|
||||
Node{Name: "homer", At: "homer.internal", Account: "jo", AccountHome: "/home/jo"}, World{})
|
||||
Node{Name: "homer", At: "homer.internal", Account: "jo", AccountHome: "/home/jo"}, forge)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -36,7 +36,7 @@ var ReservedStreamPrefixes = []string{"SEAT_", "LOG_", "KV_", "OBJ_", "DEAD_"}
|
||||
|
||||
// MeshStreamNames are the mesh's own streams by name. Written here as well as in the broker, which this
|
||||
// package cannot import; a test keeps them agreeing.
|
||||
var MeshStreamNames = []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS", "DEAD_LETTER_NOTICES", "DEAD_LETTERS", "PROV_APPLIED"}
|
||||
var MeshStreamNames = []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS", "DEAD_LETTER_NOTICES", "DEAD_LETTERS", "PROV_APPLIED", "CHECKS_READY", "CHECKS_OTHER"}
|
||||
|
||||
// StreamDeclaration is one stream a module owns.
|
||||
type StreamDeclaration struct {
|
||||
|
||||
@@ -160,7 +160,9 @@ var ControllerVerbs = []Verb{
|
||||
"the verdict is said as `checked` with the group's id. With one member and no group, that head is checked " +
|
||||
"again as its pull request is.",
|
||||
Input: schema(map[string]string{"group": "the group's id (none for one head checked again)",
|
||||
"members": "the members, as JSON, as delivery-order takes them"}, []string{"members"})},
|
||||
"members": "the members, as JSON, as delivery-order takes them",
|
||||
"ready": "\"true\" when the pull request is marked ready: its check is queued before every other check (novox/hq ADR 0406 §3)"},
|
||||
[]string{"members"})},
|
||||
{Name: "deliver", Description: "The delivery's word that a walk waiting for it may start (novox/hq ADR 0239): its " +
|
||||
"first tier is asked at the controller's next pass. Refused for a walk that waits for nobody.",
|
||||
Input: schema(map[string]string{"plan": "the walk's id", "why": "what lets it go: its turn, a person's release"},
|
||||
|
||||
@@ -112,6 +112,10 @@ type Machine struct {
|
||||
// NodeEngine and NodeTools are the builds it runs.
|
||||
NodeEngine string `json:"node-engine,omitempty"`
|
||||
NodeTools string `json:"node-tools,omitempty"`
|
||||
// Contract is the contract of the node-engine's newest health statement (0 when it never stated one): what
|
||||
// says whether the machine takes handed files (novox/hq issue 969). A check that composed it without would
|
||||
// judge a moved module the old way where the mesh hands it.
|
||||
Contract int `json:"contract,omitempty"`
|
||||
// Capabilities are what it reported it can do; the detail kept only where it is a version.
|
||||
Capabilities []Capability `json:"capabilities,omitempty"`
|
||||
// Site, Hub, Public: where it is on the private network — its site (a pseudonym), whether it is the
|
||||
|
||||
@@ -428,7 +428,8 @@ func (i *Inventory) ReadRepositories(ctx context.Context) (map[string][]ReadRepo
|
||||
}
|
||||
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select distinct on (module) module, built_contexts, build_sources, coalesce(asked, at)
|
||||
`select distinct on (module) module, built_contexts, build_sources, coalesce(asked, at),
|
||||
(select source from module where module.name = build.module)
|
||||
from build
|
||||
where module is not null and module <> '' and failed = ''
|
||||
order by module, `+newestRequestFirst)
|
||||
@@ -442,7 +443,8 @@ func (i *Inventory) ReadRepositories(ctx context.Context) (map[string][]ReadRepo
|
||||
var module string
|
||||
var raw, rawSources []byte
|
||||
var built time.Time
|
||||
if err := rows.Scan(&module, &raw, &rawSources, &built); err != nil {
|
||||
var own *string
|
||||
if err := rows.Scan(&module, &raw, &rawSources, &built, &own); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var of []ReadRepository
|
||||
@@ -462,6 +464,9 @@ func (i *Inventory) ReadRepositories(ctx context.Context) (map[string][]ReadRepo
|
||||
sources, whole = nil, "a newer build of it failed"
|
||||
}
|
||||
of = WithBuildSources(of, sources)
|
||||
if own != nil {
|
||||
of = OwnRepositoryAsOwn(of, *own)
|
||||
}
|
||||
if whole != "" {
|
||||
of = append(of, ReadRepository{Own: true, Whole: whole})
|
||||
}
|
||||
@@ -498,6 +503,57 @@ func WithBuildSources(read []ReadRepository, sources []BuildSource) []ReadReposi
|
||||
return out
|
||||
}
|
||||
|
||||
// OwnRepositoryAsOwn reads a context that names the module's own registered repository, with the paths its
|
||||
// build read there, as the module's own build source (novox/hq ADR 0420 §4): a change to the program it
|
||||
// compiles moves the module, as one in its own directory does. A context of that repository that said
|
||||
// no paths stays a context read, whole.
|
||||
func OwnRepositoryAsOwn(read []ReadRepository, repository string) []ReadRepository {
|
||||
if repository == "" {
|
||||
return read
|
||||
}
|
||||
out := make([]ReadRepository, 0, len(read))
|
||||
var moved []ReadRepository
|
||||
for _, r := range read {
|
||||
if !r.Own && len(r.Paths) > 0 && r.Repository != "" && namesRepository(repository, r.Repository) {
|
||||
moved = append(moved, r)
|
||||
continue
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
if len(moved) == 0 {
|
||||
return read
|
||||
}
|
||||
// One entry of its own: ownSource reads the first that has paths.
|
||||
at := -1
|
||||
for k := range out {
|
||||
if out[k].Own && len(out[k].Paths) > 0 {
|
||||
at = k
|
||||
break
|
||||
}
|
||||
}
|
||||
if at < 0 {
|
||||
out = append(out, ReadRepository{Own: true})
|
||||
at = len(out) - 1
|
||||
}
|
||||
for _, r := range moved {
|
||||
out[at].Paths = append(append([]string(nil), out[at].Paths...), r.Paths...)
|
||||
if out[at].Ref == "" {
|
||||
out[at].Ref = r.Ref
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// namesRepository is whether a context's repository, as owner/name, is the registered one, which may be
|
||||
// spelled as the URL it was cloned from.
|
||||
func namesRepository(registered, named string) bool {
|
||||
trim := func(s string) string {
|
||||
return strings.TrimSuffix(strings.TrimRight(strings.ToLower(strings.TrimSpace(s)), "/"), ".git")
|
||||
}
|
||||
reg, name := trim(registered), trim(named)
|
||||
return reg == name || strings.HasSuffix(reg, "/"+name)
|
||||
}
|
||||
|
||||
// manifestOrNil keeps the difference between "declared nothing" and "predates this being kept".
|
||||
//
|
||||
// A build recorded before the mesh kept manifests has no manifest, and that is not the same as one
|
||||
|
||||
@@ -119,6 +119,8 @@ type PlanBatch struct {
|
||||
ClosesAt time.Time `json:"closes_at"`
|
||||
AtMost time.Time `json:"at_most"`
|
||||
Behind string `json:"behind,omitempty"`
|
||||
// BehindWhy says why the batch waits behind that walk: what the two share (issue 973).
|
||||
BehindWhy string `json:"behind_why,omitempty"`
|
||||
// Own says the batch holds merges on the controller's own path, whose walk waits for nobody's word: such
|
||||
// a merge never shares a batch with one that waits for mesh-delivery's (ADR 0276, decided during the
|
||||
// build, 2026-10-10).
|
||||
|
||||
@@ -81,18 +81,10 @@ type ProvisionedCollection struct {
|
||||
}
|
||||
|
||||
// ProvisionedCredential is one provision the module requires, under one local name, and its provider (issue 760);
|
||||
// with a name, a credential issued for this holder alone (issue 757).
|
||||
type ProvisionedCredential struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Provision string `json:"provision"`
|
||||
Local string `json:"local,omitempty"`
|
||||
Provider string `json:"provider"`
|
||||
ProviderModule string `json:"provider-module"`
|
||||
// CredentialForm is the form the provider's offer declares for its credential (issue 768): shared, pair,
|
||||
// issued or given; empty when the provider's manifest was not in hand.
|
||||
CredentialForm string `json:"credential-form,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
}
|
||||
// with a name, a credential issued for this holder alone (issue 757). It is the composition's own type, not a copy:
|
||||
// the plan is kept and read back as the one it was sent as, so no field of it (contributes, at, from, as, names) can
|
||||
// be lost between the two (issue 923).
|
||||
type ProvisionedCredential = catalogue.PlanProvision
|
||||
|
||||
// ProvisionedSecret is one of the module's own secrets, by its name in the manifest. Path, when set, is the file
|
||||
// holding the secret alone (the manifest's `own-secrets`), which the client fills with its value and nothing else.
|
||||
|
||||
@@ -352,3 +352,42 @@ func TestAPlanMayHoldSeveralNamedItemsOfOneCollectionAndRefusesABadOne(t *testin
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **A kept plan's provisions keep every field the composition sets** (issue 923): the names, the place, the machine
|
||||
// and the identity the provider's helper writes into the consumer's entry, and what the manifest contributes, are
|
||||
// read back as they were sent, so the assignments mirror carries them to the provider.
|
||||
func TestAKeptPlansProvisionsKeepContributesAtFromAsAndNames(t *testing.T) {
|
||||
sent := catalogue.PlanProvision{Provision: "route", Provider: "novox", ProviderModule: "route-proxy",
|
||||
Contributes: map[string]any{"database": "gitea"}, At: "10.0.0.1", From: "novox", As: "gitea@novox",
|
||||
Names: map[string]string{"name-web": "git.example.org", "internal-name": "gitea.mesh"}}
|
||||
raw, err := json.Marshal(map[string]any{"gitea": map[string]any{
|
||||
"provisions": []catalogue.PlanProvision{sent}, "required-by": "forge"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var kept map[string]keptPlan
|
||||
if err := json.Unmarshal(raw, &kept); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := kept["gitea"].Provisions
|
||||
if len(got) != 1 || !reflect.DeepEqual(got[0], sent) {
|
||||
t.Fatalf("the kept plan's provisions are %+v, want %+v", got, sent)
|
||||
}
|
||||
}
|
||||
|
||||
// A kept plan's provision keeps the tool the vault asks for a software-issued credential (issue 928).
|
||||
func TestAKeptPlansProvisionKeepsItsCredentialTool(t *testing.T) {
|
||||
sent := catalogue.PlanProvision{Provision: "gitea-read", Provider: "novox", ProviderModule: "gitea",
|
||||
CredentialForm: catalogue.CredentialFormIssued, CredentialTool: "gitea_agent_token"}
|
||||
raw, err := json.Marshal(map[string]any{"claude-code": map[string]any{"provisions": []catalogue.PlanProvision{sent}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var kept map[string]keptPlan
|
||||
if err := json.Unmarshal(raw, &kept); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := kept["claude-code"].Provisions; len(got) != 1 || got[0].CredentialTool != "gitea_agent_token" {
|
||||
t.Fatalf("the kept plan's provisions are %+v, want the tool kept", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -143,9 +143,9 @@ func MeshNamed(stream, name string) bool {
|
||||
return false // the mesh defines no durable consumer on a bucket's stream, or the memberships'
|
||||
case stream == "NODES":
|
||||
return true
|
||||
case strings.HasPrefix(stream, "SEAT_"):
|
||||
case strings.HasPrefix(stream, "SEAT_") || broker.IsCheckStream(stream):
|
||||
return strings.HasSuffix(name, "_worker")
|
||||
case name == broker.ControllerName:
|
||||
case name == broker.ControllerName || name == broker.MergesConsumerName:
|
||||
return true
|
||||
case stream == broker.EventsStream:
|
||||
return strings.Contains(name, "_")
|
||||
@@ -156,10 +156,12 @@ func MeshNamed(stream, name string) bool {
|
||||
// ConsumerInWords is a durable consumer as the mesh says it: whose, and for what.
|
||||
func ConsumerInWords(stream, name string) string {
|
||||
switch {
|
||||
case name == broker.ControllerName:
|
||||
case name == broker.ControllerName || name == broker.MergesConsumerName:
|
||||
return "the controller's consumer on " + stream
|
||||
case stream == "NODES":
|
||||
return "how " + name + " hears what it should be (its declaration consumer)"
|
||||
case broker.IsCheckStream(stream):
|
||||
return "the worker every holder of " + broker.TheBuildSeat + " takes its checks from (" + stream + ")"
|
||||
case strings.HasPrefix(stream, "SEAT_") && strings.HasSuffix(name, "_worker"):
|
||||
seat := strings.ToLower(strings.ReplaceAll(strings.TrimPrefix(stream, "SEAT_"), "_", "-"))
|
||||
return "the worker every holder of " + seat + " takes its asks from"
|
||||
|
||||
@@ -144,6 +144,12 @@ type CheckRequest struct {
|
||||
Repo string `json:"repo"`
|
||||
Number int `json:"number,omitempty"`
|
||||
Base string `json:"base,omitempty"`
|
||||
// Trunk is the base branch's head the check is asked against, as the forge announced it with the head (novox/hq ADR
|
||||
// 0406 §4): with the head, what the pull request's key says the current check is for, and what a waiting request
|
||||
// is dropped by and a running check stopped by when the key moves.
|
||||
Trunk string `json:"trunk,omitempty"`
|
||||
// Ready says the pull request is marked ready (ADR 0406 §3): its check is queued before every other check.
|
||||
Ready bool `json:"ready,omitempty"`
|
||||
// Paths are the files the pull request changes, for the width of its rebuild.
|
||||
Paths []string `json:"paths,omitempty"`
|
||||
// Beside are the repositories the check reads next to this one, each cloned at the ref given — the
|
||||
|
||||
+208
-74
@@ -76,11 +76,9 @@ func (b *natsBuilds) Ask(ctx context.Context, request BuildRequest) error {
|
||||
}
|
||||
publish, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
if _, err := b.js.Context().Publish(BuildWorkOf(b.role()), body, nats.Context(publish)); err != nil {
|
||||
if _, err := b.js.Context().Publish(WorkSubjectOf(b.role(), request), body, nats.Context(publish)); err != nil {
|
||||
return fmt.Errorf("cannot submit a build: %w", err)
|
||||
}
|
||||
// The acting controller puts it in order (novox/hq issue 558); this only says an ask joined.
|
||||
signalAsked()
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -112,11 +110,9 @@ func (b *natsBuilds) Submit(ctx context.Context, request BuildRequest,
|
||||
// be assumed, because nothing else will ever say so.
|
||||
publish, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
if _, err := b.js.Context().Publish(BuildWorkOf(b.role()), body, nats.Context(publish)); err != nil {
|
||||
if _, err := b.js.Context().Publish(WorkSubjectOf(b.role(), request), body, nats.Context(publish)); err != nil {
|
||||
return BuildResult{}, fmt.Errorf("%w: cannot submit a build: %w", ErrNotHandedOver, err)
|
||||
}
|
||||
// The acting controller puts it in order (novox/hq issue 558); this only says an ask joined.
|
||||
signalAsked()
|
||||
|
||||
waiting, cancelWait := context.WithTimeout(ctx, wait)
|
||||
defer cancelWait()
|
||||
@@ -162,10 +158,29 @@ type natsMachine struct {
|
||||
js *broker.JetStream
|
||||
on string
|
||||
seat string
|
||||
sub *nats.Subscription
|
||||
// subs are the queues this machine takes from, in the order it takes from them: the seat's own, where a delivery
|
||||
// build waits, then the checks of pull requests marked ready, then every other check (novox/hq ADR 0406 §3).
|
||||
subs []*takeQueue
|
||||
opts MachineOptions
|
||||
}
|
||||
|
||||
// takeQueue is one queue a machine pulls from.
|
||||
type takeQueue struct {
|
||||
sub *nats.Subscription
|
||||
// kind is what its asks are: a build, a ready check, or another check.
|
||||
kind string
|
||||
// worker is the consumer bound; bound is false until a bind succeeded, for a queue the mesh may not have made yet.
|
||||
worker broker.Consumer
|
||||
bound bool
|
||||
}
|
||||
|
||||
// The kinds of queue a machine takes from, in the order it takes from them.
|
||||
const (
|
||||
queueBuilds = "builds"
|
||||
queueReady = "ready"
|
||||
queueOther = "other"
|
||||
)
|
||||
|
||||
// MachineOptions is what a holder tells the taking loop about itself (novox/hq ADR 0219).
|
||||
type MachineOptions struct {
|
||||
// Paused is asked before every fetch: while it says so, nothing new is taken, and a build
|
||||
@@ -207,47 +222,56 @@ const (
|
||||
)
|
||||
|
||||
func (m *natsMachine) Close() {
|
||||
if m.sub != nil {
|
||||
_ = m.sub.Unsubscribe()
|
||||
for _, q := range m.subs {
|
||||
if q.sub != nil {
|
||||
_ = q.sub.Unsubscribe()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Take binds to the role's worker and pulls one request at a time, handing each over.
|
||||
// Take binds to the role's workers and pulls one request at a time, handing each over.
|
||||
//
|
||||
// **Bound, never created.** The work queue and the worker on it are the controller's to define
|
||||
// **Bound, never created.** The work queues and the workers on them are the controller's to define
|
||||
// (design 25 §3), and a build machine reaches no part of the JetStream API — so a missing one is said
|
||||
// as the mesh's to answer rather than quietly created with whatever this client defaults to.
|
||||
//
|
||||
// **Pulled, one at a time, by whichever holder is free** (novox/hq ADR 0190). Every machine holding
|
||||
// the role binds this same worker; a machine asks for the next request only when it has finished
|
||||
// the role binds these same workers; a machine asks for the next request only when it has finished
|
||||
// the last, so a slow machine never holds an ask an idle one could take, and a machine that took
|
||||
// five at once would run five container builds against one runtime and finish all of them slower
|
||||
// than the first.
|
||||
//
|
||||
// **In this order** (novox/hq ADR 0406 §3): a delivery build, then a check of a pull request marked ready, then any
|
||||
// other check, each queue tried without waiting; when all three are empty the first is waited on for a second at most
|
||||
// and the three are tried again, so a delivery build is never much more than a second behind and nothing spins.
|
||||
func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build)) error {
|
||||
worker, found := broker.HolderConsumerFor(m.on, "build-agent",
|
||||
broker.DeclaredSeat{Name: m.seat, Accepts: []string{"build"}})
|
||||
if !found {
|
||||
return fmt.Errorf("%s accepts no work, so there is nothing for this machine to take", m.seat)
|
||||
}
|
||||
|
||||
// **The consumer's own filter, not the one subject this machine cares about.** The client checks
|
||||
// what is asked for against the consumer's filter and refuses anything that is not the same —
|
||||
// "subject does not match consumer" — so subscribing `…accept.build` against a consumer filtered
|
||||
// on `…accept.>` is rejected even though it is narrower. Learned twice now, on two different
|
||||
// consumers, which is why it is written down here.
|
||||
filter := worker.Filters[0]
|
||||
sub, err := m.js.Context().PullSubscribe(filter, worker.Name,
|
||||
nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
||||
if err != nil {
|
||||
m.subs = []*takeQueue{{kind: queueBuilds, worker: worker}}
|
||||
if m.seat == broker.TheBuildSeat {
|
||||
for i, w := range broker.CheckWorkers() {
|
||||
kind := queueReady
|
||||
if i == 1 {
|
||||
kind = queueOther
|
||||
}
|
||||
m.subs = append(m.subs, &takeQueue{kind: kind, worker: w})
|
||||
}
|
||||
}
|
||||
// The seat's own queue is bound at once and its absence is this machine's error, as it always was. The check queues
|
||||
// are bound when they are there: a controller that has not made them yet is no reason to take no build.
|
||||
if err := m.bind(m.subs[0]); err != nil {
|
||||
return fmt.Errorf(
|
||||
"this machine cannot take work from %s: %w. The mesh creates that queue and this "+
|
||||
"machine's worker on it, and a build machine may not create one itself — so this is "+
|
||||
"the mesh's to answer, not this machine's", m.seat, err)
|
||||
}
|
||||
m.sub = sub
|
||||
if m.opts.Taking != nil {
|
||||
m.opts.Taking()
|
||||
}
|
||||
var bindAgain time.Time
|
||||
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
@@ -264,29 +288,31 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
|
||||
}
|
||||
continue
|
||||
}
|
||||
// One, and wait a while for it; an empty queue is a timeout, which is the normal state of a
|
||||
// machine with nothing to build, and is asked again.
|
||||
// Asked for a few seconds at a time when the holder can be paused, so a pause reaches a pull
|
||||
// already waiting within that, rather than when the client's own wait runs out.
|
||||
asking, endAsking := ctx, func() {}
|
||||
if m.opts.Paused != nil {
|
||||
asking, endAsking = context.WithTimeout(ctx, pausedFetch)
|
||||
if time.Now().After(bindAgain) {
|
||||
for _, q := range m.subs[1:] {
|
||||
if !q.bound {
|
||||
if err := m.bind(q); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "the %s check queue is not there to take from yet: %v\n", q.kind, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
bindAgain = time.Now().Add(30 * time.Second)
|
||||
}
|
||||
fetched, err := sub.Fetch(1, nats.Context(asking))
|
||||
endAsking()
|
||||
msg, q, err := m.next(ctx)
|
||||
switch {
|
||||
case ctx.Err() != nil:
|
||||
// Ours ended: the machine is being stopped.
|
||||
return nil
|
||||
case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded), errors.Is(err, nats.ErrTimeout):
|
||||
case msg != nil:
|
||||
case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded), errors.Is(err, nats.ErrTimeout), err == nil:
|
||||
// **An empty queue, not the end.** A fetch on a context without a deadline waits the
|
||||
// client's own while and then says the deadline passed — the client's, not ours. Read
|
||||
// as "stop", every idle build machine exited clean every half minute and was started
|
||||
// again by its supervisor, which looked like a crash loop with nothing in the log to
|
||||
// say why (2026-10-03, the first build agents). Asked again.
|
||||
continue
|
||||
case err != nil:
|
||||
if sub.IsValid() {
|
||||
default:
|
||||
if m.subs[0].sub.IsValid() {
|
||||
// A transient fault in asking — a reconnect, a slow server — is asked past rather
|
||||
// than ending the machine; one that outlasts the ack wait redelivers nothing lost.
|
||||
time.Sleep(time.Second)
|
||||
@@ -294,49 +320,151 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
|
||||
}
|
||||
return fmt.Errorf("the bus stopped delivering build work: %w", err)
|
||||
}
|
||||
for _, msg := range fetched {
|
||||
// **Paused while the pull was answered** (ADR 0219): "takes no new build" holds even for
|
||||
// the one that arrived in that moment. Handed back at once for another holder — counted as
|
||||
// a delivery, which a pause landing exactly then costs and nothing else does.
|
||||
if m.opts.Paused != nil && m.opts.Paused() {
|
||||
_ = msg.Nak()
|
||||
// **Paused while the pull was answered** (ADR 0219): "takes no new build" holds even for
|
||||
// the one that arrived in that moment. Handed back at once for another holder — counted as
|
||||
// a delivery, which a pause landing exactly then costs and nothing else does.
|
||||
if m.opts.Paused != nil && m.opts.Paused() {
|
||||
_ = msg.Nak()
|
||||
continue
|
||||
}
|
||||
var request BuildRequest
|
||||
if err := json.Unmarshal(msg.Data, &request); err != nil {
|
||||
// Unreadable: terminated rather than retried, because the next attempt reads the same
|
||||
// bytes. Nobody waiting hears an answer, which is right — there was no request.
|
||||
_ = msg.Term()
|
||||
continue
|
||||
}
|
||||
// **Cancelled in the moment it was fetched** (novox/hq ADR 0219): the controller wrote the
|
||||
// id into the seat's cancelled set before deleting the ask, so one taken in between is
|
||||
// ended here, terminated rather than redelivered, and its outcome said as failed — never
|
||||
// built. A set that cannot be read is said and the ask built: a cancel is a person's
|
||||
// exception, and a holder that refused every build while its grant was missing would
|
||||
// stop the mesh building for it.
|
||||
build := &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat, delivery: deliveryOf(msg)}
|
||||
if cancelled, err := IsCancelled(m.js.Conn(), m.seat, request.ID); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read whether %s was cancelled, so it is built: %v\n", request.ID, err)
|
||||
} else if cancelled {
|
||||
endCancelled(ctx, build)
|
||||
continue
|
||||
}
|
||||
// **A check left behind is dropped when it is taken** (novox/hq ADR 0406 §3): a request on the third queue for a
|
||||
// head and a trunk commit its pull request's key already says were judged is acknowledged and not run.
|
||||
if q.kind == queueOther {
|
||||
if why := m.judgedAlready(ctx, request); why != "" {
|
||||
fmt.Fprintf(os.Stderr, "%s was left behind in the check queue and is dropped: %s\n", request.ID, why)
|
||||
_ = msg.Ack()
|
||||
continue
|
||||
}
|
||||
var request BuildRequest
|
||||
if err := json.Unmarshal(msg.Data, &request); err != nil {
|
||||
// Unreadable: terminated rather than retried, because the next attempt reads the same
|
||||
// bytes. Nobody waiting hears an answer, which is right — there was no request.
|
||||
_ = msg.Term()
|
||||
continue
|
||||
}
|
||||
// **Cancelled in the moment it was fetched** (novox/hq ADR 0219): the controller wrote the
|
||||
// id into the seat's cancelled set before deleting the ask, so one taken in between is
|
||||
// ended here, terminated rather than redelivered, and its outcome said as failed — never
|
||||
// built. A set that cannot be read is said and the ask built: a cancel is a person's
|
||||
// exception, and a holder that refused every build while its grant was missing would
|
||||
// stop the mesh building for it.
|
||||
build := &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat, delivery: deliveryOf(msg)}
|
||||
if cancelled, err := IsCancelled(m.js.Conn(), m.seat, request.ID); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read whether %s was cancelled, so it is built: %v\n", request.ID, err)
|
||||
} else if cancelled {
|
||||
endCancelled(ctx, build)
|
||||
continue
|
||||
}
|
||||
// **Drained in the moment it was fetched** (novox/hq ADR 0335): handed back at once, as a pause
|
||||
// landing then is.
|
||||
if m.opts.Claim != nil && !m.opts.Claim() {
|
||||
_ = msg.Nak()
|
||||
continue
|
||||
}
|
||||
// A build outlives the acknowledgement window many times over; said while it runs,
|
||||
// as the controller says it for its own long handlers, so the server neither hands
|
||||
// the ask to a second machine nor counts the wait against its deliveries.
|
||||
working := make(chan struct{})
|
||||
go stillWorking(msg, working, heartbeatWhileWorking)
|
||||
do(ctx, build)
|
||||
close(working)
|
||||
}
|
||||
// **Drained in the moment it was fetched** (novox/hq ADR 0335): handed back at once, as a pause
|
||||
// landing then is.
|
||||
if m.opts.Claim != nil && !m.opts.Claim() {
|
||||
_ = msg.Nak()
|
||||
continue
|
||||
}
|
||||
// A build outlives the acknowledgement window many times over; said while it runs,
|
||||
// as the controller says it for its own long handlers, so the server neither hands
|
||||
// the ask to a second machine nor counts the wait against its deliveries.
|
||||
working := make(chan struct{})
|
||||
go stillWorking(msg, working, heartbeatWhileWorking)
|
||||
// And a check says it is running, to the other builders (ADR 0406 §3): its key put now and at each heartbeat,
|
||||
// deleted when it ends, whatever the outcome.
|
||||
stopRunning := keepRunning(ctx, m.js.Conn(), request, m.on, heartbeatWhileWorking)
|
||||
do(ctx, build)
|
||||
stopRunning()
|
||||
close(working)
|
||||
}
|
||||
}
|
||||
|
||||
// bind binds a queue's worker, which exists and is not created here.
|
||||
//
|
||||
// **The consumer's own filter, not the one subject this machine cares about.** The client checks what is asked for
|
||||
// against the consumer's filter and refuses anything that is not the same — "subject does not match consumer" — so
|
||||
// subscribing `…accept.build` against a consumer filtered on `…accept.>` is rejected even though it is narrower. Learned
|
||||
// twice now, on two different consumers, which is why it is written down here.
|
||||
func (m *natsMachine) bind(q *takeQueue) error {
|
||||
sub, err := m.js.Context().PullSubscribe(q.worker.Filters[0], q.worker.Name,
|
||||
nats.Bind(q.worker.Stream, q.worker.Name), nats.ManualAck())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
q.sub, q.bound = sub, true
|
||||
return nil
|
||||
}
|
||||
|
||||
// next takes the next request in the order of the queues: each tried for a moment, and when all are empty the first waited
|
||||
// on for a second. Nothing, with no error, when there was nothing.
|
||||
func (m *natsMachine) next(ctx context.Context) (*nats.Msg, *takeQueue, error) {
|
||||
var last error
|
||||
for _, q := range m.subs {
|
||||
if !q.bound {
|
||||
continue
|
||||
}
|
||||
trying, end := context.WithTimeout(ctx, checkTakeWait)
|
||||
fetched, err := q.sub.Fetch(1, nats.Context(trying))
|
||||
end()
|
||||
if len(fetched) > 0 {
|
||||
return fetched[0], q, nil
|
||||
}
|
||||
if err != nil && !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, nats.ErrTimeout) &&
|
||||
!errors.Is(err, context.Canceled) {
|
||||
last = err
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return nil, nil, nil
|
||||
}
|
||||
}
|
||||
if last != nil {
|
||||
return nil, nil, last
|
||||
}
|
||||
waiting, end := context.WithTimeout(ctx, takeIdleWait)
|
||||
defer end()
|
||||
fetched, err := m.subs[0].sub.Fetch(1, nats.Context(waiting))
|
||||
if len(fetched) > 0 {
|
||||
return fetched[0], m.subs[0], nil
|
||||
}
|
||||
if err != nil && !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, nats.ErrTimeout) &&
|
||||
!errors.Is(err, context.Canceled) {
|
||||
return nil, nil, err
|
||||
}
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
// judgedAlready says why a check just taken from the other checks' queue need not run, or nothing: its pull request's key
|
||||
// holds its head and trunk commit and a verdict. A key that cannot be read, or none, runs the check (the key is
|
||||
// mesh-delivery's, and a check wrongly dropped is worse than one run twice).
|
||||
func (m *natsMachine) judgedAlready(ctx context.Context, r BuildRequest) string {
|
||||
c := r.Check
|
||||
if c == nil || c.Try || (c.Group == "" && c.Number <= 0) {
|
||||
return ""
|
||||
}
|
||||
if c.Group == "" {
|
||||
key, found, err := ReadPullKey(ctx, m.js.Conn(), c.Owner, c.Repo, c.Number)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read the key of %s/%s#%d, so its check runs: %v\n", c.Owner, c.Repo, c.Number, err)
|
||||
} else if found && key.Judged(r.Ref, c.Trunk) {
|
||||
return fmt.Sprintf("%s/%s#%d is judged already at %.12s on %.12s (%s)", c.Owner, c.Repo, c.Number, r.Ref, c.Trunk,
|
||||
key.Verdict)
|
||||
}
|
||||
}
|
||||
return m.beingChecked(ctx, r)
|
||||
}
|
||||
|
||||
// beingChecked says why a request need not run, or nothing: another builder's check of the same head and trunk commit is
|
||||
// running, and says so under its pull request's key in the bucket of the checks running (ADR 0406 §3). A key a builder
|
||||
// stopped renewing is gone within the bucket's age, and the request is taken then. Never the ask's own id: a redelivered ask
|
||||
// is its own, and a key an older builder wrote that names no head or trunk is no match.
|
||||
func (m *natsMachine) beingChecked(ctx context.Context, r BuildRequest) string {
|
||||
running, ok, err := ReadRunning(ctx, m.js.Conn(), r)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read whether %s is being checked, so its check runs: %v\n", r.ID, err)
|
||||
return ""
|
||||
}
|
||||
if ok && running.Build != r.ID && running.Same(r) {
|
||||
return fmt.Sprintf("%s is being checked at %.12s on %.12s by %s as %s", r.ID, r.Ref, r.Check.Trunk, running.Builder,
|
||||
running.Build)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type natsBuild struct {
|
||||
@@ -398,6 +526,12 @@ func (b *natsBuild) Announce(ctx context.Context, result BuildResult) error {
|
||||
|
||||
func (b *natsBuild) Done() error { return b.msg.Ack() }
|
||||
|
||||
// WatchPull has moved called once the pull request's key moves on from what this check was asked for; the stop ends the
|
||||
// watch (see WatchPull). A request that is no pull request's check watches nothing.
|
||||
func (b *natsBuild) WatchPull(ctx context.Context, moved func(PullKey)) (func(), error) {
|
||||
return WatchPull(ctx, b.js.Conn(), b.request, moved)
|
||||
}
|
||||
|
||||
// Began publishes that this machine has taken the build, into the stream like the outcome, so a
|
||||
// reader that asks afterwards sees when it started as well as how it ended.
|
||||
func (b *natsBuild) Began(ctx context.Context) error {
|
||||
|
||||
@@ -44,6 +44,12 @@ func aBusWithTheBuildRole(t *testing.T) *broker.JetStream {
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// And the two check queues' workers, which the controller makes beside the streams (novox/hq ADR 0406 §3).
|
||||
for _, w := range broker.CheckWorkers() {
|
||||
if err := js.EnsureConsumer(w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
clean := func() {
|
||||
_ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT")
|
||||
for _, s := range broker.MeshStreams() {
|
||||
|
||||
@@ -179,6 +179,19 @@ func (b OverNATS) PublishMembership(ctx context.Context, node, module string, bo
|
||||
return nil
|
||||
}
|
||||
|
||||
// PublishMembershipAfter issues a membership only if the one read at sequence seq is still the newest of its
|
||||
// subject: a push that issued another since is not overwritten, and the refusal comes back (novox/hq issue 930).
|
||||
func (b OverNATS) PublishMembershipAfter(ctx context.Context, node, module string, body []byte, seq uint64) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, MembershipWait)
|
||||
defer cancel()
|
||||
_, err := b.JS.Publish(broker.MembershipSubject(node, module), body, nats.Context(ctx),
|
||||
nats.ExpectLastSequencePerSubject(seq))
|
||||
if err != nil {
|
||||
return fmt.Errorf("issuing %s on %s its membership after sequence %d: %w", module, node, seq, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b OverNATS) PublishDeclaration(ctx context.Context, node string, body []byte) error {
|
||||
_, err := b.JS.Publish(DeclareSubject(node), body, nats.Context(ctx))
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,293 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// A pull request's check is queued by the pull request (novox/hq ADR 0406 §3, §4, issue 974).
|
||||
|
||||
// SupersededByKey is the outcome of a check that stopped because its pull request's key moved on: its head or its trunk
|
||||
// commit is no longer the one it was asked for. Said as the check's failure, as a kill is; nothing is asked of the forge.
|
||||
const SupersededByKey = "superseded: the pull request's key moved on"
|
||||
|
||||
// unsafeToken is what may not stand in a subject token or a key.
|
||||
var unsafeToken = strings.NewReplacer(".", "_", "*", "_", ">", "_", " ", "_", "/", "_", "\t", "_", "\n", "_")
|
||||
|
||||
// tokenOf is a name as one subject token: any dot, wildcard or space replaced by an underscore.
|
||||
func tokenOf(name string) string {
|
||||
if name == "" {
|
||||
return "_"
|
||||
}
|
||||
return unsafeToken.Replace(name)
|
||||
}
|
||||
|
||||
// PullKeyOf is a pull request's key in mesh-delivery's bucket, and the last tokens of the subject its check is asked on:
|
||||
// `<owner>.<repo>.<number>`, a dot in a name made an underscore.
|
||||
func PullKeyOf(owner, repo string, number int) string {
|
||||
return tokenOf(owner) + "." + tokenOf(repo) + "." + strconv.Itoa(number)
|
||||
}
|
||||
|
||||
// CheckSubjectOf is the subject a check is asked on, and whether it has one of its own: a check of a pull request on
|
||||
// the ready queue or the other, by the pull request; a delivery group's composed check by `group.<name>`; any other
|
||||
// check (a branch tried before its pull request) by its own id, so it replaces nothing.
|
||||
func CheckSubjectOf(seat string, r BuildRequest) (string, bool) {
|
||||
c := r.Check
|
||||
if seat != broker.TheBuildSeat || c == nil {
|
||||
return "", false
|
||||
}
|
||||
prefix := broker.CheckSubjectPrefix(seat, c.Ready)
|
||||
switch {
|
||||
case c.Group != "":
|
||||
return prefix + "group." + tokenOf(c.Group), true
|
||||
case c.Try || c.Number <= 0:
|
||||
return prefix + "try." + tokenOf(r.ID), true
|
||||
}
|
||||
return prefix + PullKeyOf(c.Owner, c.Repo, c.Number), true
|
||||
}
|
||||
|
||||
// WorkSubjectOf is where an ask is put: a check on the build seat on its own queue, anything else on the seat's.
|
||||
func WorkSubjectOf(seat string, r BuildRequest) string {
|
||||
if subject, ok := CheckSubjectOf(seat, r); ok {
|
||||
return subject
|
||||
}
|
||||
return BuildWorkOf(seat)
|
||||
}
|
||||
|
||||
// PullKey is what mesh-delivery keeps of a pull request (ADR 0406 §4): its head, the trunk commit its current check is
|
||||
// asked against, and its verdict once judged.
|
||||
type PullKey struct {
|
||||
Head string `json:"head"`
|
||||
Trunk string `json:"trunk"`
|
||||
Verdict string `json:"verdict,omitempty"`
|
||||
}
|
||||
|
||||
// sameCommit says two spellings name one commit; either may be short, but not under twelve characters.
|
||||
func sameCommit(a, b string) bool {
|
||||
return a == b || SameHead(a, b)
|
||||
}
|
||||
|
||||
// Moved says a key no longer holds what a check was asked for: its head or its trunk commit differs.
|
||||
func (k PullKey) Moved(head, trunk string) bool {
|
||||
return !sameCommit(k.Head, head) || (trunk != "" && k.Trunk != "" && !sameCommit(k.Trunk, trunk))
|
||||
}
|
||||
|
||||
// Judged says a key holds the verdict of the check of this head against this trunk commit.
|
||||
func (k PullKey) Judged(head, trunk string) bool {
|
||||
// A verdict of a check that ran: a pass, a warning or a fail. An error is a check that could not run, and a request
|
||||
// after it must run (issue 974, review 2).
|
||||
judgedBy := k.Verdict == "pass" || k.Verdict == "warning" || k.Verdict == "fail"
|
||||
return judgedBy && sameCommit(k.Head, head) && sameCommit(k.Trunk, trunk) && trunk != ""
|
||||
}
|
||||
|
||||
// ReadPullKey reads a pull request's key from mesh-delivery's bucket: false when it has none.
|
||||
func ReadPullKey(ctx context.Context, conn *nats.Conn, owner, repo string, number int) (PullKey, bool, error) {
|
||||
api, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return PullKey{}, false, err
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, broker.PullsBucket)
|
||||
if err != nil {
|
||||
return PullKey{}, false, fmt.Errorf("the pull requests' keys: %w", err)
|
||||
}
|
||||
e, err := kv.Get(ctx, PullKeyOf(owner, repo, number))
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return PullKey{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return PullKey{}, false, err
|
||||
}
|
||||
var key PullKey
|
||||
if err := json.Unmarshal(e.Value(), &key); err != nil {
|
||||
return PullKey{}, false, fmt.Errorf("the key of %s/%s#%d is not a pull request's: %w", owner, repo, number, err)
|
||||
}
|
||||
return key, true, nil
|
||||
}
|
||||
|
||||
// WatchPull calls moved once when the pull request's key moves on from what this check was asked for — its head or its
|
||||
// trunk commit differs — and not for a key that only gains its verdict. It hears updates, not the key as it stands: the
|
||||
// key and the request are written for one announcement by two writers, so a key found older than the request is not
|
||||
// a move. The check is then stopped by whoever asked for it, without asking the forge (novox/hq issue 726). The stop
|
||||
// ends the watch.
|
||||
func WatchPull(ctx context.Context, conn *nats.Conn, r BuildRequest, moved func(PullKey)) (stop func(), err error) {
|
||||
c := r.Check
|
||||
if c == nil || c.Group != "" || c.Try || c.Number <= 0 {
|
||||
return func() {}, nil
|
||||
}
|
||||
api, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, broker.PullsBucket)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the pull requests' keys: %w", err)
|
||||
}
|
||||
watching, end := context.WithCancel(ctx)
|
||||
w, err := kv.Watch(watching, PullKeyOf(c.Owner, c.Repo, c.Number), jetstream.UpdatesOnly())
|
||||
if err != nil {
|
||||
end()
|
||||
return nil, err
|
||||
}
|
||||
go func() {
|
||||
defer w.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-watching.Done():
|
||||
return
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if e == nil || e.Operation() != jetstream.KeyValuePut {
|
||||
continue
|
||||
}
|
||||
var key PullKey
|
||||
if json.Unmarshal(e.Value(), &key) != nil {
|
||||
continue
|
||||
}
|
||||
if key.Moved(r.Ref, c.Trunk) {
|
||||
moved(key)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return end, nil
|
||||
}
|
||||
|
||||
// checkTakeWait is how long a taking loop waits on one queue for an ask that is not there, and takeIdleWait how long it
|
||||
// waits on the first when all three are empty.
|
||||
const (
|
||||
checkTakeWait = 30 * time.Millisecond
|
||||
takeIdleWait = time.Second
|
||||
)
|
||||
|
||||
// CheckRunning is what a builder says of the check it is running, in the bucket of the checks running (novox/hq ADR 0406
|
||||
// §3): the head and trunk commit it checks, the builder and its build. The key is the pull request's (RunningKeyOf).
|
||||
type CheckRunning struct {
|
||||
Head string `json:"head"`
|
||||
Trunk string `json:"trunk"`
|
||||
Builder string `json:"builder"`
|
||||
Build string `json:"build"`
|
||||
}
|
||||
|
||||
// RunningKeyOf is the key a check is said running under: its pull request's tokens, or `group.<name>` for a group's
|
||||
// composed check; none for a try, which no other request replaces.
|
||||
func RunningKeyOf(r BuildRequest) (string, bool) {
|
||||
c := r.Check
|
||||
switch {
|
||||
case c == nil, c.Try:
|
||||
return "", false
|
||||
case c.Group != "":
|
||||
return "group." + tokenOf(c.Group), true
|
||||
case c.Number <= 0:
|
||||
return "", false
|
||||
}
|
||||
return PullKeyOf(c.Owner, c.Repo, c.Number), true
|
||||
}
|
||||
|
||||
// runningBucket opens the bucket of the checks running.
|
||||
func runningBucket(ctx context.Context, conn *nats.Conn) (jetstream.KeyValue, error) {
|
||||
api, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, broker.CheckRunningBucket)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the checks running: %w", err)
|
||||
}
|
||||
return kv, nil
|
||||
}
|
||||
|
||||
// ReadRunning reads the key of the check running for a request's pull request: false when there is none.
|
||||
func ReadRunning(ctx context.Context, conn *nats.Conn, r BuildRequest) (CheckRunning, bool, error) {
|
||||
key, ok := RunningKeyOf(r)
|
||||
if !ok {
|
||||
return CheckRunning{}, false, nil
|
||||
}
|
||||
kv, err := runningBucket(ctx, conn)
|
||||
if err != nil {
|
||||
return CheckRunning{}, false, err
|
||||
}
|
||||
e, err := kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return CheckRunning{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return CheckRunning{}, false, err
|
||||
}
|
||||
var running CheckRunning
|
||||
if err := json.Unmarshal(e.Value(), &running); err != nil {
|
||||
return CheckRunning{}, false, nil // a key of a builder from before this means no match
|
||||
}
|
||||
return running, true, nil
|
||||
}
|
||||
|
||||
// Same says a running check is for this request's head and trunk commit: both said, and the same. A key that names neither
|
||||
// (an older builder's) is no match.
|
||||
func (c CheckRunning) Same(r BuildRequest) bool {
|
||||
return r.Check != nil && c.Head != "" && c.Trunk != "" && r.Check.Trunk != "" && sameCommit(c.Head, r.Ref) &&
|
||||
sameCommit(c.Trunk, r.Check.Trunk)
|
||||
}
|
||||
|
||||
// keepRunning says a check is running: its key put now, put again at each heartbeat, and deleted by the stop it returns,
|
||||
// whatever the outcome. Said to nobody that cannot be told: a bucket that is not there is no reason not to run the check.
|
||||
func keepRunning(ctx context.Context, conn *nats.Conn, r BuildRequest, builder string, every time.Duration) (stop func()) {
|
||||
key, ok := RunningKeyOf(r)
|
||||
if !ok {
|
||||
return func() {}
|
||||
}
|
||||
kv, err := runningBucket(ctx, conn)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s runs without saying so: %v\n", r.ID, err)
|
||||
return func() {}
|
||||
}
|
||||
trunk := ""
|
||||
if r.Check != nil {
|
||||
trunk = r.Check.Trunk
|
||||
}
|
||||
body, _ := json.Marshal(CheckRunning{Head: r.Ref, Trunk: trunk, Builder: builder, Build: r.ID})
|
||||
put := func() {
|
||||
putting, end := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||
defer end()
|
||||
if _, err := kv.Put(putting, key, body); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s could not say it is running: %v\n", r.ID, err)
|
||||
}
|
||||
}
|
||||
put()
|
||||
done := make(chan struct{})
|
||||
finished := make(chan struct{})
|
||||
go func() {
|
||||
defer close(finished)
|
||||
tick := time.NewTicker(every)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
case <-tick.C:
|
||||
put()
|
||||
}
|
||||
}
|
||||
}()
|
||||
return func() {
|
||||
close(done)
|
||||
<-finished
|
||||
deleting, end := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||
defer end()
|
||||
if err := kv.Delete(deleting, key); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s could not take its running key away; it ages out: %v\n", r.ID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,650 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// The checks of pull requests are queued by the pull request (novox/hq ADR 0406 §3, §4, issue 974).
|
||||
|
||||
func aPullCheck(id string, number int, head, trunk string, ready bool) BuildRequest {
|
||||
return BuildRequest{ID: id, Repository: "https://forge.example/novox/a.git", Ref: head,
|
||||
Check: &CheckRequest{Owner: "novox", Repo: "a", Number: number, Trunk: trunk, Ready: ready}}
|
||||
}
|
||||
|
||||
// The subject a check is asked on says its queue and its pull request: ready or other, `<owner>.<repo>.<number>` with
|
||||
// a dot in a name made an underscore, `group.<name>` for a group's, the id for a try; a build is on the seat's own queue.
|
||||
func TestACheckIsAskedOnItsQueueBySubject(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
r BuildRequest
|
||||
want string
|
||||
}{
|
||||
{aPullCheck("b1", 7, "h", "t", false), "mesh.seat.node-build-agent.check.other.novox.a.7"},
|
||||
{aPullCheck("b2", 7, "h", "t", true), "mesh.seat.node-build-agent.check.ready.novox.a.7"},
|
||||
{BuildRequest{ID: "b3", Check: &CheckRequest{Owner: "no.vox", Repo: "a.b", Number: 12}}, "mesh.seat.node-build-agent.check.other.no_vox.a_b.12"},
|
||||
{BuildRequest{ID: "b4", Check: &CheckRequest{Owner: "novox", Repo: "a", Group: "g.1", Ready: true}}, "mesh.seat.node-build-agent.check.ready.group.g_1"},
|
||||
{BuildRequest{ID: "b5", Check: &CheckRequest{Owner: "novox", Repo: "a", Try: true}}, "mesh.seat.node-build-agent.check.other.try.b5"},
|
||||
{BuildRequest{ID: "b6", Repository: "/r"}, "mesh.seat.node-build-agent.accept.build"},
|
||||
{BuildRequest{ID: "b7", Repository: "/r", DryRun: true}, "mesh.seat.node-build-agent.accept.build"},
|
||||
} {
|
||||
if got := WorkSubjectOf(TheBuildMachine, c.r); got != c.want {
|
||||
t.Errorf("%s is asked on %s; want %s", c.r.ID, got, c.want)
|
||||
}
|
||||
}
|
||||
// The retired role has no check queues: its checks wait with its builds.
|
||||
if got := WorkSubjectOf(TheBuildMachineBefore, aPullCheck("b8", 7, "h", "t", true)); got != "mesh.seat.mesh-build-machine.accept.build" {
|
||||
t.Errorf("a check of the retired role is asked on %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func queueOf(t *testing.T, js *broker.JetStream) Queue {
|
||||
t.Helper()
|
||||
q, err := ReadQueue(context.Background(), js, TheBuildMachine)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return q
|
||||
}
|
||||
|
||||
// A newer request for a pull request replaces the one still waiting, by the stream's rule, and each open pull request is
|
||||
// left with one queued check (acceptance 5).
|
||||
func TestANewerCheckRequestReplacesOneStillWaiting(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
ctx := t.Context()
|
||||
asks := BuildsOn(js, TheBuildMachine)
|
||||
for _, r := range []BuildRequest{
|
||||
aPullCheck("check-7-a", 7, "head-a-0000000", "trunk-0-00000", false),
|
||||
aPullCheck("check-8-a", 8, "head-b-0000000", "trunk-0-00000", false),
|
||||
aPullCheck("check-7-b", 7, "head-a-0000000", "trunk-1-00000", false), // a merge moved the trunk
|
||||
aPullCheck("check-8-b", 8, "head-b-0000000", "trunk-1-00000", false),
|
||||
aPullCheck("check-7-c", 7, "head-a-0000000", "trunk-2-00000", false), // and a second merge within the minute
|
||||
} {
|
||||
if err := asks.Ask(ctx, r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var got []string
|
||||
for _, a := range queueOf(t, js).Asks {
|
||||
if a.State != AskWaiting {
|
||||
t.Errorf("%s is %s", a.ID, a.State)
|
||||
}
|
||||
got = append(got, a.ID)
|
||||
}
|
||||
if want := []string{"check-8-b", "check-7-c"}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("queued %v; want one check a pull request, the newest: %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A delivery build is taken before a check of a ready pull request, that before any other check, and the others in the
|
||||
// order they were asked.
|
||||
func TestADeliveryBuildIsTakenBeforeAReadyCheckAndThatBeforeAnyOther(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
defer stop()
|
||||
asks := BuildsOn(js, TheBuildMachine)
|
||||
for _, r := range []BuildRequest{
|
||||
aPullCheck("other-1", 1, "h1", "t", false),
|
||||
aPullCheck("other-2", 2, "h2", "t", false),
|
||||
aPullCheck("ready-3", 3, "h3", "t", true),
|
||||
{ID: "delivery-4", Repository: "/d"},
|
||||
aPullCheck("other-5", 5, "h5", "t", false),
|
||||
aPullCheck("ready-6", 6, "h6", "t", true),
|
||||
} {
|
||||
if err := asks.Ask(ctx, r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
took := make(chan string, 8)
|
||||
machine := MachineOverNATS(js, "anchor")
|
||||
defer machine.Close()
|
||||
go func() {
|
||||
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
|
||||
took <- work.Request().ID
|
||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
|
||||
_ = work.Done()
|
||||
})
|
||||
}()
|
||||
var order []string
|
||||
for len(order) < 6 {
|
||||
select {
|
||||
case id := <-took:
|
||||
order = append(order, id)
|
||||
case <-time.After(15 * time.Second):
|
||||
t.Fatalf("taken so far: %v", order)
|
||||
}
|
||||
}
|
||||
if want := []string{"delivery-4", "ready-3", "ready-6", "other-1", "other-2", "other-5"}; !reflect.DeepEqual(order, want) {
|
||||
t.Fatalf("taken as %v; want %v", order, want)
|
||||
}
|
||||
}
|
||||
|
||||
// aKeyBucket makes mesh-delivery's bucket of pull requests' keys, which the bus here has not been given by that module.
|
||||
func aKeyBucket(t *testing.T, js *broker.JetStream) jetstream.KeyValue {
|
||||
t.Helper()
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv, err := api.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: broker.PullsBucket})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return kv
|
||||
}
|
||||
|
||||
func putKey(t *testing.T, kv jetstream.KeyValue, number int, k PullKey) {
|
||||
t.Helper()
|
||||
body, _ := json.Marshal(k)
|
||||
if _, err := kv.Put(t.Context(), PullKeyOf("novox", "a", number), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// A request left on the third queue for a head and a trunk commit its key says were judged is acknowledged and dropped when a
|
||||
// builder takes it; one for a head the key does not say judged is run.
|
||||
func TestARequestLeftBehindForAJudgedHeadIsDroppedWhenTaken(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
kv := aKeyBucket(t, js)
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
defer stop()
|
||||
putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000", Verdict: "pass"})
|
||||
putKey(t, kv, 8, PullKey{Head: "head-8-000000000", Trunk: "trunk-1-000000"}) // asked, not judged
|
||||
putKey(t, kv, 9, PullKey{Head: "head-9-000000000", Trunk: "trunk-2-000000", Verdict: "pass"}) // judged against another trunk
|
||||
asks := BuildsOn(js, TheBuildMachine)
|
||||
for _, r := range []BuildRequest{
|
||||
aPullCheck("left-7", 7, "head-7-000000000", "trunk-1-000000", false),
|
||||
aPullCheck("fresh-8", 8, "head-8-000000000", "trunk-1-000000", false),
|
||||
aPullCheck("moved-9", 9, "head-9-000000000", "trunk-1-000000", false),
|
||||
} {
|
||||
if err := asks.Ask(ctx, r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
took := make(chan string, 8)
|
||||
machine := MachineOverNATS(js, "anchor")
|
||||
defer machine.Close()
|
||||
go func() {
|
||||
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
|
||||
took <- work.Request().ID
|
||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
|
||||
_ = work.Done()
|
||||
})
|
||||
}()
|
||||
var ran []string
|
||||
for len(ran) < 2 {
|
||||
select {
|
||||
case id := <-took:
|
||||
ran = append(ran, id)
|
||||
case <-time.After(15 * time.Second):
|
||||
t.Fatalf("run so far: %v", ran)
|
||||
}
|
||||
}
|
||||
if want := []string{"fresh-8", "moved-9"}; !reflect.DeepEqual(ran, want) {
|
||||
t.Fatalf("run %v; want %v, the judged one dropped", ran, want)
|
||||
}
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if n := len(queueOf(t, js).Asks); n != 0 {
|
||||
t.Errorf("%d ask(s) are left in the queues; the dropped one was to be acknowledged", n)
|
||||
}
|
||||
}
|
||||
|
||||
// A running check stops when its key's head moves or its trunk commit does, not when the key only gains its verdict, and a
|
||||
// group's check, which a member's merge in the group's order writes nothing to the key of, is not stopped at all.
|
||||
func TestARunningCheckStopsWhenItsKeysHeadMovesAndAGroupsDoesNot(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
kv := aKeyBucket(t, js)
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
defer stop()
|
||||
moved := make(chan PullKey, 4)
|
||||
req := aPullCheck("check-7", 7, "head-7-000000000", "trunk-1-000000", false)
|
||||
end, err := WatchPull(ctx, js.Conn(), req, func(k PullKey) { moved <- k })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer end()
|
||||
putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000"})
|
||||
putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000", Verdict: "pass"})
|
||||
select {
|
||||
case k := <-moved:
|
||||
t.Fatalf("stopped by a key that moved nothing: %+v", k)
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
}
|
||||
putKey(t, kv, 7, PullKey{Head: "head-7-111111111", Trunk: "trunk-1-000000"})
|
||||
select {
|
||||
case k := <-moved:
|
||||
if k.Head != "head-7-111111111" {
|
||||
t.Errorf("stopped by %+v", k)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("a key whose head moved did not stop the check")
|
||||
}
|
||||
|
||||
// And the trunk commit.
|
||||
req2 := aPullCheck("check-8", 8, "head-8-000000000", "trunk-1-000000", false)
|
||||
end2, err := WatchPull(ctx, js.Conn(), req2, func(k PullKey) { moved <- k })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer end2()
|
||||
putKey(t, kv, 8, PullKey{Head: "head-8-000000000", Trunk: "trunk-2-000000"})
|
||||
select {
|
||||
case <-moved:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("a key whose trunk commit moved did not stop the check")
|
||||
}
|
||||
|
||||
// A group's check watches nothing: a sibling's merge in the group's order writes nothing to a key, and a key it does
|
||||
// have is the pull request's own.
|
||||
group := BuildRequest{ID: "group-1", Ref: "head-7-000000000",
|
||||
Check: &CheckRequest{Owner: "novox", Repo: "a", Number: 7, Group: "g1"}}
|
||||
end3, err := WatchPull(ctx, js.Conn(), group, func(k PullKey) { moved <- k })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer end3()
|
||||
putKey(t, kv, 7, PullKey{Head: "head-7-222222222", Trunk: "trunk-9-000000"})
|
||||
select {
|
||||
case k := <-moved:
|
||||
t.Fatalf("a group's check was stopped by %+v", k)
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPullKeyIsMovedByHeadOrTrunkAndJudgedByVerdict(t *testing.T) {
|
||||
k := PullKey{Head: "aaaaaaaaaaaa1111", Trunk: "tttttttttttt1111", Verdict: "pass"}
|
||||
if k.Moved("aaaaaaaaaaaa", "tttttttttttt") || !k.Moved("bbbbbbbbbbbbbbbb", "tttttttttttt") || !k.Moved("aaaaaaaaaaaa", "uuuuuuuuuuuuuu") {
|
||||
t.Error("moved is wrong")
|
||||
}
|
||||
if !k.Judged("aaaaaaaaaaaa1111", "tttttttttttt1111") || k.Judged("aaaaaaaaaaaa1111", "") || (PullKey{Head: "a", Trunk: "t"}).Judged("a", "t") {
|
||||
t.Error("judged is wrong")
|
||||
}
|
||||
}
|
||||
|
||||
// The queue is read across the three streams in the order they are handed out, each ask says the stream it waits in, and an
|
||||
// ask is removed from that stream.
|
||||
func TestTheQueueIsReadAcrossTheThreeStreamsAndAnAskIsDeletedFromItsOwn(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
ctx := t.Context()
|
||||
asks := BuildsOn(js, TheBuildMachine)
|
||||
for _, r := range []BuildRequest{
|
||||
aPullCheck("other-1", 1, "h1", "t", false),
|
||||
aPullCheck("ready-2", 2, "h2", "t", true),
|
||||
{ID: "delivery-3", Repository: "/d"},
|
||||
} {
|
||||
if err := asks.Ask(ctx, r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
q := queueOf(t, js)
|
||||
var ids, streams []string
|
||||
for _, a := range q.Asks {
|
||||
ids, streams = append(ids, a.ID), append(streams, a.Stream)
|
||||
}
|
||||
if want := []string{"delivery-3", "ready-2", "other-1"}; !reflect.DeepEqual(ids, want) {
|
||||
t.Fatalf("read %v; want the order they are handed out, %v", ids, want)
|
||||
}
|
||||
if want := []string{"SEAT_NODE_BUILD_AGENT", broker.ChecksReadyStream, broker.ChecksOtherStream}; !reflect.DeepEqual(streams, want) {
|
||||
t.Fatalf("streams %v; want %v", streams, want)
|
||||
}
|
||||
ready, _ := q.Find("ready-2")
|
||||
if w := WorkerOf(TheBuildMachine, ready); w.Name != broker.ChecksReadyWorker {
|
||||
t.Errorf("the ready check is handed out by %s", w.Name)
|
||||
}
|
||||
if removed, err := DeleteAsk(ctx, js, TheBuildMachine, ready); err != nil || removed != ready.Seq {
|
||||
t.Fatalf("deleted %d, %v", removed, err)
|
||||
}
|
||||
if removed, err := DeleteAsk(ctx, js, TheBuildMachine, ready); err != nil || removed != 0 {
|
||||
t.Fatalf("deleted again: %d, %v", removed, err)
|
||||
}
|
||||
if n := len(queueOf(t, js).Asks); n != 2 {
|
||||
t.Errorf("%d asks left; want 2", n)
|
||||
}
|
||||
}
|
||||
|
||||
// aRunningBucket makes the bucket the builders say the checks they run in, with an age of its own short enough for a test
|
||||
// (the mesh's is two minutes): a key not renewed within it is gone.
|
||||
func aRunningBucket(t *testing.T, js *broker.JetStream, age time.Duration) jetstream.KeyValue {
|
||||
t.Helper()
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv, err := api.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: broker.CheckRunningBucket, History: 1, TTL: age})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return kv
|
||||
}
|
||||
|
||||
// A builder taking a request for a pull request's head and trunk commit that another builder is running a check of drops it:
|
||||
// the bucket of running checks names the head and the trunk, the builder and the build (issue 974).
|
||||
func TestARunningCheckOnOneBuilderMakesAnotherDropTheSameRequestWaiting(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
aKeyBucket(t, js)
|
||||
aRunningBucket(t, js, 2*time.Minute)
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
defer stop()
|
||||
asks := BuildsOn(js, TheBuildMachine)
|
||||
running := make(chan string, 4)
|
||||
release := make(chan struct{})
|
||||
a := MachineOverNATS(js, "builder-a")
|
||||
defer a.Close()
|
||||
go func() {
|
||||
_ = a.Take(ctx, func(ctx context.Context, work Build) {
|
||||
running <- work.Request().ID
|
||||
<-release
|
||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"})
|
||||
_ = work.Done()
|
||||
})
|
||||
}()
|
||||
if err := asks.Ask(ctx, aPullCheck("first", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-running:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("builder a took nothing")
|
||||
}
|
||||
// The same head and trunk commit asked again while it runs, and a pull request's other head asked beside it.
|
||||
if err := asks.Ask(ctx, aPullCheck("again", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := MachineOverNATS(js, "builder-b")
|
||||
defer b.Close()
|
||||
took := make(chan string, 4)
|
||||
go func() {
|
||||
_ = b.Take(ctx, func(ctx context.Context, work Build) {
|
||||
took <- work.Request().ID
|
||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"})
|
||||
_ = work.Done()
|
||||
})
|
||||
}()
|
||||
select {
|
||||
case id := <-took:
|
||||
t.Fatalf("builder b took %s, which builder a is running a check of", id)
|
||||
case <-time.After(3 * time.Second):
|
||||
}
|
||||
if n := len(queueOf(t, js).Of(AskWaiting)); n != 0 {
|
||||
t.Errorf("%d request(s) still wait; builder b was to drop the one for a head and trunk commit being checked", n)
|
||||
}
|
||||
// Another trunk commit is another check.
|
||||
if err := asks.Ask(ctx, aPullCheck("moved", 7, "head-7-000000000", "trunk-2-000000", false)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case id := <-took:
|
||||
if id != "moved" {
|
||||
t.Fatalf("builder b took %s", id)
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("a check against another trunk commit was dropped too")
|
||||
}
|
||||
close(release)
|
||||
}
|
||||
|
||||
// A key that is not renewed ages out, and the request for its head and trunk commit is then taken: a builder that died
|
||||
// running a check leaves nothing that holds the next one back for longer than the bucket's age.
|
||||
func TestAKeyNotRenewedAgesOutAndTheRequestIsThenTaken(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
aKeyBucket(t, js)
|
||||
kv := aRunningBucket(t, js, 2*time.Second)
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
defer stop()
|
||||
body, _ := json.Marshal(CheckRunning{Head: "head-7-000000000", Trunk: "trunk-1-000000", Builder: "dead", Build: "gone"})
|
||||
if _, err := kv.Put(ctx, PullKeyOf("novox", "a", 7), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asks := BuildsOn(js, TheBuildMachine)
|
||||
took := make(chan string, 4)
|
||||
b := MachineOverNATS(js, "builder-b")
|
||||
defer b.Close()
|
||||
go func() {
|
||||
_ = b.Take(ctx, func(ctx context.Context, work Build) {
|
||||
took <- work.Request().ID
|
||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"})
|
||||
_ = work.Done()
|
||||
})
|
||||
}()
|
||||
if err := asks.Ask(ctx, aPullCheck("while-it-lives", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case id := <-took:
|
||||
t.Fatalf("%s was taken while the key was fresh", id)
|
||||
case <-time.After(1200 * time.Millisecond):
|
||||
}
|
||||
time.Sleep(2500 * time.Millisecond) // the key was not renewed
|
||||
if err := asks.Ask(ctx, aPullCheck("after-it-aged", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case id := <-took:
|
||||
if id != "after-it-aged" {
|
||||
t.Fatalf("took %s", id)
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("the request was not taken after the key aged out")
|
||||
}
|
||||
}
|
||||
|
||||
// A builder running a check renews its key on each heartbeat, so it outlives the bucket's age, and deletes it when the check
|
||||
// ends, whatever the outcome.
|
||||
func TestARunningCheckRenewsItsKeyAndDeletesItWhenItEnds(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
kv := aRunningBucket(t, js, 2*time.Second)
|
||||
was := heartbeatWhileWorking
|
||||
heartbeatWhileWorking = 400 * time.Millisecond
|
||||
t.Cleanup(func() { heartbeatWhileWorking = was })
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
defer stop()
|
||||
running := make(chan struct{}, 1)
|
||||
release := make(chan struct{})
|
||||
m := MachineOverNATS(js, "builder-a")
|
||||
defer m.Close()
|
||||
go func() {
|
||||
_ = m.Take(ctx, func(ctx context.Context, work Build) {
|
||||
running <- struct{}{}
|
||||
<-release
|
||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"})
|
||||
_ = work.Done()
|
||||
})
|
||||
}()
|
||||
if err := BuildsOn(js, TheBuildMachine).Ask(ctx, aPullCheck("long", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
<-running
|
||||
key := PullKeyOf("novox", "a", 7)
|
||||
e, err := kv.Get(ctx, key)
|
||||
if err != nil {
|
||||
t.Fatalf("no key while the check runs: %v", err)
|
||||
}
|
||||
var got CheckRunning
|
||||
_ = json.Unmarshal(e.Value(), &got)
|
||||
if got.Head != "head-7-000000000" || got.Trunk != "trunk-1-000000" || got.Builder != "builder-a" || got.Build != "long" {
|
||||
t.Errorf("the key says %+v", got)
|
||||
}
|
||||
time.Sleep(3 * time.Second) // longer than the bucket's age
|
||||
if _, err := kv.Get(ctx, key); err != nil {
|
||||
t.Fatalf("the key aged out under a check still running: %v", err)
|
||||
}
|
||||
close(release)
|
||||
for end := time.Now().Add(5 * time.Second); ; time.Sleep(50 * time.Millisecond) {
|
||||
if _, err := kv.Get(ctx, key); err != nil {
|
||||
break
|
||||
}
|
||||
if time.Now().After(end) {
|
||||
t.Fatal("the key stays after the check ended")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A key whose verdict is "error" — a check that could not run, or was stopped — judged nothing: a request for that head and
|
||||
// trunk commit must run (issue 974, review 2).
|
||||
func TestAnErrorVerdictJudgesNothing(t *testing.T) {
|
||||
for verdict, judged := range map[string]bool{"pass": true, "warning": true, "fail": true, "error": false, "": false, "pending": false} {
|
||||
k := PullKey{Head: "aaaaaaaaaaaa1111", Trunk: "tttttttttttt1111", Verdict: verdict}
|
||||
if got := k.Judged("aaaaaaaaaaaa1111", "tttttttttttt1111"); got != judged {
|
||||
t.Errorf("a key with verdict %q judged = %v; want %v", verdict, got, judged)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A check stopped because the trunk moved leaves the newer request, for the head against the new trunk commit, taken and
|
||||
// run: the stop is the stopped one's failure, not the key's verdict, and the newer request is not dropped by it.
|
||||
func TestACheckStoppedByATrunkMoveLeavesTheNewerRequestTakenAndRun(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
kv := aKeyBucket(t, js)
|
||||
aRunningBucket(t, js, 2*time.Minute)
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
defer stop()
|
||||
asks := BuildsOn(js, TheBuildMachine)
|
||||
outcomes := make(chan BuildResult, 8)
|
||||
watching, err := js.Conn().Subscribe(BuildOutcome(), func(m *nats.Msg) {
|
||||
var r BuildResult
|
||||
if json.Unmarshal(m.Data, &r) == nil {
|
||||
outcomes <- r
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = watching.Unsubscribe() }()
|
||||
|
||||
ran := make(chan string, 4)
|
||||
m := MachineOverNATS(js, "builder-a")
|
||||
defer m.Close()
|
||||
go func() {
|
||||
_ = m.Take(ctx, func(ctx context.Context, work Build) {
|
||||
building, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
ran <- work.Request().ID
|
||||
stopped := make(chan struct{}, 1)
|
||||
end, err := work.(interface {
|
||||
WatchPull(context.Context, func(PullKey)) (func(), error)
|
||||
}).WatchPull(building, func(PullKey) { stopped <- struct{}{}; cancel() })
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
defer end()
|
||||
if work.Request().ID == "older" {
|
||||
select {
|
||||
case <-stopped:
|
||||
_ = work.Announce(ctx, BuildResult{ID: "older", On: "builder-a", Failed: SupersededByKey})
|
||||
case <-time.After(15 * time.Second):
|
||||
_ = work.Announce(ctx, BuildResult{ID: "older", On: "builder-a", Failed: "never stopped"})
|
||||
}
|
||||
} else {
|
||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Check: &CheckOutcome{Verdict: "pass"}})
|
||||
}
|
||||
_ = work.Done()
|
||||
})
|
||||
}()
|
||||
if err := asks.Ask(ctx, aPullCheck("older", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if id := <-ran; id != "older" {
|
||||
t.Fatalf("took %s", id)
|
||||
}
|
||||
time.Sleep(300 * time.Millisecond) // the watch is made
|
||||
// The trunk moved; the head did not. mesh-delivery writes the key for the new request, and the request is asked.
|
||||
putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-2-000000"})
|
||||
if err := asks.Ask(ctx, aPullCheck("newer", 7, "head-7-000000000", "trunk-2-000000", false)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case id := <-ran:
|
||||
if id != "newer" {
|
||||
t.Fatalf("took %s", id)
|
||||
}
|
||||
case <-time.After(15 * time.Second):
|
||||
t.Fatal("the newer request, for the same head on the new trunk commit, was not taken")
|
||||
}
|
||||
got := map[string]string{}
|
||||
for len(got) < 2 {
|
||||
select {
|
||||
case r := <-outcomes:
|
||||
got[r.ID] = r.Failed
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatalf("outcomes so far: %v", got)
|
||||
}
|
||||
}
|
||||
if got["older"] != SupersededByKey || got["newer"] != "" {
|
||||
t.Errorf("outcomes %v; want the older stopped and the newer run to a verdict", got)
|
||||
}
|
||||
}
|
||||
|
||||
// An unacknowledged check goes to a second builder: taken from a check worker and not acknowledged, it is handed out again
|
||||
// when the worker's wait is up.
|
||||
func TestAnUnacknowledgedCheckGoesToASecondBuilder(t *testing.T) {
|
||||
for _, ready := range []bool{false, true} {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
name, worker := "other", broker.ChecksOtherWorker
|
||||
if ready {
|
||||
name, worker = "ready", broker.ChecksReadyWorker
|
||||
}
|
||||
// A worker whose wait is a second, as the mesh's is a minute.
|
||||
for _, w := range broker.CheckWorkers() {
|
||||
if w.Name == worker {
|
||||
w.AckWaitSeconds = 1
|
||||
if err := js.EnsureConsumer(w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
if err := BuildsOn(js, TheBuildMachine).Ask(ctx, aPullCheck("dies-"+name, 7, "head-7-000000000", "trunk-1-000000", ready)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The first builder takes it and dies: its pull is fetched, never acknowledged.
|
||||
if _, err := fetchOne(js, worker); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
took := make(chan BuildRequest, 2)
|
||||
second := MachineOverNATS(js, "builder-b")
|
||||
go func() {
|
||||
_ = second.Take(ctx, func(ctx context.Context, work Build) {
|
||||
took <- work.Request()
|
||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"})
|
||||
_ = work.Done()
|
||||
})
|
||||
}()
|
||||
select {
|
||||
case r := <-took:
|
||||
if r.ID != "dies-"+name {
|
||||
t.Errorf("%s: took %s", name, r.ID)
|
||||
}
|
||||
case <-time.After(15 * time.Second):
|
||||
t.Errorf("%s: the unacknowledged check did not go to a second builder", name)
|
||||
}
|
||||
second.Close()
|
||||
stop()
|
||||
}
|
||||
}
|
||||
|
||||
// fetchOne takes the next message of a worker and leaves it unacknowledged, as a builder that died would.
|
||||
func fetchOne(js *broker.JetStream, worker string) (*nats.Msg, error) {
|
||||
for _, w := range broker.CheckWorkers() {
|
||||
if w.Name != worker {
|
||||
continue
|
||||
}
|
||||
sub, err := js.Context().PullSubscribe(w.Filters[0], w.Name, nats.Bind(w.Stream, w.Name), nats.ManualAck())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
got, err := sub.Fetch(1, nats.MaxWait(5*time.Second))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return got[0], nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
@@ -51,6 +51,8 @@ var Contracts = map[string]Contract{
|
||||
"at the router, and the controller acts on it once, recording that it did under the ask's id — so a word " +
|
||||
"heard again, or late, does nothing more (novox/hq ADR 0259)",
|
||||
Tests: []string{"TestAWarrantIsActedOnOnce"}},
|
||||
KindDeliveryMerged: {Unordered: "mesh-delivery's word that it merged a pull request, by the pull request and its head: " +
|
||||
"kept under that key once, so a word heard again or late rewrites the same value (novox/hq ADR 0420 §7)"},
|
||||
KindCatchUp: {Unordered: "a catalogue asking what it missed: answered from the record, whenever asked"},
|
||||
KindProvisioner: {Unordered: "a provider's newest word about a consumer, said again every fifteen minutes " +
|
||||
"while it holds (ADR 0224): the condition keeps the last observed, and S8 says when the words stop. " +
|
||||
|
||||
@@ -18,6 +18,7 @@ func TestEveryConsumedKindHasAContract(t *testing.T) {
|
||||
subjects := []string{EnrolSubject, BuiltSubject, ReportSubject("anchor"), AliveSubject("anchor"),
|
||||
ToolsAliveSubject("anchor"), HealthSubject("anchor"), "mesh.mod.postgres.event.provisioner.failing"}
|
||||
subjects = append(subjects, broker.ControllerFollows...)
|
||||
subjects = append(subjects, broker.ControllerHears...)
|
||||
kinds := map[string]bool{}
|
||||
for _, s := range subjects {
|
||||
kind, known := kindOfSubject(s)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user