Compare commits
51
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
96441ea28a | ||
|
|
abe356a262 | ||
|
|
41c9628757 | ||
|
|
3e3400f510 | ||
|
|
e022a3fd6f | ||
|
|
684a8783e5 | ||
|
|
11e364bd43 | ||
|
|
1e9b87e03a | ||
|
|
45a09f90e3 | ||
|
|
4538d05732 | ||
|
|
980d396d40 | ||
|
|
b8f77577fe | ||
|
|
28a278c146 | ||
|
|
c34ab0e43a | ||
|
|
771cc24fcb | ||
|
|
b3786cc67e | ||
|
|
b7d03b6897 | ||
|
|
21954ad1bc | ||
|
|
485eb44de4 | ||
|
|
8d562f9b22 | ||
|
|
8f2d22db0c | ||
|
|
cc8a74fce4 | ||
|
|
aa9da94260 | ||
|
|
01f615aa0a | ||
|
|
f501eaca43 | ||
|
|
995f8237ce | ||
|
|
707bfba4ea | ||
|
|
4463991e06 | ||
|
|
4357b3688f | ||
|
|
03927a6f6c | ||
|
|
499ff5c218 | ||
|
|
3d4ad6fefb | ||
|
|
6bf38072ca | ||
|
|
b174d743c0 | ||
|
|
37119ea31f | ||
|
|
af2c6b5e94 | ||
|
|
a929df9b0e | ||
|
|
a3c965b9a0 | ||
|
|
b9ec1919b8 | ||
|
|
d491356f59 | ||
|
|
47e1819101 | ||
|
|
0e25ed7c5a | ||
|
|
79e2923dc8 | ||
|
|
1f3fdf3111 | ||
|
|
11815c0fb3 | ||
|
|
696300b9e2 | ||
|
|
e7cc315fd0 | ||
|
|
60253c82b7 | ||
|
|
bb8f076cc0 | ||
|
|
bb4d2e1da4 | ||
|
|
a14dc04b7b |
@@ -90,15 +90,6 @@ redis-provisioner-image:
|
||||
@echo
|
||||
@docker image inspect $(REDIS_PROVISIONER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||
|
||||
# The proxy that turns a route grant into traffic reaching a workload.
|
||||
PROXY_IMAGE ?= mesh-route-proxy:$(VERSION)
|
||||
PROXY_DEV_TAG ?= mesh-route-proxy:development
|
||||
|
||||
proxy-image:
|
||||
docker build --build-arg GO_BASE=$(GO_BASE) -f modules/route-proxy/Dockerfile -t $(PROXY_IMAGE) -t $(PROXY_DEV_TAG) .
|
||||
@echo
|
||||
@docker image inspect $(PROXY_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||
|
||||
# The whole check. Raises a database, runs the repository's own check against it -- merge-check.sh,
|
||||
# the very script `mesh/repo-check` runs -- and takes the database down again, including when the
|
||||
# check fails, which is why the teardown is not conditional and the script's exit status is the
|
||||
@@ -117,7 +108,7 @@ proxy-image:
|
||||
# as the script runs them: a bus per test, of the release the mesh runs, makes the suite the same in any
|
||||
# order, and the timeout bounds a hang to a failure with a stack, never a stalled gate.
|
||||
check: postgres
|
||||
@sh merge-check.sh ; status=$$? ; docker rm -f $(PG_CONTAINER) >/dev/null 2>&1 || true ; exit $$status
|
||||
@sh merge-check.sh ; status=$$? ; docker rm -f -v $(PG_CONTAINER) >/dev/null 2>&1 || true ; exit $$status
|
||||
|
||||
# The unit level alone (untagged: no bus, no store, no other process), which a developer runs continuously. The tests
|
||||
# that need a bus or the store carry `//go:build integration` and run in `check`, not here; without a database the
|
||||
@@ -135,7 +126,7 @@ fmt:
|
||||
if [ -n "$$unformatted" ] ; then echo "not gofmt'd:" ; echo "$$unformatted" ; exit 1 ; fi
|
||||
|
||||
postgres:
|
||||
@docker rm -f $(PG_CONTAINER) >/dev/null 2>&1 || true
|
||||
@docker rm -f -v $(PG_CONTAINER) >/dev/null 2>&1 || true
|
||||
@docker run -d --name $(PG_CONTAINER) -e POSTGRES_PASSWORD=check \
|
||||
-p 127.0.0.1:$(PG_PORT):5432 $(PG_IMAGE) >/dev/null
|
||||
@printf 'waiting for postgres'
|
||||
@@ -147,7 +138,7 @@ postgres:
|
||||
echo ' — never came up' ; docker logs $(PG_CONTAINER) | tail -20 ; exit 1
|
||||
|
||||
postgres-stop:
|
||||
@docker rm -f $(PG_CONTAINER) >/dev/null 2>&1 || true
|
||||
@docker rm -f -v $(PG_CONTAINER) >/dev/null 2>&1 || true
|
||||
|
||||
clean:
|
||||
rm -rf build/
|
||||
|
||||
@@ -6,7 +6,11 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
)
|
||||
|
||||
// **The build agent says whether it is busy, and being asked drains it** (novox/hq ADR 0335 §1, §5). Its
|
||||
@@ -73,14 +77,77 @@ func drainedTooLong(now time.Time) error {
|
||||
"within %s (novox/hq ADR 0335)", began.UTC().Format(time.RFC3339), drainTooLong)
|
||||
}
|
||||
|
||||
// inHand is true while this process has claimed a build and not yet released it. The mark in the temporary directory
|
||||
// is only believed while it is: a mark with no build of this process behind it is stale (a crash, a fault in an
|
||||
// earlier path) and is removed, because left alone it hands back every ask for ever (issue 1768).
|
||||
var inHand atomic.Bool
|
||||
|
||||
// markRetry is how long an agent that cannot mark a build in hand takes no work, and markFault when it last failed to.
|
||||
// Taking nothing is the point: an ask fetched and handed back at once, many times a second, spends all its deliveries.
|
||||
const markRetry = 30 * time.Second
|
||||
|
||||
var (
|
||||
markMu sync.Mutex
|
||||
markFault time.Time
|
||||
markSaid string
|
||||
)
|
||||
|
||||
// markFaulty says the agent could not mark a build in hand and has not yet been able to since: it takes no work. Once
|
||||
// markRetry has passed it tries the mark itself, with no ask in hand, so that an ask is fetched only when marking works.
|
||||
func markFaulty(now time.Time) bool {
|
||||
markMu.Lock()
|
||||
failed := markFault
|
||||
markMu.Unlock()
|
||||
if failed.IsZero() {
|
||||
return false
|
||||
}
|
||||
if now.Sub(failed) < markRetry {
|
||||
return true
|
||||
}
|
||||
f, err := os.OpenFile(inHandMark(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
|
||||
if err != nil && errors.Is(err, os.ErrExist) && !inHand.Load() {
|
||||
_ = os.Remove(inHandMark())
|
||||
f, err = os.OpenFile(inHandMark(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
|
||||
}
|
||||
if err != nil {
|
||||
markFailed(err)
|
||||
return true
|
||||
}
|
||||
_ = f.Close()
|
||||
_ = os.Remove(inHandMark())
|
||||
markMu.Lock()
|
||||
markFault, markSaid = time.Time{}, ""
|
||||
markMu.Unlock()
|
||||
fmt.Fprintf(os.Stderr, "the mark of a build in hand can be made again (%s): taking builds\n", inHandMark())
|
||||
return false
|
||||
}
|
||||
|
||||
// markFailed notes that a build could not be marked in hand, and says it when it is a new reason.
|
||||
func markFailed(err error) {
|
||||
markMu.Lock()
|
||||
defer markMu.Unlock()
|
||||
markFault = time.Now()
|
||||
if said := err.Error(); said != markSaid {
|
||||
markSaid = said
|
||||
fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so this agent takes no build for %s and tries again: %v\n",
|
||||
inHandMark(), markRetry, err)
|
||||
}
|
||||
}
|
||||
|
||||
// claim marks a request in hand, unless the agent is drained: then nothing is in hand, and the request goes
|
||||
// back to the queue.
|
||||
func claim() bool {
|
||||
if !inHand.Load() {
|
||||
// Nothing of this process is in hand, so a mark already there is stale.
|
||||
if err := os.Remove(inHandMark()); err == nil {
|
||||
fmt.Fprintf(os.Stderr, "removed the mark of a build in hand (%s): no build of this agent is in hand\n", inHandMark())
|
||||
}
|
||||
}
|
||||
// Made, never written over: one build at a time, and a mark already there is a build in hand.
|
||||
f, err := os.OpenFile(inHandMark(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
|
||||
if err != nil {
|
||||
// Unmarked, an ask would say idle while this build runs: the build is not taken.
|
||||
fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so it is handed back: %v\n", inHandMark(), err)
|
||||
markFailed(err)
|
||||
return false
|
||||
}
|
||||
_, err = f.WriteString(time.Now().UTC().Format(time.RFC3339) + "\n")
|
||||
@@ -89,7 +156,7 @@ func claim() bool {
|
||||
}
|
||||
if err != nil {
|
||||
_ = os.Remove(inHandMark())
|
||||
fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so it is handed back: %v\n", inHandMark(), err)
|
||||
markFailed(err)
|
||||
return false
|
||||
}
|
||||
if drained() {
|
||||
@@ -97,6 +164,7 @@ func claim() bool {
|
||||
fmt.Fprintln(os.Stderr, "asked whether busy, so this agent takes no new build: handed back")
|
||||
return false
|
||||
}
|
||||
inHand.Store(true)
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -107,6 +175,7 @@ func holding(what string) {
|
||||
|
||||
// released ends the build in hand.
|
||||
func released() {
|
||||
inHand.Store(false)
|
||||
if err := os.Remove(inHandMark()); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
fmt.Fprintf(os.Stderr, "cannot remove the mark of the build in hand (%s): %v\n", inHandMark(), err)
|
||||
}
|
||||
@@ -144,3 +213,23 @@ func busy() (string, error) {
|
||||
}
|
||||
return fmt.Sprintf("busy: %s, in hand since %s; taking no new build", what, since), nil
|
||||
}
|
||||
|
||||
var (
|
||||
roomMu sync.Mutex
|
||||
roomSaid bool
|
||||
)
|
||||
|
||||
// noRoom says this node is under the floor of free space (novox/hq issue 1834): it takes no work, so another build node
|
||||
// takes the ask instead of this one taking it and ending it in an error. Said once when it starts and again when it ends.
|
||||
func noRoom(workspace string) bool {
|
||||
err := builder.RoomToWork("/", workspace)
|
||||
roomMu.Lock()
|
||||
defer roomMu.Unlock()
|
||||
if err != nil && !roomSaid {
|
||||
fmt.Fprintf(os.Stderr, "taking no builds: %v\n", err)
|
||||
} else if err == nil && roomSaid {
|
||||
fmt.Fprintf(os.Stderr, "there is room again: taking builds\n")
|
||||
}
|
||||
roomSaid = err != nil
|
||||
return err != nil
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -160,3 +161,79 @@ func TestAnUnknownCommandFails(t *testing.T) {
|
||||
t.Fatalf("help failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1768/AC2: a mark naming no build of this agent (left by a crash or a fault) is stale: the next ask removes it and is
|
||||
// run, never fetched and handed back in a loop.
|
||||
func TestAStaleInHandMarkIsRemovedAndTheAskRuns(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
inHand.Store(false)
|
||||
if err := undrained(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(inHandMark(), []byte("2026-10-21T18:45:06Z\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !claim() {
|
||||
t.Fatal("a stale in-hand mark made the agent hand back an ask")
|
||||
}
|
||||
// A live one — this agent's own build in hand — is never taken over.
|
||||
if claim() {
|
||||
t.Fatal("a second build was taken while one is in hand")
|
||||
}
|
||||
if _, err := busy(); err != nil {
|
||||
t.Fatalf("the build in hand was lost: %v", err)
|
||||
}
|
||||
released()
|
||||
}
|
||||
|
||||
// 1768/AC3: an agent that cannot mark a build in hand for another reason takes no work, instead of fetching asks and
|
||||
// handing them back; it takes work again once marking works.
|
||||
func TestAnAgentThatCannotMarkABuildTakesNoWorkUntilItCan(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Setenv("TMPDIR", dir)
|
||||
inHand.Store(false)
|
||||
markFault, markSaid = time.Time{}, ""
|
||||
t.Cleanup(func() { markFault, markSaid = time.Time{}, "" })
|
||||
if err := undrained(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The mark's place is a directory that cannot be removed or made over: a directory named as the mark.
|
||||
if err := os.MkdirAll(inHandMark()+"/x", 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if claim() {
|
||||
t.Fatal("an agent that cannot mark a build took it")
|
||||
}
|
||||
if !markFaulty(time.Now()) {
|
||||
t.Fatal("an agent that cannot mark a build still takes work")
|
||||
}
|
||||
// Still failing after the retry time: still takes nothing, and no ask was needed to find out.
|
||||
if !markFaulty(time.Now().Add(2 * markRetry)) {
|
||||
t.Fatal("an agent whose mark still cannot be made takes work")
|
||||
}
|
||||
if err := os.RemoveAll(inHandMark()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if markFaulty(time.Now().Add(4 * markRetry)) {
|
||||
t.Fatal("an agent whose mark can be made again takes no work")
|
||||
}
|
||||
if !claim() {
|
||||
t.Fatal("an agent that can mark takes no build")
|
||||
}
|
||||
released()
|
||||
}
|
||||
|
||||
// 1834/AC4: given a node under the floor of free space, when the taking loop asks, then it takes nothing, and takes again
|
||||
// once there is room.
|
||||
func TestANodeUnderTheFloorOfFreeSpaceTakesNoWork(t *testing.T) {
|
||||
was := builder.FreeBytes
|
||||
t.Cleanup(func() { builder.FreeBytes = was })
|
||||
builder.FreeBytes = func(string) (uint64, error) { return 2 << 30, nil }
|
||||
if !noRoom(t.TempDir()) {
|
||||
t.Error("a node with 2 GiB free took work")
|
||||
}
|
||||
builder.FreeBytes = func(string) (uint64, error) { return 200 << 30, nil }
|
||||
if noRoom(t.TempDir()) {
|
||||
t.Error("a node with 200 GiB free refused work")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,3 +36,17 @@ func TestTheSiblingsAnAskNamesReachTheSpecAndTheOutcome(t *testing.T) {
|
||||
t.Fatalf("a request that names none has siblings %+v", plain.Siblings)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC2: the held values a check request carries reach the spec; a request that carries none leaves it empty.
|
||||
func TestTheHeldValuesAnAskCarriesReachTheSpec(t *testing.T) {
|
||||
spec := checkSpecOf(link.BuildRequest{Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog",
|
||||
HeldConfig: []link.HeldConfig{{Pull: "novox/mesh-catalog#434", Head: "a", Node: "ace", Module: "n8n", Key: "media-gid",
|
||||
Value: []byte("2000"), Approved: "the operator"}}}})
|
||||
if len(spec.Held) != 1 || spec.Held[0].Node != "ace" || spec.Held[0].Key != "media-gid" || string(spec.Held[0].Value) != "2000" ||
|
||||
spec.Held[0].Approved != "the operator" {
|
||||
t.Fatalf("the spec reads %+v", spec.Held)
|
||||
}
|
||||
if plain := checkSpecOf(link.BuildRequest{Check: &link.CheckRequest{Owner: "o", Repo: "r"}}); plain.Held != nil {
|
||||
t.Fatalf("a request that carries none has %+v", plain.Held)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -91,7 +91,7 @@ func TestKillEndsTheBuildRunningHereAndNoOther(t *testing.T) {
|
||||
t.Errorf("kill said %q", said)
|
||||
}
|
||||
// Removed at the kill and again once the build had ended: a container made in between is caught.
|
||||
if len(removed) != 4 || !strings.Contains(removed[0], "label=mesh.build=build-1") || !strings.Contains(removed[2], "label=mesh.build=build-1") {
|
||||
if len(removed) != 6 || !strings.Contains(removed[0], "label=mesh.build=build-1") || !strings.Contains(removed[3], "label=mesh.build=build-1") {
|
||||
t.Errorf("removed %v", removed)
|
||||
}
|
||||
if c := h.current(); c.Running != nil {
|
||||
@@ -160,7 +160,7 @@ func TestAHolderStartingRemovesWhatEarlierBuildsLeftHere(t *testing.T) {
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
if n := h.removeLeftBehind(); n != 2 || len(removed) != 2 || removed[1] != "docker rm -f c1 c2" {
|
||||
if n := h.removeLeftBehind(); n != 2 || len(removed) != 3 || removed[2] != "docker rm -f -v c1 c2" {
|
||||
t.Fatalf("removed %d: %v", n, removed)
|
||||
}
|
||||
h.remove = func(context.Context, string, string, ...string) (string, error) {
|
||||
|
||||
@@ -64,6 +64,8 @@ is dialled except the broker.
|
||||
MESH_NPM_TOKEN the token for it, likewise
|
||||
MESH_NPM_SCOPE the scope it answers for (default: @novox)
|
||||
MESH_WORKSPACE where to clone and build (default: a temporary directory)
|
||||
MESH_BUILD_CGROUP_PARENT the slice every container a build or check starts joins, which carries the
|
||||
machine's limits on builds (novox/hq issue 1798; default: the runtime's own)
|
||||
|
||||
It also builds one module and stops, which is how a mesh is raised — before there is a
|
||||
broker to take work from or a registry to publish into:
|
||||
@@ -137,6 +139,10 @@ func run() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// A build and a check run below the machine's own work (novox/hq issue 1798): the agent itself at the
|
||||
// lowest priority, which its git and go children inherit, and every container in the build slice.
|
||||
_ = syscall.Setpriority(syscall.PRIO_PROCESS, 0, 19)
|
||||
builder.CgroupParent = strings.TrimSpace(os.Getenv("MESH_BUILD_CGROUP_PARENT"))
|
||||
workspace := os.Getenv("MESH_WORKSPACE")
|
||||
if workspace == "" {
|
||||
workspace = os.TempDir() + "/mesh-builder"
|
||||
@@ -184,7 +190,7 @@ func run() error {
|
||||
// Paused or drained, it takes nothing new (novox/hq ADR 0219, ADR 0335): a drained agent finishes the build
|
||||
// in hand and then says idle, and its container is replaced between builds.
|
||||
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{
|
||||
Paused: func() bool { return h.Paused() || drained() },
|
||||
Paused: func() bool { return h.Paused() || drained() || markFaulty(time.Now()) || noRoom(workspace) },
|
||||
Taking: func() { taking(seat) },
|
||||
Claim: claim,
|
||||
})
|
||||
@@ -499,6 +505,10 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
|
||||
Modules: c.Modules, New: c.New, Manifests: c.Manifests, Judge: c.Judge, Base: c.Base,
|
||||
Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held),
|
||||
Tests: c.Tests, StoreOnly: c.StoreOnly, SiblingsUnknown: c.SiblingsUnknown}
|
||||
for _, h := range c.HeldConfig {
|
||||
spec.Held = append(spec.Held, builder.HeldValue{Pull: h.Pull, Head: h.Head, Node: h.Node, Module: h.Module, Key: h.Key,
|
||||
Value: h.Value, Approved: h.Approved, By: h.By, At: h.At})
|
||||
}
|
||||
for _, s := range c.Siblings {
|
||||
spec.Siblings = append(spec.Siblings, builder.Sibling{Repository: s.Repository, Number: s.Number, Head: s.Head,
|
||||
Base: s.Base, Dir: s.Dir})
|
||||
|
||||
@@ -800,6 +800,9 @@ type keptMerge struct {
|
||||
// FirstBuild says the merge is a first merge into a repository the mesh builds nothing from, kept to ask the first
|
||||
// build of its modules once mesh-delivery's word on it is kept (novox/hq ADR 0432, issue 1146).
|
||||
FirstBuild bool `json:"first_build,omitempty"`
|
||||
// Held says which first builds of the merge were refused for a word the running build-agent does not read: the merge
|
||||
// waits for a build-agent that reads them, and is never failed (novox/hq ADR 0482 §5, issue 1173).
|
||||
Held []heldFirstBuild `json:"held,omitempty"`
|
||||
}
|
||||
|
||||
// announcedOf is a kept merge as the forge announced it, and what it moved; empty where the record says none.
|
||||
|
||||
@@ -191,6 +191,16 @@ func (f following) pullUpdated(ctx context.Context, p link.PullUpdated, w io.Wri
|
||||
scope.Modules, scope.Dependents, scope.New, scope.Manifests = nil, nil, nil, nil
|
||||
composedGate(&direct)
|
||||
}
|
||||
// **Mesh-config is told what the head declares at every announced head of a pull request in a repository that holds
|
||||
// modules** (novox/hq ADR 0502 clauses 2 and 6, issue 1826), also when the head touches no module: a head that reverts
|
||||
// the change must drop the value held for it. Asked of nothing else here.
|
||||
this := pullAtOf(p)
|
||||
if this.Truncated {
|
||||
this.Paths, this.Truncated = truncatedPaths(entries, p.Owner, p.Repo, scope.Modules), false
|
||||
}
|
||||
if scope.gated() || holdsModules(entries, p.Owner, p.Repo) {
|
||||
tellPullDeclares(ctx, w, []pullAt{this})
|
||||
}
|
||||
switch {
|
||||
case !scope.gated() && !scope.Mesh:
|
||||
fmt.Fprintf(w, "%s/%s#%d (%.8s): %s, and the repository is not the mesh's: said, nothing run\n",
|
||||
@@ -209,6 +219,9 @@ func (f following) pullUpdated(ctx context.Context, p link.PullUpdated, w io.Wri
|
||||
}
|
||||
request.Check.Plan = &plan
|
||||
request.Check.Composed = composed
|
||||
if !composed {
|
||||
carryHeld(ctx, w, &request, []pullAt{this})
|
||||
}
|
||||
// **The group's other open pull requests are read beside it** (novox/hq ADR 0382 §1).
|
||||
siblingsFrom(&request, p, entries, lines, groupsErr)
|
||||
seat := buildSeatHeld(ctx)
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// 1840/AC5 (ADR 0507 §5): the source of the declared-packages mirror composes each node and names, for a package two modules
|
||||
// declare and one declared absent, the modules that declare it; the same records give the same bytes on a second pass, so
|
||||
// the mirror writes nothing; a node with no package has a key that says so.
|
||||
func TestTheDeclaredPackagesMirrorNamesWhatEachNodesModulesDeclare(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "viewer", Version: "1", Resources: []map[string]any{
|
||||
{"id": "pkg-curl", "type": "package", "package": "curl"},
|
||||
{"id": "pkg-nano", "type": "package", "package": "nano", "absent": true}}})
|
||||
register(t, open, catalogue.Manifest{Module: "fetcher", Version: "1", Resources: []map[string]any{
|
||||
{"id": "pkg-curl", "type": "package", "package": "curl"}}})
|
||||
for _, module := range []string{"viewer", "fetcher"} {
|
||||
if _, err := assign(ctx, open, "anchor", module); err != nil {
|
||||
t.Fatalf("assign anchor %s: %v", module, err)
|
||||
}
|
||||
}
|
||||
source := declaredPackagesMirror(open)
|
||||
first, err := source(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got inventory.DeclaredPackagesMirrored
|
||||
if err := json.Unmarshal(first["anchor"], &got); err != nil {
|
||||
t.Fatalf("anchor has no value: %v", err)
|
||||
}
|
||||
// Every node also runs the mesh's own modules, which declare packages of their own: curl and nano are looked
|
||||
// up by name, and no package names viewer or fetcher but those they declare.
|
||||
byName := map[string]inventory.PackageDeclared{}
|
||||
for _, p := range got.Packages {
|
||||
byName[p.Package] = p
|
||||
}
|
||||
same := func(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
for _, p := range []inventory.PackageDeclared{{Package: "curl", Modules: []string{"fetcher", "viewer"}},
|
||||
{Package: "nano", Modules: []string{"viewer"}, Absent: true}} {
|
||||
if g, ok := byName[p.Package]; !ok || g.Absent != p.Absent || !same(g.Modules, p.Modules) {
|
||||
t.Errorf("anchor declares %s as %+v, want %+v (all: %+v)", p.Package, g, p, got.Packages)
|
||||
}
|
||||
}
|
||||
for _, p := range got.Packages {
|
||||
if p.Package == "curl" || p.Package == "nano" {
|
||||
continue
|
||||
}
|
||||
for _, m := range p.Modules {
|
||||
if m == "viewer" || m == "fetcher" {
|
||||
t.Errorf("%s is said to be declared by %s, which does not declare it", p.Package, m)
|
||||
}
|
||||
}
|
||||
}
|
||||
var laptop inventory.DeclaredPackagesMirrored
|
||||
if err := json.Unmarshal(first["laptop"], &laptop); err != nil || laptop.Node != "laptop" || laptop.Packages == nil {
|
||||
t.Errorf("a node assigned neither module has no value of its own: %s (%v)", first["laptop"], err)
|
||||
}
|
||||
for _, p := range laptop.Packages {
|
||||
if p.Package == "curl" || p.Package == "nano" {
|
||||
t.Errorf("laptop is assigned neither module and declares %+v", p)
|
||||
}
|
||||
}
|
||||
second, err := source(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for node, value := range first {
|
||||
if string(second[node]) != string(value) {
|
||||
t.Errorf("the value of %s changed between two passes over the same records:\n%s\n%s", node, value, second[node])
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1177,6 +1177,12 @@ func askGroupCheck(ctx context.Context, open *stores, group string, members []or
|
||||
return "", err
|
||||
}
|
||||
request.Check.Group = group
|
||||
// **The values held for the keys the members introduce are told and carried** (novox/hq ADR 0502, issue 1826).
|
||||
var pulls []pullAt
|
||||
for i, s := range changeSourcesOf(members, entries, reach) {
|
||||
pulls = append(pulls, pullAt{changeSource: s, Number: members[i].pull().Number})
|
||||
}
|
||||
withHeldConfig(ctx, os.Stderr, &request, pulls, true)
|
||||
// A group's check is queued as ready when any member's pull request is marked ready.
|
||||
request.Check.Ready = slices.ContainsFunc(members, func(m orderMember) bool { return m.Ready })
|
||||
world, err := restOfTheMesh(ctx, open, shelfOf(entries), "")
|
||||
|
||||
@@ -58,8 +58,17 @@ type stalledLine struct {
|
||||
// Severity is "urgent" for a group held or half merged and a refused member (novox/hq issues 1275, 1278); empty
|
||||
// or anything else is a warning, as before.
|
||||
Severity string `json:"severity,omitempty"`
|
||||
// Needs is "new-head" for a delivery whose cure is its author's new head (novox/hq ADR 0505 §5), Author that author
|
||||
// as the operator reads it ("the author of feat/x", with the session that pushed it when its pull request names one)
|
||||
// and Branch its branch. A mesh-delivery from before says none, and the words are as before.
|
||||
Needs string `json:"needs,omitempty"`
|
||||
Author string `json:"author,omitempty"`
|
||||
Branch string `json:"branch,omitempty"`
|
||||
}
|
||||
|
||||
// needsNewHead is what a stalled line needs when its cure is its author's new head (ADR 0505 §5).
|
||||
const needsNewHead = "new-head"
|
||||
|
||||
// severity is the condition's severity a stalled line asks: urgent when it says so, a warning otherwise.
|
||||
func (l stalledLine) severity() conditions.Severity {
|
||||
if l.Severity == string(conditions.Urgent) {
|
||||
|
||||
@@ -59,6 +59,18 @@ func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testin
|
||||
}
|
||||
}
|
||||
|
||||
// heardOf has this mesh's own view of the plans take in the plan as the bucket holds it. A verb of the
|
||||
// controller (deliveryCommand, plansCommand) opens an inventory of its own and writes the plan there; this
|
||||
// mesh's inventory learns of the write from the bucket's watch, a moment later, and a loaded machine makes the
|
||||
// moment long enough for advanceHeld to read the plan as it was, still waiting for its word, and ask nothing
|
||||
// (issue 1399). Reading the plan from the bucket puts it into the view at once, with no wait to bound.
|
||||
heardOf := func(id string) {
|
||||
t.Helper()
|
||||
if _, err := inv.PlanByID(ctx, id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// No holder on record: the merge starts its walk, as before.
|
||||
p := merge("c1aaaaaaaa", "modules/app/index.ts")
|
||||
if p.Waiting() || len(*asked) != 1 {
|
||||
@@ -90,6 +102,7 @@ func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testin
|
||||
if err := deliveryCommand(ctx, []string{"go", p.ID}); err == nil {
|
||||
t.Fatal("a walk was let go twice")
|
||||
}
|
||||
heardOf(p.ID)
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) != 2 {
|
||||
t.Fatalf("the word did not start the walk: %v", *asked)
|
||||
@@ -119,6 +132,7 @@ func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testin
|
||||
if err := plansCommand(ctx, []string{"go", p.ID, "--why", "mesh-delivery is down"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
heardOf(p.ID)
|
||||
advanceHeld(ctx, open)
|
||||
got, _ = inv.PlanByID(ctx, p.ID)
|
||||
if got.Waiting() || !strings.HasPrefix(got.Delivery.By, "a person") || len(*asked) < 4 {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -102,7 +103,7 @@ func (f following) resumeFirstBuilds(ctx context.Context, d deliveryMerge) error
|
||||
}
|
||||
for _, w := range waiting {
|
||||
k := announcedOf(w)
|
||||
if !k.FirstBuild || !strings.EqualFold(k.Owner+"/"+k.Repo, d.Owner+"/"+d.Repo) || k.Number != d.Number ||
|
||||
if !k.FirstBuild || len(k.Held) > 0 || !strings.EqualFold(k.Owner+"/"+k.Repo, d.Owner+"/"+d.Repo) || k.Number != d.Number ||
|
||||
k.HeadSHA != d.HeadSHA {
|
||||
continue
|
||||
}
|
||||
@@ -172,12 +173,122 @@ func firstBuildsOf(ctx context.Context, inv catalogueReader, m link.SourceMoved,
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
// heldFirstBuild says a first build was refused because its manifest uses a word the running build-agent does not read,
|
||||
// and its merge is kept waiting for a build-agent that reads it (novox/hq ADR 0482 §5, issue 1173).
|
||||
type heldFirstBuild struct {
|
||||
Module string `json:"module"`
|
||||
Path string `json:"path"`
|
||||
Word string `json:"word"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// judgeRefusedFirstBuild is the hold a first build's outcome puts its merge under: every module the merge holds already,
|
||||
// and this one, unless it is among them; nil unless the build failed on a word of its manifest the running build-agent
|
||||
// does not know, and was asked for a first merge, answered by the record of merges no plan answers or put back to waiting
|
||||
// by an earlier hold (a second refusal in one merge is held too, review of issue 1173).
|
||||
func judgeRefusedFirstBuild(result link.BuildResult, req inventory.BuildRequest, merge inventory.BatchedMerge) []heldFirstBuild {
|
||||
word := refusedWord(result.Failed)
|
||||
if word == "" || req.For != "merge" {
|
||||
return nil
|
||||
}
|
||||
var held []heldFirstBuild
|
||||
switch k := announcedOf(merge); {
|
||||
case merge.Plan == inventory.WaitingRecord:
|
||||
// Answered when its builds were asked: a hold from before (a build asked again) is over, and starts anew.
|
||||
case merge.Plan == "" && len(k.Held) > 0:
|
||||
held = k.Held
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
this := heldFirstBuild{Module: req.Name(), Path: strings.Trim(req.Path, "/"), Word: word, At: time.Now().UTC()}
|
||||
for i, h := range held {
|
||||
if strings.Trim(h.Path, "/") == this.Path {
|
||||
out := append([]heldFirstBuild(nil), held...)
|
||||
out[i] = this
|
||||
return out
|
||||
}
|
||||
}
|
||||
return append(append([]heldFirstBuild(nil), held...), this)
|
||||
}
|
||||
|
||||
// heldDirsOf is the directories a held merge asks again: its held modules' own, each once, "." for the repository's root.
|
||||
func heldDirsOf(k keptMerge) []string {
|
||||
var dirs []string
|
||||
seen := map[string]bool{}
|
||||
for _, h := range k.Held {
|
||||
dir := strings.Trim(h.Path, "/")
|
||||
if dir == "" {
|
||||
dir = "."
|
||||
}
|
||||
if !seen[dir] {
|
||||
seen[dir] = true
|
||||
dirs = append(dirs, dir)
|
||||
}
|
||||
}
|
||||
return dirs
|
||||
}
|
||||
|
||||
// unknownWord is how the strict reading of a manifest says a word is not one it knows: encoding/json's own words, which
|
||||
// the build-agent's answer carries whole. The build's answer has no typed field for it (Failed is the builder's words).
|
||||
var unknownWord = regexp.MustCompile(`json: unknown field "([^"]+)"`)
|
||||
|
||||
// refusedWord is the word a build's failure says the reader did not know; empty when it says none.
|
||||
func refusedWord(failed string) string {
|
||||
if m := unknownWord.FindStringSubmatch(failed); m != nil {
|
||||
return m[1]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// holdRefusedFirstBuild keeps a first build's merge waiting when the build was refused for a word the running build-agent
|
||||
// does not read (novox/hq ADR 0482 §5, issue 1173): no failed build is recorded, nothing expires the merge, and it is
|
||||
// named by a condition until a delivery of build-agent asks the build again. Answers whether the outcome was dealt with.
|
||||
func holdRefusedFirstBuild(ctx context.Context, inv *inventory.Inventory, result link.BuildResult) (bool, error) {
|
||||
if result.Failed == "" || refusedWord(result.Failed) == "" {
|
||||
return false, nil
|
||||
}
|
||||
req, found, err := inv.BuildRequestByID(ctx, result.ID)
|
||||
if err != nil || !found {
|
||||
return false, err
|
||||
}
|
||||
merge, found, err := inv.MergeOf(ctx, req.Repository, req.Commit)
|
||||
if err != nil || !found {
|
||||
return false, err
|
||||
}
|
||||
held := judgeRefusedFirstBuild(result, req, merge)
|
||||
if len(held) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
k := announcedOf(merge)
|
||||
k.Held = held
|
||||
event, err := json.Marshal(k)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := inv.HoldMerge(ctx, merge.Repository, merge.Commit, event); err != nil {
|
||||
return false, err
|
||||
}
|
||||
this := held[len(held)-1]
|
||||
for _, h := range held {
|
||||
if h.Path == strings.Trim(req.Path, "/") {
|
||||
this = h
|
||||
}
|
||||
}
|
||||
fmt.Printf("%s: the first build of %s was refused for the word %q, which the running build-agent does not read; the "+
|
||||
"merge %.8s of %s is kept waiting for a build-agent that reads it (holding %s), and the build is asked again when "+
|
||||
"one is delivered\n", result.ID, this.Module, this.Word, merge.Commit, merge.Repository, heldNames(held))
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// probeFirstBuildsID is the probe that reads the first merges waiting for mesh-delivery's word.
|
||||
const probeFirstBuildsID = "D18"
|
||||
|
||||
// kindFirstBuildUnasked is what a first merge waiting too long for mesh-delivery's word raises.
|
||||
const kindFirstBuildUnasked = "first-build-unasked"
|
||||
|
||||
// kindFirstBuildHeld is what a first merge held for a build-agent that reads its manifest raises.
|
||||
const kindFirstBuildHeld = "first-build-held"
|
||||
|
||||
// firstBuildWaitBound is how long a first merge waits for mesh-delivery's announcement of the operator's Approve before
|
||||
// it is said: a bound in the sense of ADR 0426 §3, not a delay. Nothing waits on it: the word's arrival asks the builds
|
||||
// at once, and the merge is said, never dropped, once it is past this (issue 1146).
|
||||
@@ -189,6 +300,10 @@ func firstBuildObservations(waiting []inventory.BatchedMerge, now time.Time) []c
|
||||
var out []conditions.Observation
|
||||
for _, w := range waiting {
|
||||
k := announcedOf(w)
|
||||
if len(k.Held) > 0 {
|
||||
out = append(out, heldFirstBuildObservation(k))
|
||||
continue
|
||||
}
|
||||
if !k.FirstBuild || now.Sub(w.Heard) <= firstBuildWaitBound {
|
||||
continue
|
||||
}
|
||||
@@ -211,6 +326,37 @@ func firstBuildObservations(waiting []inventory.BatchedMerge, now time.Time) []c
|
||||
return out
|
||||
}
|
||||
|
||||
// heldNames is the modules a merge holds, each with the word it waits for: "watchman (key-ttl), view (per-machine)".
|
||||
func heldNames(held []heldFirstBuild) string {
|
||||
names := make([]string, 0, len(held))
|
||||
for _, h := range held {
|
||||
names = append(names, fmt.Sprintf("%s (%q)", h.Module, h.Word))
|
||||
}
|
||||
return strings.Join(names, ", ")
|
||||
}
|
||||
|
||||
// heldFirstBuildObservation names a first merge held for a build-agent that reads its manifests: at once, since nothing
|
||||
// is lost and nothing times out, but the operator can see what the merge waits for. One condition names every module held.
|
||||
func heldFirstBuildObservation(k keptMerge) conditions.Observation {
|
||||
repository := k.Owner + "/" + k.Repo
|
||||
modules := make([]string, 0, len(k.Held))
|
||||
for _, h := range k.Held {
|
||||
modules = append(modules, h.Module)
|
||||
}
|
||||
held := heldNames(k.Held)
|
||||
return conditions.Observation{Scope: conditions.ScopeMerge, ID: fmt.Sprintf("%s_%.8s", repository, k.Commit),
|
||||
Kind: kindFirstBuildHeld, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the merge %.8s of %s adds modules whose manifests use words the running build-agent does not "+
|
||||
"read: %s; the merge waits for a build-agent that reads them", k.Commit, repository, held),
|
||||
Said: fmt.Sprintf("%s at %s waits for a build-agent that reads the words of %s", repository, k.Commit, held),
|
||||
Headline: fmt.Sprintf("The first build of %s waits for a newer build-agent", strings.Join(modules, ", ")),
|
||||
Explanation: fmt.Sprintf("The merge %.8s into %s adds modules whose manifests use words the build-agent now "+
|
||||
"running does not know: %s. The merge is kept, not failed: their first builds are asked again when a delivery "+
|
||||
"of build-agent ends.", k.Commit, repository, held),
|
||||
Resolved: "a delivery of build-agent ends and the first builds register the modules, or a hand `build` does",
|
||||
}
|
||||
}
|
||||
|
||||
// probeFirstBuilds is D18: no first merge waits past its bound for mesh-delivery's word unsaid.
|
||||
func probeFirstBuilds(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
waiting, err := d.open.inventory.WaitingMerges(ctx)
|
||||
@@ -219,3 +365,61 @@ func probeFirstBuilds(ctx context.Context, d *doctor) ([]conditions.Observation,
|
||||
}
|
||||
return firstBuildObservations(waiting, time.Now().UTC()), nil
|
||||
}
|
||||
|
||||
// askHeldFirstBuildsAgain asks again the first build of each first merge held for a build-agent that reads its manifest,
|
||||
// when the plan that ended delivered build-agent (novox/hq issue 1173): once for each end, since the ask answers the merge
|
||||
// and a refusal that comes again holds it anew. Answers how many merges it asked for.
|
||||
func askHeldFirstBuildsAgain(ctx context.Context, open *stores, p inventory.Plan) int {
|
||||
if !deliversBuildAgent(p) {
|
||||
return 0
|
||||
}
|
||||
actingOnMerges.Lock()
|
||||
defer actingOnMerges.Unlock()
|
||||
inv := open.inventory
|
||||
waiting, err := inv.WaitingMerges(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("%s delivered build-agent, and the first merges held for it could not be read: %v\n", p.ID, err)
|
||||
return 0
|
||||
}
|
||||
asked := 0
|
||||
for _, w := range waiting {
|
||||
k := announcedOf(w)
|
||||
if len(k.Held) == 0 {
|
||||
continue
|
||||
}
|
||||
// Its own held modules, not the rule of a first merge: a module of the merge that passed is registered by now,
|
||||
// and the repository no longer counts as one the mesh builds nothing from (review of issue 1173).
|
||||
why, err := approvedAnnounced(ctx, inv, k.Owner, k.Repo, keptMerge{SourceMoved: k.SourceMoved})
|
||||
if err != nil {
|
||||
fmt.Printf("%s delivered build-agent, and the merge %.8s of %s held for it asks no build now (%v): it stays "+
|
||||
"waiting\n", p.ID, w.Commit, w.Repository, err)
|
||||
continue
|
||||
}
|
||||
dirs := heldDirsOf(k)
|
||||
fmt.Printf("%s delivered build-agent: the first builds of %s are asked again\n", p.ID, heldNames(k.Held))
|
||||
paths := make([]string, 0, len(dirs))
|
||||
for _, dir := range dirs {
|
||||
if dir == "." {
|
||||
dir = ""
|
||||
}
|
||||
paths = append(paths, dir)
|
||||
}
|
||||
repository := k.Owner + "/" + k.Repo
|
||||
askBuilds(withFirstBuildWarrant(ctx, repository, paths, why), inv, k.SourceMoved,
|
||||
buildSource{Repository: repository, Seat: gitSeat}, dirs)
|
||||
// Answered: it no longer waits, so this end asks nothing twice; a refusal that comes again holds it anew.
|
||||
if err := inv.AnswerMerge(ctx, w.Repository, w.Commit, inventory.WaitingRecord, false); err != nil {
|
||||
fmt.Printf(" and the merge could not be marked answered: %v\n", err)
|
||||
continue
|
||||
}
|
||||
asked++
|
||||
}
|
||||
return asked
|
||||
}
|
||||
|
||||
// deliversBuildAgent says a delivery plan ended delivered with build-agent built in it: the reader a held first build
|
||||
// waits for is the one now running.
|
||||
func deliversBuildAgent(p inventory.Plan) bool {
|
||||
m := p.Modules["build-agent"]
|
||||
return p.WalkSaid == inventory.WalkEnded && p.Outcome() == inventory.WalkDelivered && m != nil && m.State == "built"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// 1173/AC1, 1173/AC3 whole against the store: a first merge's build refused for a word the running build-agent does not
|
||||
// read keeps the merge waiting with no failed build recorded; a delivery plan that does not deliver build-agent asks
|
||||
// nothing; the one that ends delivering it asks the first build again once, and its pass registers the module.
|
||||
func TestAHeldFirstMergeIsAskedAgainWhenABuildAgentIsDelivered(t *testing.T) {
|
||||
const head, commit = "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", "0177ea7a00000000"
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
|
||||
a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": head})
|
||||
wasKept, wasAsk, realAsk := keptDeliveryMerge, routerAskOf, askABuild
|
||||
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||
return a.merge, true, nil
|
||||
}
|
||||
routerAskOf = func(context.Context, catalogueReader, string, string) (*asks.Ask, *asks.Warrant, error) {
|
||||
return &a.ask, &a.warrant, nil
|
||||
}
|
||||
var asked []string
|
||||
askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) {
|
||||
id := "build-" + string(rune('1'+len(asked)))
|
||||
asked = append(asked, id)
|
||||
return id, nil
|
||||
}
|
||||
t.Cleanup(func() { keptDeliveryMerge, routerAskOf, askABuild = wasKept, wasAsk, realAsk })
|
||||
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main", HeadSHA: head, Commit: commit, Number: 7,
|
||||
MergedAt: time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano), Paths: []string{"module.json", "main.go"},
|
||||
ModuleDirs: []string{"."}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(asked) != 1 {
|
||||
t.Fatalf("1173/AC3: the first merge asked %v, want the one first build", asked)
|
||||
}
|
||||
|
||||
// The running build-agent refuses the manifest's word.
|
||||
refused := onTrunk("build-1", "novox/mesh-delivery", "git", "", nil)
|
||||
refused.Failed, refused.Manifest, refused.Commit = refusedKeyTTL, nil, commit
|
||||
if err := (builds{inv: inv, open: open}).Built(ctx, refused); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, recorded, _ := inv.BuildByID(ctx, "build-1"); recorded {
|
||||
t.Fatal("1173/AC1: a refused first build was recorded as a failed build")
|
||||
}
|
||||
waiting, err := inv.WaitingMerges(ctx)
|
||||
if err != nil || len(waiting) != 1 || len(announcedOf(waiting[0]).Held) == 0 {
|
||||
t.Fatalf("1173/AC1: the merges waiting are %+v (%v), want the first merge, held", waiting, err)
|
||||
}
|
||||
if obs := firstBuildObservations(waiting, time.Now()); len(obs) != 1 || obs[0].Kind != kindFirstBuildHeld {
|
||||
t.Fatalf("1173/AC2: the held merge is named by %+v", obs)
|
||||
}
|
||||
|
||||
// A plan that ended without delivering build-agent asks nothing; one that delivered it asks once.
|
||||
other := inventory.Plan{ID: "p1", State: inventory.PlanDone, WalkSaid: inventory.WalkEnded,
|
||||
Modules: map[string]*inventory.PlanModule{"mesh-delivery": {State: "built"}}}
|
||||
if n := askHeldFirstBuildsAgain(ctx, open, other); n != 0 || len(asked) != 1 {
|
||||
t.Fatalf("1173/AC3: a plan that delivered no build-agent asked %d builds again (%v)", n, asked)
|
||||
}
|
||||
delivered := inventory.Plan{ID: "p2", State: inventory.PlanDone, WalkSaid: inventory.WalkEnded,
|
||||
Modules: map[string]*inventory.PlanModule{"build-agent": {State: "built"}}}
|
||||
if n := askHeldFirstBuildsAgain(ctx, open, delivered); n != 1 || len(asked) != 2 {
|
||||
t.Fatalf("1173/AC3: the plan that delivered build-agent asked %d builds again (%v), want one", n, asked)
|
||||
}
|
||||
if n := askHeldFirstBuildsAgain(ctx, open, delivered); n != 0 || len(asked) != 2 {
|
||||
t.Fatalf("1173/AC3: the same end asked again: %d (%v)", n, asked)
|
||||
}
|
||||
|
||||
// Its pass registers the module.
|
||||
pass := onTrunk("build-2", "novox/mesh-delivery", "git", "", map[string]any{"module": "mesh-delivery-watchman", "version": "1"})
|
||||
pass.Commit = commit
|
||||
if err := (builds{inv: inv, open: open}).Built(ctx, pass); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, known := inventoryHolds(t, inv, "mesh-delivery-watchman"); !known {
|
||||
t.Fatal("1173/AC3: the module is not registered after the first build passed")
|
||||
}
|
||||
}
|
||||
|
||||
// aTwoModuleFirstMerge is a first merge into novox/mesh-delivery adding the modules watchman and view, made on the
|
||||
// operator's Approve of its head, whose first builds are asked; asked says the build id asked for each path, in order.
|
||||
func aTwoModuleFirstMerge(t *testing.T, open *stores, commit string) *[][2]string {
|
||||
t.Helper()
|
||||
const head = "b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2"
|
||||
a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@b2b2b2b2b2b2": head})
|
||||
wasKept, wasAsk, realAsk := keptDeliveryMerge, routerAskOf, askABuild
|
||||
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||
return a.merge, true, nil
|
||||
}
|
||||
routerAskOf = func(context.Context, catalogueReader, string, string) (*asks.Ask, *asks.Warrant, error) {
|
||||
return &a.ask, &a.warrant, nil
|
||||
}
|
||||
asked := &[][2]string{}
|
||||
askABuild = func(_ context.Context, _ buildSource, path, _ string) (string, error) {
|
||||
id := fmt.Sprintf("build-%d", len(*asked)+1)
|
||||
*asked = append(*asked, [2]string{path, id})
|
||||
return id, nil
|
||||
}
|
||||
t.Cleanup(func() { keptDeliveryMerge, routerAskOf, askABuild = wasKept, wasAsk, realAsk })
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main", HeadSHA: head, Commit: commit, Number: 7,
|
||||
MergedAt: time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano),
|
||||
Paths: []string{"view/module.json", "view/main.go", "watchman/module.json", "watchman/main.go"},
|
||||
ModuleDirs: []string{"view", "watchman"}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(t.Context(), m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(*asked) != 2 {
|
||||
t.Fatalf("the first merge asked %v, want the two first builds", *asked)
|
||||
}
|
||||
return asked
|
||||
}
|
||||
|
||||
// idOf is the build asked last for path.
|
||||
func idOf(t *testing.T, asked [][2]string, path string) string {
|
||||
t.Helper()
|
||||
for i := len(asked) - 1; i >= 0; i-- {
|
||||
if asked[i][0] == path {
|
||||
return asked[i][1]
|
||||
}
|
||||
}
|
||||
t.Fatalf("no build asked for %s: %v", path, asked)
|
||||
return ""
|
||||
}
|
||||
|
||||
// built hands the controller a build's outcome: refused for the word, or passing as module.
|
||||
func built(t *testing.T, open *stores, id, path, commit, refusedFor, module string) {
|
||||
t.Helper()
|
||||
r := onTrunk(id, "novox/mesh-delivery", "git", path, map[string]any{"module": module, "version": "1"})
|
||||
r.Commit = commit
|
||||
if refusedFor != "" {
|
||||
r.Failed, r.Manifest = `this is not a module manifest: json: unknown field "`+refusedFor+`"`, nil
|
||||
}
|
||||
if err := (builds{inv: open.inventory, open: open}).Built(t.Context(), r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
var buildAgentDelivered = inventory.Plan{ID: "p-ba", State: inventory.PlanDone, WalkSaid: inventory.WalkEnded,
|
||||
Modules: map[string]*inventory.PlanModule{"build-agent": {State: "built"}}}
|
||||
|
||||
// 1173/AC3 (review round 1, item 1): in a first merge of two modules where view passes and watchman is refused, the
|
||||
// held watchman is asked again when build-agent is delivered, though the repository builds view by then.
|
||||
func TestAHeldModuleIsAskedAgainAfterItsSiblingRegistered(t *testing.T) {
|
||||
const commit = "0177ea7b00000000"
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := aTwoModuleFirstMerge(t, open, commit)
|
||||
|
||||
built(t, open, idOf(t, *asked, "view"), "view", commit, "", "mesh-delivery-view")
|
||||
if _, known := inventoryHolds(t, open.inventory, "mesh-delivery-view"); !known {
|
||||
t.Fatal("view is not registered after its first build passed")
|
||||
}
|
||||
refused := idOf(t, *asked, "watchman")
|
||||
built(t, open, refused, "watchman", commit, "key-ttl", "")
|
||||
if _, recorded, _ := open.inventory.BuildByID(ctx, refused); recorded {
|
||||
t.Fatal("1173/AC1: the refused first build of watchman was recorded as a failed build")
|
||||
}
|
||||
waiting, err := open.inventory.WaitingMerges(ctx)
|
||||
if err != nil || len(waiting) != 1 || len(announcedOf(waiting[0]).Held) != 1 {
|
||||
t.Fatalf("1173/AC1: the merges waiting are %+v (%v), want the first merge, holding watchman", waiting, err)
|
||||
}
|
||||
|
||||
if n := askHeldFirstBuildsAgain(ctx, open, buildAgentDelivered); n != 1 || len(*asked) != 3 || (*asked)[2][0] != "watchman" {
|
||||
t.Fatalf("1173/AC3: the delivery of build-agent asked %d merges again (%v), want watchman's first build alone", n, *asked)
|
||||
}
|
||||
built(t, open, idOf(t, *asked, "watchman"), "watchman", commit, "", "mesh-delivery-watchman")
|
||||
if _, known := inventoryHolds(t, open.inventory, "mesh-delivery-watchman"); !known {
|
||||
t.Fatal("1173/AC3: watchman is not registered after the first build asked again passed")
|
||||
}
|
||||
}
|
||||
|
||||
// 1173/AC1, AC2, AC3 (review round 1, item 2): in a first merge of two modules both refused, neither is recorded as a
|
||||
// failed build, the condition names both, and the delivery of build-agent asks both again.
|
||||
func TestTwoRefusedModulesOfOneFirstMergeAreBothHeld(t *testing.T) {
|
||||
const commit = "0177ea7c00000000"
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := aTwoModuleFirstMerge(t, open, commit)
|
||||
|
||||
for _, path := range []string{"watchman", "view"} {
|
||||
id := idOf(t, *asked, path)
|
||||
built(t, open, id, path, commit, "key-ttl", "")
|
||||
if _, recorded, _ := open.inventory.BuildByID(ctx, id); recorded {
|
||||
t.Fatalf("1173/AC1: the refused first build of %s was recorded as a failed build", path)
|
||||
}
|
||||
}
|
||||
waiting, err := open.inventory.WaitingMerges(ctx)
|
||||
if err != nil || len(waiting) != 1 || len(announcedOf(waiting[0]).Held) != 2 {
|
||||
t.Fatalf("1173/AC1: the merges waiting are %+v (%v), want the first merge, holding both", waiting, err)
|
||||
}
|
||||
obs := firstBuildObservations(waiting, time.Now())
|
||||
if len(obs) != 1 || !strings.Contains(obs[0].Summary, "watchman") || !strings.Contains(obs[0].Summary, "view") {
|
||||
t.Fatalf("1173/AC2: the held merge is named by %+v, want one condition naming both", obs)
|
||||
}
|
||||
|
||||
if n := askHeldFirstBuildsAgain(ctx, open, buildAgentDelivered); n != 1 || len(*asked) != 4 {
|
||||
t.Fatalf("1173/AC3: the delivery of build-agent asked %d merges again (%v), want both first builds", n, *asked)
|
||||
}
|
||||
built(t, open, idOf(t, *asked, "watchman"), "watchman", commit, "", "mesh-delivery-watchman")
|
||||
built(t, open, idOf(t, *asked, "view"), "view", commit, "", "mesh-delivery-view")
|
||||
for _, module := range []string{"mesh-delivery-watchman", "mesh-delivery-view"} {
|
||||
if _, known := inventoryHolds(t, open.inventory, module); !known {
|
||||
t.Fatalf("1173/AC3: %s is not registered after the first build asked again passed", module)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
const refusedKeyTTL = `this is not a module manifest: a state is either a name or {name, history, ttl-seconds, per-machine}: ` +
|
||||
`json: unknown field "key-ttl"`
|
||||
|
||||
// aFirstMergeAnswered is a first merge as askFirstBuilds leaves it: answered by the record of merges no plan answers.
|
||||
func aFirstMergeAnswered() (inventory.BuildRequest, inventory.BatchedMerge) {
|
||||
req := inventory.BuildRequest{ID: "build-1", Repository: "novox/mesh-delivery", Seat: "git", Path: "watchman", Ref: "main",
|
||||
Commit: "0177ea7a00", For: "merge"}
|
||||
event, _ := json.Marshal(keptMerge{SourceMoved: link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main",
|
||||
Commit: "0177ea7a00", Number: 3, HeadSHA: "a1a1a1a1a1a1"}, FirstBuild: true})
|
||||
return req, inventory.BatchedMerge{Repository: "novox/mesh-delivery", Branch: "main", Commit: "0177ea7a00",
|
||||
Plan: inventory.WaitingRecord, Event: event}
|
||||
}
|
||||
|
||||
// 1173/AC1: a first build refused for a word its manifest uses and the running build-agent does not know holds its merge.
|
||||
func TestAFirstBuildRefusedForAnUnknownWordHoldsItsMerge(t *testing.T) {
|
||||
req, merge := aFirstMergeAnswered()
|
||||
held := judgeRefusedFirstBuild(link.BuildResult{ID: "build-1", Failed: refusedKeyTTL}, req, merge)
|
||||
if len(held) != 1 || held[0].Word != "key-ttl" || held[0].Module != "watchman" || held[0].Path != "watchman" {
|
||||
t.Fatalf("1173/AC1: a first build refused for \"key-ttl\" is judged %+v, want a hold on that word for watchman", held)
|
||||
}
|
||||
}
|
||||
|
||||
// 1173/AC2: a held first merge is named at once by one condition: the merge, the module, the word and what it waits for.
|
||||
func TestAHeldFirstMergeIsNamedByACondition(t *testing.T) {
|
||||
now := time.Date(2026, 10, 22, 14, 0, 0, 0, time.UTC)
|
||||
_, merge := aFirstMergeAnswered()
|
||||
var k keptMerge
|
||||
_ = json.Unmarshal(merge.Event, &k)
|
||||
k.Held = []heldFirstBuild{{Module: "watchman", Path: "watchman", Word: "key-ttl", At: now},
|
||||
{Module: "view", Path: "view", Word: "per-machine", At: now}}
|
||||
merge.Event, _ = json.Marshal(k)
|
||||
merge.Heard = now.Add(-time.Second)
|
||||
got := firstBuildObservations([]inventory.BatchedMerge{merge}, now)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("1173/AC2: %d conditions for a held first merge, want one: %+v", len(got), got)
|
||||
}
|
||||
o := got[0]
|
||||
if o.Kind != kindFirstBuildHeld || o.Scope != conditions.ScopeMerge {
|
||||
t.Fatalf("1173/AC2: the condition is %+v", o)
|
||||
}
|
||||
for _, want := range []string{"0177ea7a", "watchman", "key-ttl", "view", "per-machine", "build-agent"} {
|
||||
if !strings.Contains(o.Summary, want) {
|
||||
t.Fatalf("1173/AC2: the summary %q does not name %q", o.Summary, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1173/AC4: a first build that fails for any other reason is not held.
|
||||
func TestAFirstBuildFailingForAnyOtherReasonIsNotHeld(t *testing.T) {
|
||||
req, merge := aFirstMergeAnswered()
|
||||
if judgeRefusedFirstBuild(link.BuildResult{Failed: refusedKeyTTL}, req, merge) == nil {
|
||||
t.Fatal("1173/AC4: the control, a refusal for an unknown word, is not held")
|
||||
}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
failed string
|
||||
change func(*inventory.BuildRequest, *inventory.BatchedMerge)
|
||||
}{
|
||||
{what: "a compile error", failed: "main.go:3: undefined: foo"},
|
||||
{what: "no failure", failed: ""},
|
||||
{what: "a missing field", failed: `json: cannot unmarshal string into Go struct field`},
|
||||
{what: "a build a person asked", failed: refusedKeyTTL, change: func(r *inventory.BuildRequest, _ *inventory.BatchedMerge) { r.For = "build" }},
|
||||
{what: "a merge a plan answers", failed: refusedKeyTTL, change: func(_ *inventory.BuildRequest, m *inventory.BatchedMerge) { m.Plan = "plan-7" }},
|
||||
{what: "a waiting merge held for nothing", failed: refusedKeyTTL, change: func(_ *inventory.BuildRequest, m *inventory.BatchedMerge) { m.Plan = "" }},
|
||||
} {
|
||||
r, m := req, merge
|
||||
if c.change != nil {
|
||||
c.change(&r, &m)
|
||||
}
|
||||
if held := judgeRefusedFirstBuild(link.BuildResult{Failed: c.failed}, r, m); held != nil {
|
||||
t.Fatalf("1173/AC4: %s is held: %+v", c.what, held)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1173/AC1, AC2 (review round 1, item 2): a second refusal in a merge already held is held too, its module added to the
|
||||
// hold, so no failed build is recorded and the condition names both.
|
||||
func TestASecondRefusalInAHeldMergeIsAddedToTheHold(t *testing.T) {
|
||||
req, merge := aFirstMergeAnswered()
|
||||
first := judgeRefusedFirstBuild(link.BuildResult{Failed: refusedKeyTTL}, req, merge)
|
||||
k := announcedOf(merge)
|
||||
k.Held = first
|
||||
merge.Event, _ = json.Marshal(k)
|
||||
merge.Plan = "" // HoldMerge put it back to waiting
|
||||
req.ID, req.Path = "build-2", "view"
|
||||
held := judgeRefusedFirstBuild(link.BuildResult{Failed: `json: unknown field "per-machine"`}, req, merge)
|
||||
if len(held) != 2 || held[0].Module != "watchman" || held[1].Module != "view" || held[1].Word != "per-machine" {
|
||||
t.Fatalf("1173/AC1: the second refusal in a held merge is judged %+v, want watchman and view held", held)
|
||||
}
|
||||
// The same module refused again is held once.
|
||||
k.Held = held
|
||||
merge.Event, _ = json.Marshal(k)
|
||||
if again := judgeRefusedFirstBuild(link.BuildResult{Failed: refusedKeyTTL}, req, merge); len(again) != 2 {
|
||||
t.Fatalf("1173/AC1: view refused again is held as %+v, want it once", again)
|
||||
}
|
||||
}
|
||||
|
||||
// 1173/AC3 (review round 1, item 1): a held merge is asked again for its own held modules' directories, not by the rule of
|
||||
// a first merge, which finds nothing once another module of the repository is registered.
|
||||
func TestAHeldMergeIsAskedAgainForItsHeldModulesAlone(t *testing.T) {
|
||||
k := keptMerge{Held: []heldFirstBuild{{Module: "watchman", Path: "watchman"}, {Module: "mesh-delivery", Path: ""},
|
||||
{Module: "watchman", Path: "/watchman/"}}}
|
||||
if got := heldDirsOf(k); strings.Join(got, ",") != "watchman,." {
|
||||
t.Fatalf("1173/AC3: a held merge asks again %v, want watchman and the root once each", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1173/AC3 (the part that needs no store): only a plan that ended delivered with build-agent built in it is the end a held
|
||||
// first build waits for.
|
||||
func TestOnlyADeliveredPlanOfBuildAgentIsWhatAHeldFirstBuildWaitsFor(t *testing.T) {
|
||||
plan := func(state, walk string, modules map[string]*inventory.PlanModule) inventory.Plan {
|
||||
return inventory.Plan{ID: "p", State: state, WalkSaid: walk, Modules: modules}
|
||||
}
|
||||
built := map[string]*inventory.PlanModule{"build-agent": {State: "built"}}
|
||||
if !deliversBuildAgent(plan(inventory.PlanDone, inventory.WalkEnded, built)) {
|
||||
t.Fatal("1173/AC3: a plan that delivered build-agent is not the end a held first build waits for")
|
||||
}
|
||||
for what, p := range map[string]inventory.Plan{
|
||||
"a plan still running": plan(inventory.PlanBuilding, "", built),
|
||||
"a plan that failed": plan(inventory.PlanFailed, inventory.WalkEnded, built),
|
||||
"a failed build-agent": plan(inventory.PlanDone, inventory.WalkEnded, map[string]*inventory.PlanModule{"build-agent": {State: "failed"}}),
|
||||
"another module's plan": plan(inventory.PlanDone, inventory.WalkEnded, map[string]*inventory.PlanModule{"mailu": {State: "built"}}),
|
||||
} {
|
||||
if deliversBuildAgent(p) {
|
||||
t.Fatalf("1173/AC3: %s is the end a held first build waits for", what)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -35,7 +35,7 @@ func TestAMergeSaysTheModulesWhoseBuildSourceItLeftUnchanged(t *testing.T) {
|
||||
{Own: true, Paths: []string{"modules/build-agent/Dockerfile", "modules/build-agent/module.json"}},
|
||||
},
|
||||
"route-proxy": {
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"go.mod", "examples/route-proxy/"}},
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"go.mod", "modules/route-proxy/cmd/route-proxy/"}},
|
||||
{Own: true, Paths: []string{"modules/route-proxy/Dockerfile", "modules/route-proxy/module.json"}},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -408,6 +408,13 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
||||
fmt.Printf("%s: a dry run of %s on %s, not taken in\n", result.ID, result.Repository, result.Ref)
|
||||
return nil
|
||||
}
|
||||
// **A first build refused for a word the running build-agent does not read waits for one that does** (novox/hq ADR
|
||||
// 0482 §5, issue 1173): its merge is kept, named, and never failed; no failed build is recorded.
|
||||
if held, err := holdRefusedFirstBuild(ctx, b.inv, result); err != nil {
|
||||
return err
|
||||
} else if held {
|
||||
return nil
|
||||
}
|
||||
manifest, _, err := takeIn(ctx, b.inv, result)
|
||||
// The assignments pending on this build, made or ended (novox/hq issue 325), said in the daemon's log
|
||||
// after what became of the build.
|
||||
|
||||
@@ -65,6 +65,9 @@ type mergeCheckInput struct {
|
||||
// group are the other heads of a delivery group composed with this one (novox/hq ADR 0239), each laid
|
||||
// over the mesh after the one before: the group judged as one future state.
|
||||
group []groupChange
|
||||
// held are the approved config values held for the keys the pull requests introduce (novox/hq ADR 0502 clause 4),
|
||||
// laid over the composition with the change only.
|
||||
held []heldValue
|
||||
// admin is a PostgreSQL the gate may create and drop databases in.
|
||||
admin string
|
||||
// beside is the directory holding the core clones beside the check (MESH_CHECK_BESIDE), for finding moves.
|
||||
@@ -189,6 +192,8 @@ func mergeGateCommand(ctx context.Context, args []string) error {
|
||||
"--changed, for a list too long for an argument (novox/hq issue 1247)")
|
||||
group := set.String("group", "", "a delivery group's other heads, as JSON [{repository, tree, changed}], composed "+
|
||||
"with this one as one future state (novox/hq ADR 0239)")
|
||||
heldFile := set.String("held", "", "a file of the config values approved for keys the change's pull requests introduce, as JSON "+
|
||||
"[{pull, node, module, key, value, approved_by}], laid over the composition with the change (novox/hq ADR 0502)")
|
||||
asJSON := set.Bool("json", false, "print the verdict as JSON")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
@@ -208,6 +213,10 @@ func mergeGateCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
held, err := readHeld(*heldFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out := io.Writer(os.Stdout)
|
||||
stdout := os.Stdout
|
||||
if *asJSON {
|
||||
@@ -223,7 +232,7 @@ func mergeGateCommand(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
v, err := judgeChange(ctx, mergeCheckInput{facts: f, repository: *repository, tree: *tree, changed: changedList,
|
||||
group: others, admin: *admin, say: out, beside: os.Getenv("MESH_CHECK_BESIDE")})
|
||||
group: others, held: held, admin: *admin, say: out, beside: os.Getenv("MESH_CHECK_BESIDE")})
|
||||
os.Stdout = stdout
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -387,7 +396,16 @@ func judgeChange(ctx context.Context, in mergeCheckInput) (mergeVerdict, error)
|
||||
}
|
||||
}
|
||||
sort.Strings(added)
|
||||
after, tried, notes, err := composeEveryMachine(ctx, in, change, changeSources, added)
|
||||
// **The held values are laid in this composition only** (novox/hq ADR 0502 clause 4): the mesh as it is does not
|
||||
// declare their keys, so the base composition would refuse them.
|
||||
withHeld := in
|
||||
var heldNotes []string
|
||||
withHeld.facts, heldNotes = layHeld(in.facts, in.held, change)
|
||||
for _, n := range heldNotes {
|
||||
say("%s", n)
|
||||
}
|
||||
v.Notes = append(v.Notes, heldNotes...)
|
||||
after, tried, notes, err := composeEveryMachine(ctx, withHeld, change, changeSources, added)
|
||||
if err != nil {
|
||||
return v, fmt.Errorf("the mesh with the change could not be raised: %w", err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
snapshot "git.novox.be/novox/mesh-sdk/go/facts"
|
||||
)
|
||||
|
||||
// Config values held for the keys an open pull request introduces (novox/hq ADR 0502 clause 4, issue 1826): the gate
|
||||
// lays each approved one over its module's values on its node in the composition with the change, and in that one
|
||||
// only. In the mesh as it is the key is not declared, so the value would be refused there.
|
||||
|
||||
// heldValue is one approved held value as the build agent writes it (--held): the value of Key for Module on Node,
|
||||
// a machine's real name, which the snapshot holds only as a pseudonym.
|
||||
type heldValue struct {
|
||||
Pull string `json:"pull"`
|
||||
Head string `json:"head,omitempty"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
Approved string `json:"approved,omitempty"`
|
||||
By string `json:"by,omitempty"`
|
||||
At string `json:"at,omitempty"`
|
||||
}
|
||||
|
||||
// readHeld reads the file --held names; none named, none held.
|
||||
func readHeld(path string) ([]heldValue, error) {
|
||||
if path == "" {
|
||||
return nil, nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var held []heldValue
|
||||
if err := json.Unmarshal(raw, &held); err != nil {
|
||||
return nil, fmt.Errorf("the held values are not readable: %w", err)
|
||||
}
|
||||
for _, h := range held {
|
||||
if h.Node == "" || h.Module == "" || h.Key == "" || len(h.Value) == 0 {
|
||||
return nil, fmt.Errorf("a held value of %s names no node, module, key or value", h.Pull)
|
||||
}
|
||||
}
|
||||
return held, nil
|
||||
}
|
||||
|
||||
// layHeld is the snapshot with each held value laid as the value of its key in its node's layer of its module, and what
|
||||
// it said of them. The snapshot given is not changed. **A held value for a machine the snapshot does not hold, or a
|
||||
// module the change does not define, is said and not laid**, never dropped silently; one for a module not assigned to
|
||||
// its machine is laid and said. The notes name machines as the snapshot does (roles and pseudonym), not by their names.
|
||||
func layHeld(f snapshot.Facts, held []heldValue, shelf map[string]catalogue.Manifest) (snapshot.Facts, []string) {
|
||||
if len(held) == 0 {
|
||||
return f, nil
|
||||
}
|
||||
out := f
|
||||
out.Machines = make([]snapshot.Machine, len(f.Machines))
|
||||
copy(out.Machines, f.Machines)
|
||||
var notes, laid []string
|
||||
pulls := map[string]bool{}
|
||||
for _, h := range held {
|
||||
at := -1
|
||||
pseudonym := snapshot.Pseudonym("machine", h.Node)
|
||||
for i, m := range out.Machines {
|
||||
if m.Name == pseudonym {
|
||||
at = i
|
||||
}
|
||||
}
|
||||
if at < 0 {
|
||||
notes = append(notes, fmt.Sprintf("the value held for %s of %s (%s=%s) is not composed: the snapshot holds no machine %s",
|
||||
h.Module, h.Pull, h.Key, h.Value, pseudonym))
|
||||
continue
|
||||
}
|
||||
if _, defined := shelf[h.Module]; !defined {
|
||||
notes = append(notes, fmt.Sprintf("the value held for %s of %s (%s=%s) is not composed: the change defines no module %s",
|
||||
h.Module, h.Pull, h.Key, h.Value, h.Module))
|
||||
continue
|
||||
}
|
||||
var value any
|
||||
if err := json.Unmarshal(h.Value, &value); err != nil {
|
||||
notes = append(notes, fmt.Sprintf("the value held for %s of %s (%s) is not composed: it is not a value: %v",
|
||||
h.Module, h.Pull, h.Key, err))
|
||||
continue
|
||||
}
|
||||
m := &out.Machines[at]
|
||||
layers := make([]snapshot.Settings, len(m.Settings))
|
||||
copy(layers, m.Settings)
|
||||
found := false
|
||||
for i := range layers {
|
||||
if layers[i].Module == h.Module {
|
||||
values := make(map[string]any, len(layers[i].Values)+1)
|
||||
for k, v := range layers[i].Values {
|
||||
values[k] = v
|
||||
}
|
||||
values[h.Key] = value
|
||||
layers[i].Values = values
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
layers = append(layers, snapshot.Settings{Module: h.Module, Values: map[string]any{h.Key: value}})
|
||||
sort.Slice(layers, func(a, b int) bool { return layers[a].Module < layers[b].Module })
|
||||
}
|
||||
m.Settings = layers
|
||||
assigned := false
|
||||
for _, a := range m.Assigned {
|
||||
assigned = assigned || a == h.Module
|
||||
}
|
||||
by := ""
|
||||
if h.Approved != "" {
|
||||
by = ", approved by " + h.Approved
|
||||
}
|
||||
line := fmt.Sprintf("%s on %s %s=%s%s", h.Module, m.Described(), h.Key, h.Value, by)
|
||||
if !assigned {
|
||||
line += " (not assigned there in the snapshot)"
|
||||
}
|
||||
laid = append(laid, line)
|
||||
pulls[h.Pull] = true
|
||||
}
|
||||
if len(laid) > 0 {
|
||||
var names []string
|
||||
for p := range pulls {
|
||||
names = append(names, p)
|
||||
}
|
||||
sort.Strings(names)
|
||||
notes = append([]string{fmt.Sprintf("composed with %d value(s) held for %s: %s", len(laid), strings.Join(names, ", "),
|
||||
strings.Join(laid, "; "))}, notes...)
|
||||
}
|
||||
return out, notes
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
snapshot "git.novox.be/novox/mesh-sdk/go/facts"
|
||||
)
|
||||
|
||||
func heldGateFacts() snapshot.Facts {
|
||||
return snapshot.Facts{Machines: []snapshot.Machine{
|
||||
{Name: snapshot.Pseudonym("machine", "ace"), Roles: []string{"the build node"}, Assigned: []string{"n8n"},
|
||||
Settings: []snapshot.Settings{{Module: "n8n", Values: map[string]any{"port": 5678.0}}}},
|
||||
{Name: snapshot.Pseudonym("machine", "g14")},
|
||||
}}
|
||||
}
|
||||
|
||||
var heldShelf = map[string]catalogue.Manifest{"n8n": {Module: "n8n"}}
|
||||
|
||||
// 1826/AC3: a held value is laid in its node's layer of its module, beside what the layer holds, and said.
|
||||
func TestAHeldValueIsLaidOverItsNodesLayerAndSaid(t *testing.T) {
|
||||
f := heldGateFacts()
|
||||
got, notes := layHeld(f, []heldValue{{Pull: "novox/mesh-catalog#434", Node: "ace", Module: "n8n", Key: "media-gid",
|
||||
Value: []byte("2000"), Approved: "the operator"}}, heldShelf)
|
||||
layer := got.Machines[0].Settings[0].Values
|
||||
if layer["media-gid"] != 2000.0 || layer["port"] != 5678.0 {
|
||||
t.Fatalf("the layer holds %v", layer)
|
||||
}
|
||||
if _, has := f.Machines[0].Settings[0].Values["media-gid"]; has {
|
||||
t.Fatal("the snapshot given was changed")
|
||||
}
|
||||
if len(notes) != 1 || !strings.Contains(notes[0], "composed with 1 value(s) held for novox/mesh-catalog#434") ||
|
||||
!strings.Contains(notes[0], "media-gid=2000") || !strings.Contains(notes[0], "approved by the operator") ||
|
||||
strings.Contains(notes[0], "ace") {
|
||||
t.Fatalf("said %q", notes)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC3: a machine with no layer of the module gets one.
|
||||
func TestAHeldValueMakesTheLayerAMachineLacks(t *testing.T) {
|
||||
got, _ := layHeld(heldGateFacts(), []heldValue{{Pull: "p", Node: "g14", Module: "n8n", Key: "k", Value: []byte(`"x"`)}}, heldShelf)
|
||||
s := got.Machines[1].Settings
|
||||
if len(s) != 1 || s[0].Module != "n8n" || s[0].Values["k"] != "x" {
|
||||
t.Fatalf("layers %+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC3: a held value for a machine or a module the snapshot or the change does not hold is said, never laid.
|
||||
func TestAHeldValueForWhatIsNotHeldIsSaidNotLaid(t *testing.T) {
|
||||
f := heldGateFacts()
|
||||
got, notes := layHeld(f, []heldValue{
|
||||
{Pull: "p", Node: "nowhere", Module: "n8n", Key: "k", Value: []byte("1")},
|
||||
{Pull: "p", Node: "ace", Module: "ghost", Key: "k", Value: []byte("1")},
|
||||
}, heldShelf)
|
||||
if len(notes) != 2 || !strings.Contains(notes[0], "holds no machine") || !strings.Contains(notes[1], "defines no module ghost") {
|
||||
t.Fatalf("said %q", notes)
|
||||
}
|
||||
if _, has := got.Machines[0].Settings[0].Values["k"]; has {
|
||||
t.Fatal("laid")
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC3: none held lays nothing and says nothing; the file is read and refuses a value without a node.
|
||||
func TestNoHeldValuesLayNothingAndTheFileIsRead(t *testing.T) {
|
||||
if got, notes := layHeld(heldGateFacts(), nil, heldShelf); len(notes) != 0 || len(got.Machines) != 2 {
|
||||
t.Fatalf("%v %v", got, notes)
|
||||
}
|
||||
dir := t.TempDir()
|
||||
good := filepath.Join(dir, "good.json")
|
||||
os.WriteFile(good, []byte(`[{"pull":"p","node":"ace","module":"n8n","key":"k","value":1}]`), 0o644)
|
||||
if h, err := readHeld(good); err != nil || len(h) != 1 || h[0].Key != "k" {
|
||||
t.Fatalf("%v %v", h, err)
|
||||
}
|
||||
bad := filepath.Join(dir, "bad.json")
|
||||
os.WriteFile(bad, []byte(`[{"pull":"p","module":"n8n","key":"k","value":1}]`), 0o644)
|
||||
if _, err := readHeld(bad); err == nil {
|
||||
t.Fatal("a value without a node was read")
|
||||
}
|
||||
if h, err := readHeld(""); h != nil || err != nil {
|
||||
t.Fatalf("%v %v", h, err)
|
||||
}
|
||||
}
|
||||
@@ -526,3 +526,57 @@ func TestAMemberJudgedAloneSeesAMoveThroughTheClonesBeside(t *testing.T) {
|
||||
t.Errorf("nothing beside, yet moves %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC3 (integration level, a throwaway store): a change whose container says ${config:media-gid} fails the gate for
|
||||
// want of the value, passes it with the value the operator approved held for its pull request, and the mesh as it is
|
||||
// composes the same either way.
|
||||
func TestAChangeNeedingAHeldValueFailsWithoutItAndPassesWithIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
trunk := `{"module":"n8n","version":"1","resources":[{"id":"server","type":"container","name":"n8n",
|
||||
"image":"n8nio/n8n@sha256:4846eb2f4b874ab04cde7fc1e249d2ddaec66e9aea64439beb2972cfea88e3c0"}]}`
|
||||
m, err := catalogue.ParseManifest([]byte(trunk))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog",
|
||||
Path: "modules/n8n", BuiltFrom: "c0ffee"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "n8n"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := gatherFacts(ctx, open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := range f.Machines {
|
||||
f.Machines[i].Capabilities = append(f.Machines[i].Capabilities, snapshot.Capability{Name: "container-groups", Present: true})
|
||||
}
|
||||
tree := aTree(t, map[string]string{"n8n": n8nWithAGroup})
|
||||
judge := func(held []heldValue) mergeVerdict {
|
||||
v, err := judgeChange(t.Context(), mergeCheckInput{facts: f, admin: os.Getenv("MESH_TEST_POSTGRES"),
|
||||
repository: "novox/mesh-catalog", tree: tree, changed: []string{"modules/n8n/module.json"}, held: held})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
without := judge(nil)
|
||||
if without.Verdict != "fail" || !strings.Contains(without.Report(), "media-gid") {
|
||||
t.Fatalf("a required value nothing sets passed, or failed for another reason:\n%s", without.Report())
|
||||
}
|
||||
with := judge([]heldValue{{Pull: "novox/mesh-catalog#434", Node: "laptop", Module: "n8n", Key: "media-gid",
|
||||
Value: []byte("2000"), Approved: "the operator"}})
|
||||
if with.Verdict == "fail" || with.Verdict == "error" {
|
||||
t.Fatalf("the held value did not make it compose:\n%s", with.Report())
|
||||
}
|
||||
if !strings.Contains(with.Report(), "composed with 1 value(s) held for novox/mesh-catalog#434") {
|
||||
t.Fatalf("the value laid is not said:\n%s", with.Report())
|
||||
}
|
||||
for i := range with.Machines {
|
||||
if with.Machines[i].Base.Composes != without.Machines[i].Base.Composes {
|
||||
t.Fatalf("the mesh as it is composes differently with a held value: %+v / %+v", with.Machines[i].Base, without.Machines[i].Base)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0505 §5: a delivery whose cure is its author's new head reaches the operator in words that name its
|
||||
// author and say a new head is needed, offer no Stop and never call it the operator's; the line as mesh-delivery-gate
|
||||
// says it (ready.go readyLines: needs "new-head", author, branch).
|
||||
func TestADeliveryThatNeedsANewHeadNamesItsAuthorAndOffersNoStop(t *testing.T) {
|
||||
l := stalledLine{ID: "novox/mesh-controller@499ff5c21861", Number: 432, State: "rejected", For: "2h1m0s", Bound: "2h0m0s",
|
||||
H2: "none: a new head from its author, or a recheck that passes", Needs: needsNewHead,
|
||||
Author: "the author of feat/quiet-deliveries (quiet-deliveries on shanks)", Branch: "feat/quiet-deliveries",
|
||||
Says: "novox/mesh-controller#432 passed its checks at 499ff5c21861 and is not merged: a conflict"}
|
||||
o := stalledObservations([]stalledLine{l})[0]
|
||||
for _, words := range []string{o.Headline, o.Explanation, o.Resolved, o.Needs} {
|
||||
if why, ok := conditions.Plain(words); !ok {
|
||||
t.Errorf("%q is not plain: %s", words, why)
|
||||
}
|
||||
for _, never := range []string{"stop it", "operator's"} {
|
||||
if strings.Contains(words, never) {
|
||||
t.Errorf("%q says %q", words, never)
|
||||
}
|
||||
}
|
||||
}
|
||||
if o.Headline != "Pull request 432 of mesh-controller needs a new head from its author" ||
|
||||
!strings.Contains(o.Explanation, "until the author of feat/quiet-deliveries (quiet-deliveries on shanks) pushes a new head") ||
|
||||
!strings.Contains(o.Explanation, "for 2 hours") || len(o.Actions) != 0 ||
|
||||
o.Needs != "ask the author of feat/quiet-deliveries (quiet-deliveries on shanks) for a new head." {
|
||||
t.Fatalf("words: %q / %q / %q / %v", o.Headline, o.Explanation, o.Needs, o.Actions)
|
||||
}
|
||||
// A branch in code is not said; the author is then "its author".
|
||||
l.Author = "the author of fix/a1b2c3d4e5f6"
|
||||
if o := stalledObservations([]stalledLine{l})[0]; !strings.Contains(o.Needs, "ask its author") {
|
||||
t.Fatalf("a dotted branch was said: %q", o.Needs)
|
||||
}
|
||||
}
|
||||
@@ -966,6 +966,9 @@ func stalledWords(l stalledLine, o conditions.Observation) (headline, explanatio
|
||||
if l.State == "unanswered" {
|
||||
return unansweredWords(l, o)
|
||||
}
|
||||
if l.Needs == needsNewHead {
|
||||
return newHeadWords(l, name)
|
||||
}
|
||||
held := l.State
|
||||
if held == "" {
|
||||
held = "held"
|
||||
@@ -999,6 +1002,29 @@ func stalledWords(l stalledLine, o conditions.Observation) (headline, explanatio
|
||||
fmt.Sprintf("Delivery of %s is no longer %s", name, held), needs, actions
|
||||
}
|
||||
|
||||
// newHeadWords are the words of a delivery whose cure is its author's new head (novox/hq ADR 0505 §5): it names the
|
||||
// author and says a new head is needed; it offers no Stop and never calls it the operator's, because the operator
|
||||
// hears of it only once the author's own watch has failed, after two hours with no new head.
|
||||
func newHeadWords(l stalledLine, name string) (headline, explanation, resolved, needs string, actions []conditions.Action) {
|
||||
pull := "A pull request of " + name
|
||||
if l.Number > 0 {
|
||||
pull = fmt.Sprintf("Pull request %d of %s", l.Number, name)
|
||||
}
|
||||
author := l.Author
|
||||
if _, plain := conditions.Plain(author); !plain {
|
||||
author = "its author" // a branch named in code (a dotted name, a number) is not said
|
||||
}
|
||||
long := "for a long time"
|
||||
if d, err := time.ParseDuration(l.For); err == nil {
|
||||
long = "for " + humanDuration(d)
|
||||
}
|
||||
return pull + " needs a new head from its author",
|
||||
fmt.Sprintf("%s cannot merge until %s pushes a new head: its check failed after its Approve, and no new head "+
|
||||
"came %s.", pull, author, long),
|
||||
pull + " has a new head, or is closed",
|
||||
"ask " + author + " for a new head.", nil
|
||||
}
|
||||
|
||||
// stoppedNeeds is what ends a delivery that is already stopped, from the stalled line's healer words ("none: a
|
||||
// merge that delivers its modules ends it — the operator's"): the part that says what ends it.
|
||||
func stoppedNeeds(l stalledLine) string {
|
||||
|
||||
@@ -192,7 +192,7 @@ func startRolloutEvents(ctx context.Context, open *stores, js *broker.JetStream)
|
||||
if err := startProvAppliedLooks(ctx, js); err != nil {
|
||||
fmt.Printf("plans: the provisioner clients' words could not be watched (%v); a walk waits for them until the next look\n", err)
|
||||
}
|
||||
if err := startPlanMovedLooks(ctx, js); err != nil {
|
||||
if err := startPlanMovedLooks(ctx, js, open); err != nil {
|
||||
fmt.Printf("plans: the plans moved by other processes could not be followed (%v)\n", err)
|
||||
}
|
||||
done := startPlanLooks(ctx, open)
|
||||
@@ -206,8 +206,13 @@ func servingPlanSaved(ctx context.Context, bus link.Bus, open *stores) func(*inv
|
||||
return func(_ *inventory.Inventory, p inventory.Plan) {
|
||||
sayPlanMoved(ctx, bus, open.inventory, p)
|
||||
planKept(p)
|
||||
keepBuildingKey(ctx, p)
|
||||
if p.WalkSaid == inventory.WalkEnded {
|
||||
go forgetChecksOfWalk(context.WithoutCancel(ctx), open.inventory, p)
|
||||
// A first build refused for a word only a newer build-agent reads is asked again (novox/hq issue 1173).
|
||||
if deliversBuildAgent(p) {
|
||||
go askHeldFirstBuildsAgain(context.WithoutCancel(ctx), open, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -562,7 +567,7 @@ func startProvAppliedLooks(ctx context.Context, js *broker.JetStream) error {
|
||||
// `plan-moved`, so a plan a person moved with a verb at the control node's terminal — a process this one did not spawn,
|
||||
// whose saves it does not hear — is looked at all the same (novox/hq ADR 0406 §2, §5, issue 974). New messages only; the
|
||||
// first look reads the rest. A look its own save asks is the same look: asks are one.
|
||||
func startPlanMovedLooks(ctx context.Context, js *broker.JetStream) error {
|
||||
func startPlanMovedLooks(ctx context.Context, js *broker.JetStream, open *stores) error {
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -577,7 +582,18 @@ func startPlanMovedLooks(ctx context.Context, js *broker.JetStream) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("following the plans moved: %w", err)
|
||||
}
|
||||
consuming, err := consumer.Consume(func(jetstream.Msg) { askToLook() })
|
||||
var bus link.Bus
|
||||
if core, err := js.Conn().JetStream(); err == nil {
|
||||
bus = link.OverNATS{Conn: js.Conn(), JS: core}
|
||||
}
|
||||
consuming, err := consumer.Consume(func(m jetstream.Msg) {
|
||||
askToLook()
|
||||
if bus != nil && open != nil {
|
||||
// A walk's start or end, said by this controller or a verb: said again with the machines whose
|
||||
// declaration it changes (novox/hq ADR 0505 §2, walk_declared.go).
|
||||
go sayDeclared(context.WithoutCancel(ctx), bus, open, m.Data())
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("following the plans moved: %w", err)
|
||||
}
|
||||
|
||||
@@ -174,7 +174,7 @@ func TestAPlanMovedByAnotherProcessAsksForALook(t *testing.T) {
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
go l.run(ctx)
|
||||
if err := startPlanMovedLooks(ctx, js); err != nil {
|
||||
if err := startPlanMovedLooks(ctx, js, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
legacy, err := js.Conn().JetStream()
|
||||
|
||||
@@ -52,7 +52,7 @@ func TestASharedRepositoryIsPlannedFromTheRecordedBuildSources(t *testing.T) {
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: append([]string{"cmd/mesh-builder/", "internal/broker/"}, gomod...)}}
|
||||
sourcesOf["route-proxy"] = []inventory.BuildSource{
|
||||
{Paths: []string{"modules/route-proxy/Dockerfile", "modules/route-proxy/module.json"}},
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: append([]string{"examples/route-proxy/", "internal/broker/"}, gomod...)}}
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: append([]string{"modules/route-proxy/cmd/route-proxy/", "internal/broker/"}, gomod...)}}
|
||||
register(catalogue.Manifest{Module: "mesh-controller", Version: "1"}, "novox/mesh-controller", "", nil, nil)
|
||||
register(agent, "novox/mesh-catalog", "modules/build-agent", nil, controllerRead)
|
||||
register(catalogue.Manifest{Module: "route-proxy", Version: "1"}, "novox/mesh-catalog", "modules/route-proxy", nil, controllerRead)
|
||||
@@ -133,7 +133,7 @@ func TestASharedRepositoryIsPlannedFromTheRecordedBuildSources(t *testing.T) {
|
||||
{"a package only the controller builds from: the controller alone", "mesh-controller",
|
||||
[]string{"internal/conditions/condition.go"}, "mesh-controller", true},
|
||||
{"the route proxy's program: the route proxy alone", "mesh-controller",
|
||||
[]string{"examples/route-proxy/main.go"}, "route-proxy", true},
|
||||
[]string{"modules/route-proxy/cmd/route-proxy/main.go"}, "route-proxy", true},
|
||||
{"a package all three build from: all three", "mesh-controller",
|
||||
[]string{"internal/broker/broker.go"}, "mesh-controller,route-proxy | build-agent", false},
|
||||
{"the route proxy's directory in the catalogue: it alone", "mesh-catalog",
|
||||
|
||||
@@ -259,7 +259,7 @@ func TestASharedRepositoryMovesOnlyWhatItsBuildSourceHolds(t *testing.T) {
|
||||
{Own: true, Paths: []string{"modules/build-agent/Dockerfile", "modules/build-agent/module.json"}},
|
||||
},
|
||||
"route-proxy": {
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: with("examples/route-proxy/", "internal/broker/",
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: with("modules/route-proxy/cmd/route-proxy/", "internal/broker/",
|
||||
"vendor/github.com/nats-io/nats.go/")},
|
||||
{Own: true, Paths: []string{"modules/route-proxy/Dockerfile", "modules/route-proxy/module.json"}},
|
||||
},
|
||||
@@ -288,7 +288,7 @@ func TestASharedRepositoryMovesOnlyWhatItsBuildSourceHolds(t *testing.T) {
|
||||
{"a README of the controller's repository: no module (338)", "mesh-controller",
|
||||
[]string{"README.md"}, read, "", "README.md"},
|
||||
{"the route proxy's program alone: the route proxy alone (338)", "mesh-controller",
|
||||
[]string{"examples/route-proxy/main.go"}, read, "route-proxy", ""},
|
||||
[]string{"modules/route-proxy/cmd/route-proxy/main.go"}, read, "route-proxy", ""},
|
||||
{"the build seat's program alone: its holder alone", "mesh-controller",
|
||||
[]string{"cmd/mesh-builder/main.go"}, read, "build-agent", ""},
|
||||
{"a package the build seat's program and the controller build from: both", "mesh-controller",
|
||||
@@ -305,7 +305,7 @@ func TestASharedRepositoryMovesOnlyWhatItsBuildSourceHolds(t *testing.T) {
|
||||
{"go.sum: all three", "mesh-controller",
|
||||
[]string{"go.sum"}, read, "mesh-controller,route-proxy | build-agent", ""},
|
||||
{"a file added to the proxy's package: the proxy", "mesh-controller",
|
||||
[]string{"examples/route-proxy/new.go"}, read, "route-proxy", ""},
|
||||
[]string{"modules/route-proxy/cmd/route-proxy/new.go"}, read, "route-proxy", ""},
|
||||
// Before any build source is said: as before.
|
||||
{"no build source said: a README moves all three, as before", "mesh-controller",
|
||||
[]string{"README.md"}, unsaid, "mesh-controller,route-proxy | build-agent", ""},
|
||||
@@ -345,7 +345,7 @@ func TestAModuleAPlanOvertookIsReadWhole(t *testing.T) {
|
||||
built := time.Date(2026, 10, 10, 1, 0, 0, 0, time.UTC)
|
||||
read := map[string][]inventory.ReadRepository{
|
||||
"route-proxy": {
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"examples/route-proxy/", "go.mod"}, Built: built},
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"modules/route-proxy/cmd/route-proxy/", "go.mod"}, Built: built},
|
||||
{Own: true, Paths: []string{"modules/route-proxy/module.json"}, Built: built},
|
||||
},
|
||||
"mesh-controller": {{Own: true, Paths: []string{"module.json", "cmd/mesh-controller/"}, Built: built}},
|
||||
@@ -396,11 +396,11 @@ func TestAMissedMergeMovingOnlyAPackagingModuleIsActedOnOnce(t *testing.T) {
|
||||
}
|
||||
read := map[string][]inventory.ReadRepository{
|
||||
"mesh-controller": {{Own: true, Paths: []string{"module.json", "cmd/mesh-controller/"}, Built: merged.Add(-time.Hour)}},
|
||||
"route-proxy": {{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"examples/route-proxy/"},
|
||||
"route-proxy": {{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"modules/route-proxy/cmd/route-proxy/"},
|
||||
Built: merged.Add(-time.Hour), Looked: merged.Add(-time.Hour)}},
|
||||
}
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main", Commit: "c1",
|
||||
MergedAt: merged.Format(time.RFC3339), Paths: []string{"examples/route-proxy/main.go"}}
|
||||
MergedAt: merged.Format(time.RFC3339), Paths: []string{"modules/route-proxy/cmd/route-proxy/main.go"}}
|
||||
if got := wouldMove(m, entries, planningView(read, nil)); len(got) != 1 || got[0].Manifest.Module != "route-proxy" {
|
||||
t.Fatalf("a missed merge of the proxy's program would move %v", got)
|
||||
}
|
||||
@@ -599,14 +599,14 @@ func TestTheGatePlansFromTheBuildSourcesTheSnapshotCarries(t *testing.T) {
|
||||
Sources: []snapshot.BuildSource{{Own: true, Paths: []string{"module.json", "cmd/mesh-controller/", "internal/broker/"}}}},
|
||||
{Name: "route-proxy", Repository: "novox/mesh-catalog", Path: "modules/route-proxy", Manifest: manifest("route-proxy"),
|
||||
Reads: []string{"novox/mesh-controller"},
|
||||
Sources: []snapshot.BuildSource{{Repository: "novox/mesh-controller", Paths: []string{"examples/route-proxy/", "internal/broker/"}},
|
||||
Sources: []snapshot.BuildSource{{Repository: "novox/mesh-controller", Paths: []string{"modules/route-proxy/cmd/route-proxy/", "internal/broker/"}},
|
||||
{Own: true, Paths: []string{"modules/route-proxy/module.json"}}}},
|
||||
}}
|
||||
for paths, want := range map[string]string{
|
||||
"cmd/mesh-controller/main.go": "mesh-controller",
|
||||
"examples/route-proxy/main.go": "route-proxy",
|
||||
"internal/broker/broker.go": "mesh-controller,route-proxy",
|
||||
"README.md": "",
|
||||
"cmd/mesh-controller/main.go": "mesh-controller",
|
||||
"modules/route-proxy/cmd/route-proxy/main.go": "route-proxy",
|
||||
"internal/broker/broker.go": "mesh-controller,route-proxy",
|
||||
"README.md": "",
|
||||
} {
|
||||
r, err := reachOfChange(facts, "novox/mesh-controller", []string{paths}, "")
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The plans building (novox/hq ADR 0483 clause 2, issue 1687).
|
||||
//
|
||||
// **A key per delivery plan in its build phase, so that the build seat takes no new check while a delivery builds.** The phase
|
||||
// runs from the ask of the plan's first build until every build of its last tier is built, or the plan fails, stops or waits
|
||||
// on the operator. It is derived from the plan alone (buildPhaseOf), so a controller that restarts finds the same answer. The
|
||||
// key is put when the first build is asked (sayBuilding), kept current at every save of the plan (keepBuildingKey) and by a
|
||||
// look every link.PlansBuildingRefresh while this controller holds the lease (keepingPlansBuilding), and deleted at the phase's
|
||||
// end; the bucket's age removes the key of a controller that stopped, so checks are held at most that long.
|
||||
//
|
||||
// The layout is documented in internal/link/holds.go.
|
||||
|
||||
// buildPhaseOf says whether a plan is in its build phase: open, not waiting for the operator's word, with a build asked and a
|
||||
// build of its tiers still to be built. A tier built and sent leaves the plan rolling while later tiers are still to be asked;
|
||||
// it is in the phase still, which is what keeps a check from taking a place between two tiers.
|
||||
func buildPhaseOf(p inventory.Plan) bool {
|
||||
if !p.Open() || p.Waiting() || p.Release != nil {
|
||||
return false
|
||||
}
|
||||
asked, unbuilt := false, false
|
||||
for _, tier := range p.Tiers {
|
||||
for _, m := range tier {
|
||||
st := p.Modules[m]
|
||||
if st != nil && st.State != "" {
|
||||
asked = true
|
||||
}
|
||||
if st == nil || (st.State != "built" && st.State != planDeleted) {
|
||||
unbuilt = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return asked && unbuilt
|
||||
}
|
||||
|
||||
// buildingSince is when the plan's first build was asked, or fallback when no module says.
|
||||
func buildingSince(p inventory.Plan, fallback time.Time) time.Time {
|
||||
since := time.Time{}
|
||||
for _, st := range p.Modules {
|
||||
if st != nil && st.AskedAt != nil && (since.IsZero() || st.AskedAt.Before(since)) {
|
||||
since = *st.AskedAt
|
||||
}
|
||||
}
|
||||
if since.IsZero() {
|
||||
return fallback
|
||||
}
|
||||
return since
|
||||
}
|
||||
|
||||
// buildingStore is the bucket of the plans building.
|
||||
type buildingStore interface {
|
||||
Put(ctx context.Context, plan string, value []byte) error
|
||||
Delete(ctx context.Context, plan string) error
|
||||
Keys(ctx context.Context) ([]string, error)
|
||||
}
|
||||
|
||||
// syncBuilding makes the bucket say what the plans say: a key for each plan in its build phase, put again, and none for any
|
||||
// other plan, deleted. It returns the faults, each once.
|
||||
func syncBuilding(ctx context.Context, store buildingStore, plans []inventory.Plan, now time.Time) []error {
|
||||
var faults []error
|
||||
live := map[string]bool{}
|
||||
for _, p := range plans {
|
||||
if !buildPhaseOf(p) {
|
||||
continue
|
||||
}
|
||||
live[p.ID] = true
|
||||
if err := putBuilding(ctx, store, p.ID, buildingSince(p, now), now); err != nil {
|
||||
faults = append(faults, err)
|
||||
}
|
||||
}
|
||||
keys, err := store.Keys(ctx)
|
||||
if err != nil {
|
||||
return append(faults, err)
|
||||
}
|
||||
for _, key := range keys {
|
||||
if live[key] {
|
||||
continue
|
||||
}
|
||||
if err := store.Delete(ctx, key); err != nil {
|
||||
faults = append(faults, fmt.Errorf("the key of %s could not be taken away: %w", key, err))
|
||||
}
|
||||
}
|
||||
return faults
|
||||
}
|
||||
|
||||
func putBuilding(ctx context.Context, store buildingStore, plan string, since, now time.Time) error {
|
||||
body, _ := json.Marshal(link.PlanBuilding{Plan: plan, Since: since.UTC(), Seen: now.UTC()})
|
||||
if err := store.Put(ctx, plan, body); err != nil {
|
||||
return fmt.Errorf("the key of %s could not be put: %w", plan, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// busBuilding is the store on the controller's bucket.
|
||||
type busBuilding struct {
|
||||
conn *nats.Conn
|
||||
mu sync.Mutex
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
|
||||
func (b *busBuilding) bucket(ctx context.Context) (jetstream.KeyValue, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.kv != nil {
|
||||
return b.kv, nil
|
||||
}
|
||||
api, err := jetstream.New(b.conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, broker.PlansBuildingBucket)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the plans building: %w", err)
|
||||
}
|
||||
b.kv = kv
|
||||
return kv, nil
|
||||
}
|
||||
|
||||
func (b *busBuilding) Put(ctx context.Context, plan string, value []byte) error {
|
||||
kv, err := b.bucket(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = kv.Put(ctx, plan, value)
|
||||
return err
|
||||
}
|
||||
|
||||
func (b *busBuilding) Delete(ctx context.Context, plan string) error {
|
||||
kv, err := b.bucket(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := kv.Delete(ctx, plan); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *busBuilding) Keys(ctx context.Context) ([]string, error) {
|
||||
kv, err := b.bucket(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keys, err := kv.Keys(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return keys, err
|
||||
}
|
||||
|
||||
// buildingKeys is the store of the serving controller; nil in a process that is not it, which says nothing.
|
||||
var buildingKeys atomic.Pointer[buildingStore]
|
||||
|
||||
func setBuildingStore(s buildingStore) {
|
||||
if s == nil {
|
||||
buildingKeys.Store(nil)
|
||||
return
|
||||
}
|
||||
buildingKeys.Store(&s)
|
||||
}
|
||||
|
||||
// sayBuilding puts a plan's key: its first build is being asked (or another of its builds). A key that could not be put is
|
||||
// said; the look that follows puts it again.
|
||||
func sayBuilding(ctx context.Context, plan string, since time.Time) {
|
||||
s := buildingKeys.Load()
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
if err := putBuilding(ctx, *s, plan, since, time.Now()); err != nil {
|
||||
fmt.Printf("plans: %v; checks may take a place of %s until the next look\n", err, plan)
|
||||
}
|
||||
}
|
||||
|
||||
// keepBuildingKey is told each plan this controller keeps: its key put again while it builds and taken away once it does not.
|
||||
func keepBuildingKey(ctx context.Context, p inventory.Plan) {
|
||||
s := buildingKeys.Load()
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
if buildPhaseOf(p) {
|
||||
sayBuilding(ctx, p.ID, buildingSince(p, time.Now()))
|
||||
return
|
||||
}
|
||||
if err := (*s).Delete(ctx, p.ID); err != nil {
|
||||
fmt.Printf("plans: the key of %s could not be taken away; it ages out: %v\n", p.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// keepingPlansBuilding looks at the plans every link.PlansBuildingRefresh for as long as this controller serves and holds the
|
||||
// lease, so a key is put again before its age and the key of a plan that left the phase unseen is taken away.
|
||||
func keepingPlansBuilding(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn) {
|
||||
store := &busBuilding{conn: conn}
|
||||
setBuildingStore(store)
|
||||
defer setBuildingStore(nil)
|
||||
tick := time.NewTicker(broker.PlansBuildingRefresh)
|
||||
defer tick.Stop()
|
||||
failing := ""
|
||||
for {
|
||||
if epoch, err := theLease.epoch(ctx); err == nil && epoch != 0 {
|
||||
why := ""
|
||||
if plans, err := inv.OpenPlans(ctx); err != nil {
|
||||
why = fmt.Sprintf("the plans cannot be read: %v", err)
|
||||
} else if faults := syncBuilding(ctx, store, plans, time.Now()); len(faults) > 0 {
|
||||
why = fmt.Sprint(faults)
|
||||
}
|
||||
if why != failing {
|
||||
if why != "" {
|
||||
fmt.Printf("plans: the keys of the plans building cannot be kept: %s\n", why)
|
||||
} else {
|
||||
fmt.Printf("plans: the keys of the plans building are kept again\n")
|
||||
}
|
||||
failing = why
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// Issue 1687 (ADR 0483 clause 2): a delivery plan's key lives from the ask of its first build until every build of its last
|
||||
// tier is built, or the plan fails, stops or waits on the operator.
|
||||
|
||||
type fakeBuildingStore struct {
|
||||
mu sync.Mutex
|
||||
keys map[string]link.PlanBuilding
|
||||
puts map[string]int
|
||||
}
|
||||
|
||||
func newFakeBuildingStore() *fakeBuildingStore {
|
||||
return &fakeBuildingStore{keys: map[string]link.PlanBuilding{}, puts: map[string]int{}}
|
||||
}
|
||||
|
||||
func (f *fakeBuildingStore) Put(_ context.Context, plan string, value []byte) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var v link.PlanBuilding
|
||||
if err := json.Unmarshal(value, &v); err != nil {
|
||||
return err
|
||||
}
|
||||
f.keys[plan] = v
|
||||
f.puts[plan]++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeBuildingStore) Delete(_ context.Context, plan string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
delete(f.keys, plan)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeBuildingStore) Keys(context.Context) ([]string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var out []string
|
||||
for k := range f.keys {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func aPlan(id, state string, tiers [][]string, states map[string]string) inventory.Plan {
|
||||
p := inventory.Plan{ID: id, State: state, Tiers: tiers, Modules: map[string]*inventory.PlanModule{}}
|
||||
now := time.Now().Add(-time.Minute)
|
||||
for m, st := range states {
|
||||
pm := &inventory.PlanModule{State: st}
|
||||
if st != "" {
|
||||
pm.AskedAt = &now
|
||||
}
|
||||
p.Modules[m] = pm
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func TestAPlanIsInItsBuildPhaseFromTheFirstAskToTheLastBuild(t *testing.T) {
|
||||
tiers := [][]string{{"a", "b"}, {"c"}}
|
||||
cases := []struct {
|
||||
name string
|
||||
plan inventory.Plan
|
||||
want bool
|
||||
}{
|
||||
{"no build asked yet", aPlan("p", inventory.PlanBuilding, tiers, nil), false},
|
||||
{"the first tier asked", aPlan("p", inventory.PlanBuilding, tiers, map[string]string{"a": "asked", "b": "asked"}), true},
|
||||
// Between two tiers: the first built and sent (rolling), the second not asked yet.
|
||||
{"between two tiers", aPlan("p", inventory.PlanRolling, tiers, map[string]string{"a": "built", "b": "built"}), true},
|
||||
{"the last tier asked", aPlan("p", inventory.PlanBuilding, tiers, map[string]string{"a": "built", "b": "built", "c": "asked"}), true},
|
||||
{"every build built", aPlan("p", inventory.PlanRolling, tiers, map[string]string{"a": "built", "b": "built", "c": "built"}), false},
|
||||
// A module the plan deletes is done as far as its tiers go (release_plan.go), so it holds no key through the rollout.
|
||||
{"a deleted module is done", aPlan("p", inventory.PlanRolling, tiers, map[string]string{"a": "built", "b": "built", "c": planDeleted}), false},
|
||||
{"failed", aPlan("p", inventory.PlanFailed, tiers, map[string]string{"a": "failed", "b": "asked"}), false},
|
||||
{"superseded", aPlan("p", inventory.PlanSuperseded, tiers, map[string]string{"a": "asked"}), false},
|
||||
{"a batch whose window is open", aPlan("p", inventory.PlanAssembling, tiers, map[string]string{"a": "asked"}), false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := buildPhaseOf(c.plan); got != c.want {
|
||||
t.Errorf("%s: in the build phase = %v, want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
waiting := aPlan("p", inventory.PlanBuilding, tiers, map[string]string{"a": "asked"})
|
||||
waiting.Delivery = &inventory.PlanDelivery{Awaits: "the operator's word"}
|
||||
if buildPhaseOf(waiting) {
|
||||
t.Error("a plan that waits on the operator is in its build phase")
|
||||
}
|
||||
}
|
||||
|
||||
// The key's life: put, refreshed, deleted at the phase's end; a plan that left the phase unseen loses its key at the next look.
|
||||
func TestAPlanKeyIsPutRefreshedAndDeletedAtThePhasesEnd(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := newFakeBuildingStore()
|
||||
tiers := [][]string{{"a"}, {"b"}}
|
||||
building := aPlan("plan-1", inventory.PlanBuilding, tiers, map[string]string{"a": "asked"})
|
||||
|
||||
if faults := syncBuilding(ctx, store, []inventory.Plan{building}, time.Now()); len(faults) > 0 {
|
||||
t.Fatal(faults)
|
||||
}
|
||||
if _, ok := store.keys["plan-1"]; !ok {
|
||||
t.Fatal("no key while the first build is asked")
|
||||
}
|
||||
first := store.keys["plan-1"]
|
||||
// Refreshed: put again, the phase's start kept.
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
syncBuilding(ctx, store, []inventory.Plan{building}, time.Now())
|
||||
if store.puts["plan-1"] != 2 || !store.keys["plan-1"].Seen.After(first.Seen) || !store.keys["plan-1"].Since.Equal(first.Since) {
|
||||
t.Errorf("the key was not refreshed with its start kept: %d puts, %+v then %+v", store.puts["plan-1"], first, store.keys["plan-1"])
|
||||
}
|
||||
// Between the tiers: still there.
|
||||
between := aPlan("plan-1", inventory.PlanRolling, tiers, map[string]string{"a": "built"})
|
||||
syncBuilding(ctx, store, []inventory.Plan{between}, time.Now())
|
||||
if _, ok := store.keys["plan-1"]; !ok {
|
||||
t.Error("the key was gone between two tiers")
|
||||
}
|
||||
// A key of another plan that is no longer in the phase.
|
||||
store.keys["plan-0"] = link.PlanBuilding{Plan: "plan-0"}
|
||||
done := aPlan("plan-1", inventory.PlanRolling, tiers, map[string]string{"a": "built", "b": "built"})
|
||||
syncBuilding(ctx, store, []inventory.Plan{done}, time.Now())
|
||||
if len(store.keys) != 0 {
|
||||
t.Errorf("keys remain after the last build was built: %v", store.keys)
|
||||
}
|
||||
}
|
||||
|
||||
// keepBuildingKey, told of each save, puts the key while the plan builds and deletes it when the plan fails.
|
||||
func TestASavedPlanPutsOrDeletesItsKey(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := newFakeBuildingStore()
|
||||
var s buildingStore = store
|
||||
buildingKeys.Store(&s)
|
||||
defer buildingKeys.Store(nil)
|
||||
|
||||
p := aPlan("plan-7", inventory.PlanBuilding, [][]string{{"a"}}, map[string]string{"a": "asked"})
|
||||
keepBuildingKey(ctx, p)
|
||||
if _, ok := store.keys["plan-7"]; !ok {
|
||||
t.Fatal("the key was not put when the plan was saved building")
|
||||
}
|
||||
p.State = inventory.PlanFailed
|
||||
keepBuildingKey(ctx, p)
|
||||
if _, ok := store.keys["plan-7"]; ok {
|
||||
t.Fatal("the key stayed after the plan failed")
|
||||
}
|
||||
}
|
||||
|
||||
// The plan's key is there before its first build is asked, so a check cannot take a place between the ask and the plan's save.
|
||||
func TestThePlansKeyIsPutBeforeItsFirstBuildIsAsked(t *testing.T) {
|
||||
store := newFakeBuildingStore()
|
||||
var s buildingStore = store
|
||||
buildingKeys.Store(&s)
|
||||
defer buildingKeys.Store(nil)
|
||||
p, byName := aTierOfThree()
|
||||
keyThere := false
|
||||
was := askABuild
|
||||
askABuild = func(context.Context, buildSource, string, string) (string, error) {
|
||||
_, keyThere = store.keys["plan-1"]
|
||||
return "build-x", nil
|
||||
}
|
||||
t.Cleanup(func() { askABuild = was })
|
||||
askModule(t.Context(), p, "a", byName)
|
||||
if !keyThere {
|
||||
t.Fatal("the plan's key was not there when its first build was asked")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"reflect"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// What the controller tells mesh-config of an open pull request, and asks it back (novox/hq ADR 0502 clauses 2 and 4,
|
||||
// issue 1826): a config key a pull request introduces is not declared on the trunk, so a value for it can be judged
|
||||
// only against the pull request's own declaration, which the core reads from the forge and mesh-config never does
|
||||
// (ADR 0495 §1). The call is mesh-config's `config` verb in the controller's own name, actions `pull-declares` and
|
||||
// `held`; the shapes are those of the interface the two sides agreed.
|
||||
|
||||
// meshConfigWait is how long the controller waits for mesh-config before it goes on without: a check is never held up
|
||||
// by a module that does not answer.
|
||||
const meshConfigWait = 10 * time.Second
|
||||
|
||||
// meshConfigAsk is one call of mesh-config's `config` verb in the controller's own name, answering its result. A
|
||||
// variable so a test needs no bus.
|
||||
var meshConfigAsk = func(ctx context.Context, args map[string]any) ([]byte, error) {
|
||||
var result []byte
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
answer, err := link.AskMeshSeatToolAs(ctx, conn, broker.ControllerName, "mesh-config", "config", args, meshConfigWait)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return fmt.Errorf("%s", answer.Error)
|
||||
}
|
||||
result = answer.Result
|
||||
return nil
|
||||
})
|
||||
return result, err
|
||||
}
|
||||
|
||||
// pullAt is an open pull request at a head: the change to read, and its number.
|
||||
type pullAt struct {
|
||||
changeSource
|
||||
Number int
|
||||
}
|
||||
|
||||
// pullAtOf is a pull request announced, as the change it makes over what it merges into.
|
||||
func pullAtOf(p link.PullUpdated) pullAt {
|
||||
old := p.BaseSHA
|
||||
if old == "" {
|
||||
old = p.Base
|
||||
}
|
||||
return pullAt{Number: p.Number, changeSource: changeSource{Owner: p.Owner, Repo: p.Repo, New: p.Commit, Old: old,
|
||||
Paths: p.Paths, Truncated: p.PathsTruncated, Removed: p.Removed}}
|
||||
}
|
||||
|
||||
func pullKey(owner, repo string, number int) string {
|
||||
return fmt.Sprintf("%s/%s#%d", owner, repo, number)
|
||||
}
|
||||
|
||||
func (p pullAt) key() string { return pullKey(p.Owner, p.Repo, p.Number) }
|
||||
|
||||
// pullDeclarations are the config declarations a pull request's head adds or changes over what it merges into, by
|
||||
// module, in the shape of the declared-config mirror. The bool says the head could be read: false for a pull request whose
|
||||
// paths are cut (the caller cuts them to the manifests of the modules it moves) or whose base is unknown, which is then
|
||||
// told nothing, never an empty set.
|
||||
func pullDeclarations(ctx context.Context, p pullAt) ([]inventory.DeclaredConfigMirrored, bool, error) {
|
||||
var out []inventory.DeclaredConfigMirrored
|
||||
if p.Truncated || p.Old == "" {
|
||||
return nil, false, nil
|
||||
}
|
||||
for _, file := range p.Paths {
|
||||
if path.Base(file) != moduleManifestFile || slices.Contains(p.Removed, file) {
|
||||
continue
|
||||
}
|
||||
now, exists, err := manifestCached(ctx, p.Owner, p.Repo, file, p.New)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if !exists || now.Module == "" {
|
||||
continue
|
||||
}
|
||||
was, _, err := manifestCached(ctx, p.Owner, p.Repo, file, p.Old)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
have, err := inventory.DeclaredOf(now.Settings)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("%s at %.8s: %w", file, p.New, err)
|
||||
}
|
||||
had, err := inventory.DeclaredOf(was.Settings)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("%s at %.8s: %w", file, p.Old, err)
|
||||
}
|
||||
changed := map[string]inventory.DeclaredMirrored{}
|
||||
for key, d := range have {
|
||||
if before, there := had[key]; !there || !reflect.DeepEqual(before, d) {
|
||||
changed[key] = d
|
||||
}
|
||||
}
|
||||
if len(changed) > 0 {
|
||||
out = append(out, inventory.DeclaredConfigMirrored{Module: now.Module, Settings: changed})
|
||||
}
|
||||
}
|
||||
return out, true, nil
|
||||
}
|
||||
|
||||
// tellPullDeclares tells mesh-config, for each pull request at every announced head, what the head declares, an empty set
|
||||
// when it declares nothing new: mesh-config re-judges the values held at every head, so a head that reverts the change
|
||||
// drops them (ADR 0502 clause 6). A
|
||||
// failure is said to w and the check goes on: the held values are then none (ADR 0502 clause 2).
|
||||
func tellPullDeclares(ctx context.Context, w io.Writer, pulls []pullAt) {
|
||||
for _, p := range pulls {
|
||||
mods, readable, err := pullDeclarations(ctx, p)
|
||||
if err != nil {
|
||||
fmt.Fprintf(w, "%s: what its head declares cannot be read, so mesh-config is told nothing: %v\n", p.key(), err)
|
||||
continue
|
||||
}
|
||||
if !readable {
|
||||
continue
|
||||
}
|
||||
declares := map[string]map[string]inventory.DeclaredMirrored{}
|
||||
for _, m := range mods {
|
||||
declares[m.Module] = m.Settings
|
||||
}
|
||||
if _, err := meshConfigAsk(ctx, map[string]any{"action": "pull-declares", "pull": p.key(), "head": p.New,
|
||||
"declares": declares}); err != nil {
|
||||
fmt.Fprintf(w, "%s: mesh-config cannot be told what its head declares (%v); the check goes on without held values\n",
|
||||
p.key(), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// heldFor asks mesh-config for the approved held values of these pull requests at their heads (ADR 0502 clause 4). Only
|
||||
// those of the pull requests asked about are carried. A failure is said to w and answers none.
|
||||
func heldFor(ctx context.Context, w io.Writer, pulls []pullAt) []link.HeldConfig {
|
||||
if len(pulls) == 0 {
|
||||
return nil
|
||||
}
|
||||
type pullHead struct {
|
||||
Pull string `json:"pull"`
|
||||
Head string `json:"head"`
|
||||
}
|
||||
asked := make([]pullHead, 0, len(pulls))
|
||||
for _, p := range pulls {
|
||||
asked = append(asked, pullHead{p.key(), p.New})
|
||||
}
|
||||
raw, err := meshConfigAsk(ctx, map[string]any{"action": "held", "pulls": asked})
|
||||
if err != nil {
|
||||
fmt.Fprintf(w, "mesh-config cannot say what it holds for these pull requests (%v); the check goes on without held values\n", err)
|
||||
return nil // empty-on-error: said above; a check without held values fails for want of the value and says so, it never passes on a guess
|
||||
}
|
||||
var answer struct {
|
||||
Held []link.HeldConfig `json:"held"`
|
||||
Skipped []struct {
|
||||
Pull string `json:"pull"`
|
||||
Why string `json:"why"`
|
||||
} `json:"skipped"`
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := json.Unmarshal(raw, &answer); err != nil {
|
||||
fmt.Fprintf(w, "mesh-config's held values cannot be read (%v); the check goes on without them\n", err)
|
||||
return nil // empty-on-error: said above; a check without held values fails for want of the value and says so, it never passes on a guess
|
||||
}
|
||||
for _, s := range answer.Skipped {
|
||||
fmt.Fprintf(w, "%s: mesh-config holds a value it does not carry: %s\n", s.Pull, s.Why)
|
||||
}
|
||||
var out []link.HeldConfig
|
||||
for _, h := range answer.Held {
|
||||
for _, p := range pulls {
|
||||
if h.Pull == p.key() {
|
||||
out = append(out, h)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// withHeldConfig tells mesh-config what these pull requests declare and puts the approved held values of them on the
|
||||
// check request (ADR 0502 clause 4). With carry false, a member of a group whose composed check composes, only the
|
||||
// declarations are told.
|
||||
func withHeldConfig(ctx context.Context, w io.Writer, request *link.BuildRequest, pulls []pullAt, carry bool) {
|
||||
tellPullDeclares(ctx, w, pulls)
|
||||
if carry {
|
||||
carryHeld(ctx, w, request, pulls)
|
||||
}
|
||||
}
|
||||
|
||||
// carryHeld puts the approved held values of these pull requests on the check request.
|
||||
func carryHeld(ctx context.Context, w io.Writer, request *link.BuildRequest, pulls []pullAt) {
|
||||
if request.Check != nil {
|
||||
request.Check.HeldConfig = heldFor(ctx, w, pulls)
|
||||
}
|
||||
}
|
||||
|
||||
// holdsModules is whether a module of the mesh is built from this repository.
|
||||
func holdsModules(entries []inventory.Entry, owner, repo string) bool {
|
||||
return slices.ContainsFunc(entries, func(e inventory.Entry) bool {
|
||||
return sameRepository(e.Source.Repository, link.SourceMoved{Owner: owner, Repo: repo})
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
const (
|
||||
headSHA = "2222222222222222222222222222222222222222"
|
||||
baseSHA = "1111111111111111111111111111111111111111"
|
||||
)
|
||||
|
||||
// fakeManifests serves manifests by ref, and counts what the forge was asked. Issue 1826/AC1 (unit level).
|
||||
func fakeManifests(t *testing.T, at map[string]*catalogue.Manifest) {
|
||||
t.Helper()
|
||||
was := manifestAt
|
||||
t.Cleanup(func() { manifestAt = was })
|
||||
manifestsReadMu.Lock()
|
||||
clear(manifestsRead)
|
||||
manifestsReadMu.Unlock()
|
||||
manifestAt = func(_ context.Context, _, _, _, ref string) (catalogue.Manifest, bool, error) {
|
||||
if m := at[ref]; m != nil {
|
||||
return *m, true, nil
|
||||
}
|
||||
return catalogue.Manifest{}, false, nil
|
||||
}
|
||||
}
|
||||
|
||||
// fakeConfig records the calls to mesh-config and answers each action from answers.
|
||||
func fakeConfig(t *testing.T, answers map[string]string) *[]map[string]any {
|
||||
t.Helper()
|
||||
var calls []map[string]any
|
||||
was := meshConfigAsk
|
||||
t.Cleanup(func() { meshConfigAsk = was })
|
||||
meshConfigAsk = func(_ context.Context, args map[string]any) ([]byte, error) {
|
||||
calls = append(calls, args)
|
||||
return []byte(answers[args["action"].(string)]), nil
|
||||
}
|
||||
return &calls
|
||||
}
|
||||
|
||||
func manifest(module string, keys map[string]catalogue.SettingDeclaration) *catalogue.Manifest {
|
||||
return &catalogue.Manifest{Module: module, Settings: keys}
|
||||
}
|
||||
|
||||
func thePull() pullAt {
|
||||
return pullAt{Number: 434, changeSource: changeSource{Owner: "novox", Repo: "mesh-catalog", New: headSHA, Old: baseSHA,
|
||||
Paths: []string{"modules/n8n/module.json", "modules/n8n/README.md"}}}
|
||||
}
|
||||
|
||||
// 1826/AC1: a key the head adds is sent with its declaration, under the pull request and the head.
|
||||
func TestAKeyThePullRequestAddsIsToldToMeshConfig(t *testing.T) {
|
||||
fakeManifests(t, map[string]*catalogue.Manifest{
|
||||
baseSHA: manifest("n8n", map[string]catalogue.SettingDeclaration{"port": {Type: "number"}}),
|
||||
headSHA: manifest("n8n", map[string]catalogue.SettingDeclaration{"port": {Type: "number"},
|
||||
"media-gid": {Type: "number", Required: true}}),
|
||||
})
|
||||
calls := fakeConfig(t, map[string]string{"pull-declares": `{"kept":1}`})
|
||||
tellPullDeclares(context.Background(), io.Discard, []pullAt{thePull()})
|
||||
if len(*calls) != 1 {
|
||||
t.Fatalf("mesh-config was asked %d times: %v", len(*calls), *calls)
|
||||
}
|
||||
c := (*calls)[0]
|
||||
if c["action"] != "pull-declares" || c["pull"] != "novox/mesh-catalog#434" || c["head"] != headSHA {
|
||||
t.Fatalf("the call is %v", c)
|
||||
}
|
||||
body, _ := json.Marshal(c["declares"])
|
||||
var declares map[string]map[string]inventory.DeclaredMirrored
|
||||
if err := json.Unmarshal(body, &declares); err != nil || len(declares) != 1 || declares["n8n"] == nil {
|
||||
t.Fatalf("declares %s: %v", body, err)
|
||||
}
|
||||
mods := []inventory.DeclaredConfigMirrored{{Module: "n8n", Settings: declares["n8n"]}}
|
||||
if _, has := mods[0].Settings["port"]; has || len(mods[0].Settings) != 1 {
|
||||
t.Fatalf("only the added key is told, got %v", mods[0].Settings)
|
||||
}
|
||||
d := mods[0].Settings["media-gid"]
|
||||
if d.Type != "number" || !d.Required {
|
||||
t.Fatalf("the declaration is %+v", d)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC1: a key whose declaration the head changes is sent; one it leaves as it was is not.
|
||||
func TestAnUnchangedKeyIsNotToldAndAChangedOneIs(t *testing.T) {
|
||||
fakeManifests(t, map[string]*catalogue.Manifest{
|
||||
baseSHA: manifest("n8n", map[string]catalogue.SettingDeclaration{"port": {Type: "number"}, "x": {Type: "string"}}),
|
||||
headSHA: manifest("n8n", map[string]catalogue.SettingDeclaration{"port": {Type: "number"}, "x": {Type: "string", Pattern: "a+"}}),
|
||||
})
|
||||
mods, touched, err := pullDeclarations(context.Background(), thePull())
|
||||
if err != nil || !touched || len(mods) != 1 || len(mods[0].Settings) != 1 || mods[0].Settings["x"].Pattern != "a+" {
|
||||
t.Fatalf("%+v %v %v", mods, touched, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC1: a head that changes no module.json, or removes it, is still told, with declares {}: mesh-config re-judges at every
|
||||
// head, so a head that reverts the change drops the value held for it (review round 1, finding 1).
|
||||
func TestAHeadThatDeclaresNothingIsToldAsEmpty(t *testing.T) {
|
||||
fakeManifests(t, map[string]*catalogue.Manifest{baseSHA: manifest("n8n", nil), headSHA: manifest("n8n", nil)})
|
||||
calls := fakeConfig(t, nil)
|
||||
p := thePull()
|
||||
p.Paths = []string{"modules/n8n/README.md"}
|
||||
tellPullDeclares(context.Background(), io.Discard, []pullAt{p})
|
||||
p = thePull()
|
||||
p.Removed = p.Paths
|
||||
tellPullDeclares(context.Background(), io.Discard, []pullAt{p})
|
||||
if len(*calls) != 2 {
|
||||
t.Fatalf("mesh-config was told %v", *calls)
|
||||
}
|
||||
for _, c := range *calls {
|
||||
if d, _ := json.Marshal(c["declares"]); string(d) != "{}" || c["head"] != headSHA {
|
||||
t.Fatalf("told %v", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC1: a head whose paths are cut, or whose base is unknown, cannot be read: nothing is told, never an empty set.
|
||||
func TestAHeadThatCannotBeReadIsNotToldAsEmpty(t *testing.T) {
|
||||
calls := fakeConfig(t, nil)
|
||||
p := thePull()
|
||||
p.Truncated = true
|
||||
tellPullDeclares(context.Background(), io.Discard, []pullAt{p})
|
||||
if len(*calls) != 0 {
|
||||
t.Fatalf("told %v", *calls)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC1: a module.json changed without a declaration changing is told as an empty set, so mesh-config re-judges what it holds.
|
||||
func TestAChangedManifestWithNoNewDeclarationIsToldAsEmpty(t *testing.T) {
|
||||
m := manifest("n8n", map[string]catalogue.SettingDeclaration{"port": {Type: "number"}})
|
||||
fakeManifests(t, map[string]*catalogue.Manifest{baseSHA: m, headSHA: m})
|
||||
calls := fakeConfig(t, nil)
|
||||
tellPullDeclares(context.Background(), io.Discard, []pullAt{thePull()})
|
||||
if len(*calls) != 1 {
|
||||
t.Fatalf("calls %v", *calls)
|
||||
}
|
||||
if d, _ := json.Marshal((*calls)[0]["declares"]); string(d) != "{}" {
|
||||
t.Fatalf("declares %s", d)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC1: mesh-config not answering is said and goes on; the check is not stopped.
|
||||
func TestMeshConfigNotAnsweringIsSaidAndGoneOn(t *testing.T) {
|
||||
fakeManifests(t, map[string]*catalogue.Manifest{baseSHA: manifest("n8n", nil),
|
||||
headSHA: manifest("n8n", map[string]catalogue.SettingDeclaration{"k": {Type: "string"}})})
|
||||
was := meshConfigAsk
|
||||
t.Cleanup(func() { meshConfigAsk = was })
|
||||
meshConfigAsk = func(context.Context, map[string]any) ([]byte, error) { return nil, io.ErrUnexpectedEOF }
|
||||
var said strings.Builder
|
||||
tellPullDeclares(context.Background(), &said, []pullAt{thePull()})
|
||||
got := heldFor(context.Background(), &said, []pullAt{thePull()})
|
||||
if len(got) != 0 || !strings.Contains(said.String(), "mesh-config") {
|
||||
t.Fatalf("held %v, said %q", got, said.String())
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC2: the held values mesh-config answers for the check's pull requests are what the check request carries.
|
||||
func TestTheHeldValuesOfTheChecksPullRequestsAreAsked(t *testing.T) {
|
||||
calls := fakeConfig(t, map[string]string{"held": `{"held":[{"pull":"novox/mesh-catalog#434","head":"` + headSHA +
|
||||
`","node":"ace","module":"n8n","key":"media-gid","value":2000,"approved":"the operator"}]}`})
|
||||
other := pullAt{Number: 7, changeSource: changeSource{Owner: "novox", Repo: "mesh-controller", New: baseSHA}}
|
||||
got := heldFor(context.Background(), io.Discard, []pullAt{thePull(), other})
|
||||
if len(got) != 1 || got[0].Node != "ace" || got[0].Key != "media-gid" || string(got[0].Value) != "2000" {
|
||||
t.Fatalf("got %+v", got)
|
||||
}
|
||||
c := (*calls)[0]
|
||||
pulls, _ := json.Marshal(c["pulls"])
|
||||
if c["action"] != "held" || string(pulls) != `[{"pull":"novox/mesh-catalog#434","head":"`+headSHA+
|
||||
`"},{"pull":"novox/mesh-controller#7","head":"`+baseSHA+`"}]` {
|
||||
t.Fatalf("asked %v %s", c, pulls)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC2: a value held for a pull request the check is not of is never carried.
|
||||
func TestAHeldValueOfAnotherPullRequestIsNotCarried(t *testing.T) {
|
||||
fakeConfig(t, map[string]string{"held": `{"held":[{"pull":"novox/mesh-catalog#999","head":"x","node":"ace","module":"n8n","key":"k","value":1}]}`})
|
||||
if got := heldFor(context.Background(), io.Discard, []pullAt{thePull()}); len(got) != 0 {
|
||||
t.Fatalf("carried %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC2: the check request carries what mesh-config answered; a member of a composed group asks none.
|
||||
func TestTheCheckRequestCarriesTheHeldValues(t *testing.T) {
|
||||
fakeManifests(t, map[string]*catalogue.Manifest{baseSHA: manifest("n8n", nil), headSHA: manifest("n8n", nil)})
|
||||
fakeConfig(t, map[string]string{"held": `{"held":[{"pull":"novox/mesh-catalog#434","node":"ace","module":"n8n","key":"media-gid","value":2000}]}`})
|
||||
request := link.BuildRequest{Check: &link.CheckRequest{}}
|
||||
withHeldConfig(context.Background(), io.Discard, &request, []pullAt{thePull()}, true)
|
||||
if len(request.Check.HeldConfig) != 1 || request.Check.HeldConfig[0].Key != "media-gid" {
|
||||
t.Fatalf("the request carries %+v", request.Check.HeldConfig)
|
||||
}
|
||||
request = link.BuildRequest{Check: &link.CheckRequest{}}
|
||||
withHeldConfig(context.Background(), io.Discard, &request, []pullAt{thePull()}, false)
|
||||
if len(request.Check.HeldConfig) != 0 {
|
||||
t.Fatalf("a composed member carries %+v", request.Check.HeldConfig)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC1: a repository that builds a module of the mesh holds modules; another does not.
|
||||
func TestARepositoryThatBuildsAModuleHoldsModules(t *testing.T) {
|
||||
entries := []inventory.Entry{{Manifest: catalogue.Manifest{Module: "a"}, Source: inventory.Source{Repository: "novox/mesh-catalog", Path: "modules/a"}}}
|
||||
if !holdsModules(entries, "novox", "mesh-catalog") || holdsModules(entries, "novox", "hq") {
|
||||
t.Fatal("wrong")
|
||||
}
|
||||
}
|
||||
@@ -281,9 +281,11 @@ func serve(ctx context.Context) (err error) {
|
||||
// The provisioner's plan of each handed module, as its machine was last sent it (design 57 P6), in the module's
|
||||
// assignment.
|
||||
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
|
||||
go mirroringRecords(ctx, inv, bus.Conn, bus)
|
||||
go mirroringRecords(ctx, open, inv, bus.Conn, bus)
|
||||
// And the bucket of the files that travel by digest, judged against what each machine's latest declaration names (ADR 0486 §3).
|
||||
go sweepingDeclarationFiles(ctx, inv, bus.Conn)
|
||||
// And the key of each plan in its build phase, which holds the build seat's checks while a delivery builds (ADR 0483 clause 2).
|
||||
go keepingPlansBuilding(ctx, inv, bus.Conn)
|
||||
// Every call carries the lease's epoch, and its record is written only under the lease (novox/hq
|
||||
// to-be 45 §6).
|
||||
link.Calls.UnderLease(func() (uint64, error) { return theLease.epoch(ctx) })
|
||||
|
||||
@@ -91,7 +91,9 @@ type queueAnswer struct {
|
||||
Dead int `json:"dead"`
|
||||
MaxDeliver int `json:"max-deliver"`
|
||||
Asks []queuedAnswer `json:"asks"`
|
||||
Text string `json:"text"`
|
||||
// HeldFor are the delivery plans building, while which the waiting checks are held (ADR 0483 clause 5); empty when none.
|
||||
HeldFor []string `json:"held-for,omitempty"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
// queuedAnswer is one ask as data. Never what the request carries beyond its source — not `held`, every
|
||||
@@ -120,6 +122,9 @@ type queuedAnswer struct {
|
||||
// Delivery is the how-manyth time an ask in flight was handed out; more than 1 is a restart
|
||||
// (novox/hq issue 594).
|
||||
Delivery int `json:"delivery,omitempty"`
|
||||
// LetThrough is a check that waited past 30 minutes and is let through while a delivery builds: the next one, or the one
|
||||
// that runs (ADR 0483 clause 5).
|
||||
LetThrough bool `json:"let-through,omitempty"`
|
||||
}
|
||||
|
||||
// aCommitRef is a ref that names a commit by its full hash (SHA-1 or SHA-256).
|
||||
@@ -130,7 +135,8 @@ var aCommitRef = regexp.MustCompile(`^(?:[0-9a-f]{40}|[0-9a-f]{64})$`)
|
||||
func queueAnswerOf(q link.Queue, now time.Time) queueAnswer {
|
||||
waiting, running, dead := q.Of(link.AskWaiting), q.Of(link.AskInFlight), q.Of(link.AskDead)
|
||||
out := queueAnswer{Seat: q.Seat, Waiting: len(waiting), InFlight: len(running), Dead: len(dead),
|
||||
MaxDeliver: q.MaxDeliver, Asks: []queuedAnswer{}, Text: queueText(q, now)}
|
||||
MaxDeliver: q.MaxDeliver, Asks: []queuedAnswer{}, HeldFor: q.Hold.Plans, Text: queueText(q, now)}
|
||||
letNext, letRunning := q.LetThroughOf(now)
|
||||
one := func(a link.QueuedAsk, position int) queuedAnswer {
|
||||
kind := link.KindOf(a.Request)
|
||||
since := a.AskedAt
|
||||
@@ -143,7 +149,7 @@ func queueAnswerOf(q link.Queue, now time.Time) queueAnswer {
|
||||
}
|
||||
return queuedAnswer{ID: a.ID, State: a.State, Kind: kind, Repository: link.WithoutPassword(a.Repository), Path: a.Path,
|
||||
Ref: a.Ref, Commit: commit, Machine: a.On, WasOn: a.Was, AskedAt: a.AskedAt, Since: since,
|
||||
Position: position, Seq: a.Seq, Delivery: a.Delivery}
|
||||
Position: position, Seq: a.Seq, Delivery: a.Delivery, LetThrough: a.ID != "" && (a.ID == letNext || a.ID == letRunning)}
|
||||
}
|
||||
for _, a := range running {
|
||||
out.Asks = append(out.Asks, one(a, 0))
|
||||
@@ -179,6 +185,11 @@ func queueText(q link.Queue, now time.Time) string {
|
||||
}
|
||||
return s
|
||||
}
|
||||
letNext, letRunning := q.LetThroughOf(now)
|
||||
const passed = "waited past 30 minutes, let through"
|
||||
if q.HoldFault != "" {
|
||||
fmt.Fprintf(&b, "why checks wait cannot be read, so none is said held: %s\n", q.HoldFault)
|
||||
}
|
||||
if len(running) > 0 {
|
||||
fmt.Fprintln(&b, "\nin flight:")
|
||||
for _, a := range running {
|
||||
@@ -193,14 +204,27 @@ func queueText(q link.Queue, now time.Time) string {
|
||||
case a.Was != "":
|
||||
where = fmt.Sprintf("handed back by %s, to be handed out again", a.Was)
|
||||
}
|
||||
fmt.Fprintf(&b, " %-26s %-8s %s — %s, %s (seq %d)\n", a.ID, link.KindOf(a.Request), what(a), where,
|
||||
ago(a.AskedAt), a.Seq)
|
||||
note := ""
|
||||
if a.ID != "" && a.ID == letRunning {
|
||||
note = " — " + passed
|
||||
}
|
||||
fmt.Fprintf(&b, " %-26s %-8s %s — %s, %s (seq %d)%s\n", a.ID, link.KindOf(a.Request), what(a), where,
|
||||
ago(a.AskedAt), a.Seq, note)
|
||||
}
|
||||
}
|
||||
if len(waiting) > 0 {
|
||||
fmt.Fprintln(&b, "\nwaiting, in the order they are handed out:")
|
||||
// **Why the checks wait** (ADR 0483 clause 5): above them, the plans they are held for.
|
||||
if len(q.Hold.Plans) > 0 && hasCheck(waiting) {
|
||||
fmt.Fprintf(&b, " checks are held while delivery %s builds; the oldest check that waited 30 minutes goes first, one at a time\n",
|
||||
strings.Join(q.Hold.Plans, ", "))
|
||||
}
|
||||
for _, a := range waiting {
|
||||
fmt.Fprintf(&b, " %-26s %-8s %s — %s (seq %d)\n", a.ID, link.KindOf(a.Request), what(a), ago(a.AskedAt), a.Seq)
|
||||
note := ""
|
||||
if a.ID != "" && a.ID == letNext {
|
||||
note = " — " + passed + " next"
|
||||
}
|
||||
fmt.Fprintf(&b, " %-26s %-8s %s — %s (seq %d)%s\n", a.ID, link.KindOf(a.Request), what(a), ago(a.AskedAt), a.Seq, note)
|
||||
}
|
||||
}
|
||||
if len(dead) > 0 {
|
||||
@@ -218,6 +242,16 @@ func queueText(q link.Queue, now time.Time) string {
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// hasCheck says a check is among the asks.
|
||||
func hasCheck(asks []link.QueuedAsk) bool {
|
||||
for _, a := range asks {
|
||||
if a.Request.Check != nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// cancelCommand drops one waiting or dead ask.
|
||||
func cancelCommand(ctx context.Context, args []string) error {
|
||||
if len(args) != 1 {
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// Issue 1688 / 1652 AC5 (ADR 0483 clause 5): the queue names, above the waiting checks, the plan they are held for, and says of a
|
||||
// check let through that it waited past 30 minutes.
|
||||
|
||||
func aHeldQueue(now time.Time) link.Queue {
|
||||
check := func(id string, ago time.Duration, n int) link.QueuedAsk {
|
||||
return link.QueuedAsk{Seq: uint64(n), ID: id, Repository: "ssh://forge/novox/r", AskedAt: now.Add(-ago), State: link.AskWaiting,
|
||||
Request: link.BuildRequest{ID: id, Check: &link.CheckRequest{Owner: "novox", Repo: "r", Number: n}}}
|
||||
}
|
||||
return link.Queue{Seat: link.TheBuildMachine, Asks: []link.QueuedAsk{
|
||||
check("build-young", 5*time.Minute, 1),
|
||||
check("build-old", 45*time.Minute, 2),
|
||||
check("build-older", 50*time.Minute, 3),
|
||||
}, Hold: link.CheckHold{Plans: []string{"plan-42"}}}
|
||||
}
|
||||
|
||||
func TestTheQueueNamesThePlanTheChecksAreHeldFor(t *testing.T) {
|
||||
now := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)
|
||||
text := queueText(aHeldQueue(now), now)
|
||||
heldAt := strings.Index(text, "held while delivery plan-42 builds")
|
||||
firstAt := strings.Index(text, "build-young")
|
||||
if heldAt < 0 || firstAt < 0 || heldAt > firstAt {
|
||||
t.Fatalf("the plan the checks are held for is not named above the waiting checks:\n%s", text)
|
||||
}
|
||||
// Nothing building: nothing held.
|
||||
q := aHeldQueue(now)
|
||||
q.Hold = link.CheckHold{}
|
||||
if strings.Contains(queueText(q, now), "held while") {
|
||||
t.Fatal("checks are said held with no delivery building")
|
||||
}
|
||||
}
|
||||
|
||||
func TestACheckLetThroughIsSaidToHaveWaitedPastThirtyMinutes(t *testing.T) {
|
||||
now := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)
|
||||
q := aHeldQueue(now)
|
||||
text := queueText(q, now)
|
||||
var saidOf []string
|
||||
for _, l := range strings.Split(text, "\n") {
|
||||
if strings.Contains(l, "waited past 30 minutes, let through") {
|
||||
saidOf = append(saidOf, strings.Fields(l)[0])
|
||||
}
|
||||
}
|
||||
if len(saidOf) != 1 || saidOf[0] != "build-older" {
|
||||
t.Fatalf("the check let through next (the oldest) is not the one said: %v\n%s", saidOf, text)
|
||||
}
|
||||
answer := queueAnswerOf(q, now)
|
||||
if len(answer.HeldFor) != 1 || answer.HeldFor[0] != "plan-42" {
|
||||
t.Errorf("held-for = %v", answer.HeldFor)
|
||||
}
|
||||
for _, a := range answer.Asks {
|
||||
if a.LetThrough != (a.ID == "build-older") {
|
||||
t.Errorf("%s let-through = %v", a.ID, a.LetThrough)
|
||||
}
|
||||
}
|
||||
// A check in flight under the let-through key is said too.
|
||||
q.Asks = append(q.Asks, link.QueuedAsk{Seq: 9, ID: "build-running", State: link.AskInFlight, On: "ace", AskedAt: now.Add(-time.Hour),
|
||||
Started: now.Add(-time.Minute), Request: link.BuildRequest{ID: "build-running", Check: &link.CheckRequest{Owner: "novox", Repo: "r", Number: 9}}})
|
||||
q.Hold.LetThrough = &link.LetThrough{Builder: "ace"}
|
||||
if !strings.Contains(queueText(q, now), "build-running") || !strings.Contains(queueText(q, now), "waited past 30 minutes, let through\n") {
|
||||
t.Errorf("the check in flight let through is not said:\n%s", queueText(q, now))
|
||||
}
|
||||
// Younger than the bound: none is said.
|
||||
young := aHeldQueue(now)
|
||||
young.Asks = young.Asks[:1]
|
||||
if strings.Contains(queueText(young, now), "waited past") {
|
||||
t.Error("a check of 5 minutes is said to have waited past 30")
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
@@ -29,7 +30,7 @@ import (
|
||||
const mirrorEvery = 10 * time.Second
|
||||
|
||||
// mirroringRecords keeps the mirrors current for as long as this controller serves: ctx ends when it stops.
|
||||
func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn, announce recordmirror.Announcer) {
|
||||
func mirroringRecords(ctx context.Context, open *stores, inv *inventory.Inventory, conn *nats.Conn, announce recordmirror.Announcer) {
|
||||
api, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
fmt.Printf("the record mirrors are not kept: %v\n", err)
|
||||
@@ -44,8 +45,9 @@ func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats.
|
||||
return e
|
||||
}
|
||||
sources := map[string]func(context.Context) (map[string][]byte, error){
|
||||
recordmirror.DeclaredConfig.Name: inv.DeclaredConfigMirror,
|
||||
recordmirror.Predefined.Name: inv.PredefinedMirror,
|
||||
recordmirror.DeclaredConfig.Name: inv.DeclaredConfigMirror,
|
||||
recordmirror.Predefined.Name: inv.PredefinedMirror,
|
||||
recordmirror.DeclaredPackages.Name: declaredPackagesMirror(open),
|
||||
}
|
||||
writers := map[string]*recordmirror.Writer{}
|
||||
failing := map[string]string{}
|
||||
@@ -176,3 +178,36 @@ func keepingAssignments(ctx context.Context, inv *inventory.Inventory, announce
|
||||
}
|
||||
events.known = now
|
||||
}
|
||||
|
||||
// declaredPackagesMirror is the source of the declared-packages mirror (ADR 0507 §5): each node's composition, asked as a
|
||||
// question that makes nothing, and the packages it declares with the modules that declare them. A node whose own set does
|
||||
// not compose has no key, and any other failure to read is the pass's error, so a key is never purged for a store that
|
||||
// could not be read.
|
||||
func declaredPackagesMirror(open *stores) func(context.Context) (map[string][]byte, error) {
|
||||
return func(ctx context.Context) (map[string][]byte, error) {
|
||||
in, err := inputFor(ctx, open)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Which module owns each resource is carried only for a node-engine that reads it, so every node is asked as one that does.
|
||||
asked := in.Input
|
||||
asked.Nodes = append([]compose.Node(nil), in.Nodes...)
|
||||
for i := range asked.Nodes {
|
||||
asked.Nodes[i].ReadsModules = true
|
||||
}
|
||||
out := map[string][]byte{}
|
||||
for _, n := range in.sortedNodes() {
|
||||
d, _, err := compose.ComposeIn(asked, n.Name, compose.Foreseeing)
|
||||
if compose.IsUnresolvable(err) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the packages %s declares cannot be read: %w", n.Name, err)
|
||||
}
|
||||
if out[n.Name], err = inventory.DeclaredPackagesValue(n.Name, d.Resources, d.ModuleOf); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -577,6 +577,9 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
|
||||
}
|
||||
// **The walk takes the check's build of the same key** (novox/hq ADR 0350 §2): its ask says so, with the pull
|
||||
// requests it answers, whose checks name the part of a differing key.
|
||||
// **The plan's key is there before its first build is** (ADR 0483 clause 2): a check taken between the ask and the plan's save
|
||||
// would hold a place for its whole run.
|
||||
sayBuilding(ctx, p.ID, now)
|
||||
id, err := askABuild(withReuse(ctx, reuseWithVerdicts(ctx, p.ID, reuseAskOf(p), mergedHeadsOf(p))), source,
|
||||
e.Source.Path, ref)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
package main
|
||||
|
||||
// The machines whose declaration a walk's merges change (novox/hq ADR 0505 §2): beside what the walk sends, a merge
|
||||
// can change what another machine is to run — an assignment, a provision or a requirement added or removed — and a
|
||||
// problem raised there as the walk runs, or as it settles, is the walk's as much as one on a machine it sends to. The
|
||||
// router holds what a walk touches (ADR 0308 §1); these are touches of their own, marked `declared`.
|
||||
//
|
||||
// **Compared as the mesh compares whether a machine is up to date.** Each machine's declaration is composed as a push
|
||||
// would compose it, with the number, epoch and generation it was last sent, and its digest held against the one it was
|
||||
// last sent (wouldSend, inventory.Waiting): byte for byte what it was sent when nothing changed, so a secret, a sequence
|
||||
// or a lease's epoch never makes a machine differ by itself. A machine whose declaration cannot be worked out now and
|
||||
// was sent one before is taken as changed: not known is changed, never "unchanged" (a merge that leaves a machine
|
||||
// unable to compose is the case this exists for, the 11:03 message of ADR 0505's context).
|
||||
//
|
||||
// **Said again, not waited for.** Composing every machine takes seconds; the walk's start and end are said at once as
|
||||
// before, and the serving controller, which hears every `plan-moved` — its own and a verb's — says a start or an end
|
||||
// again with the declared touches added, marked `declared_read`, which it never reads twice. The router merges a walk
|
||||
// said again (messenger walks.go).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// walkDeclaredRead marks a walk's start or end said again with its declared touches: read once.
|
||||
const walkDeclaredRead = "declared_read"
|
||||
|
||||
// declaredWithin bounds the comparison of every machine for one walk's start or end.
|
||||
const declaredWithin = time.Minute
|
||||
|
||||
// changedDeclarations is every machine whose declaration differs from what it was last sent, and every machine sent
|
||||
// one before whose declaration cannot be worked out now. A seam a test replaces.
|
||||
var changedDeclarations = func(ctx context.Context, open *stores) ([]string, error) {
|
||||
nodes, err := open.inventory.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx = withInputCache(ctx)
|
||||
would, err := wouldSend(ctx, open, nodes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
behind, err := open.inventory.Waiting(ctx, would)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sent := map[string]bool{}
|
||||
for _, n := range nodes {
|
||||
if s, known, err := open.inventory.SentTo(ctx, n.Name); err == nil && known && s.Digest != "" {
|
||||
sent[n.Name] = true
|
||||
}
|
||||
}
|
||||
return changedOf(nodes, would, behind, sent), nil
|
||||
}
|
||||
|
||||
// changedOf is changedDeclarations over what was read: the machines behind, and those sent one before that could not
|
||||
// be worked out. Pure; sorted.
|
||||
func changedOf(nodes []inventory.Node, would map[string]string, behind []inventory.Machine, sent map[string]bool) []string {
|
||||
set := map[string]bool{}
|
||||
for _, m := range behind {
|
||||
set[m.Node] = true
|
||||
}
|
||||
for _, n := range nodes {
|
||||
if _, worked := would[n.Name]; !worked && sent[n.Name] {
|
||||
set[n.Name] = true
|
||||
}
|
||||
}
|
||||
out := make([]string, 0, len(set))
|
||||
for n := range set {
|
||||
out = append(out, n)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// withDeclared is a walk's start or end said again with the machines whose declaration changed added as declared
|
||||
// touches: false when the body says no start or end, was said again already, or names no machine it does not touch.
|
||||
func withDeclared(body []byte, changed []string) ([]byte, bool, error) {
|
||||
said, wanted, err := toSayAgain(body)
|
||||
if err != nil || !wanted {
|
||||
return nil, false, err
|
||||
}
|
||||
var touches []walkTouch
|
||||
if raw := said["touches"]; raw != nil && string(raw) != "null" {
|
||||
if err := json.Unmarshal(raw, &touches); err != nil {
|
||||
return nil, false, fmt.Errorf("its touches are not a list of {machine, modules}: %w", err)
|
||||
}
|
||||
}
|
||||
added := declaredTouches(touches, changed)
|
||||
if len(added) == 0 {
|
||||
return nil, false, nil
|
||||
}
|
||||
touches = append(touches, added...)
|
||||
sort.Slice(touches, func(i, j int) bool { return touches[i].Machine < touches[j].Machine })
|
||||
if said["touches"], err = json.Marshal(touches); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
said[walkDeclaredRead] = json.RawMessage(`true`)
|
||||
out, err := json.Marshal(said)
|
||||
return out, err == nil, err
|
||||
}
|
||||
|
||||
// toSayAgain reads a plan-moved: whether it says a walk's start or end not said again yet.
|
||||
func toSayAgain(body []byte) (map[string]json.RawMessage, bool, error) {
|
||||
var said map[string]json.RawMessage
|
||||
if err := json.Unmarshal(body, &said); err != nil {
|
||||
return nil, false, fmt.Errorf("a plan-moved that is not an object: %w", err)
|
||||
}
|
||||
var phase string
|
||||
_ = json.Unmarshal(said["phase"], &phase)
|
||||
return said, (phase == inventory.WalkStarted || phase == inventory.WalkEnded) && said[walkDeclaredRead] == nil, nil
|
||||
}
|
||||
|
||||
// declaredTouches are the machines changed that a walk's touches do not name, each a declared touch. Pure; sorted.
|
||||
func declaredTouches(touches []walkTouch, changed []string) []walkTouch {
|
||||
named := map[string]bool{}
|
||||
for _, t := range touches {
|
||||
named[t.Machine] = true
|
||||
}
|
||||
var out []walkTouch
|
||||
for _, n := range changed {
|
||||
if !named[n] {
|
||||
out = append(out, walkTouch{Machine: n, Modules: []string{}, Declared: true})
|
||||
named[n] = true
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Machine < out[j].Machine })
|
||||
return out
|
||||
}
|
||||
|
||||
// sayDeclared says a walk's start or end again with its declared touches, when it has any. What cannot be read is
|
||||
// said on the controller's log: the walk is then held as before, never less.
|
||||
func sayDeclared(ctx context.Context, bus link.Bus, open *stores, body []byte) {
|
||||
if _, wanted, _ := toSayAgain(body); !wanted {
|
||||
return // no start or end, or said again already: nothing to compare
|
||||
}
|
||||
comparing, cancel := context.WithTimeout(ctx, declaredWithin)
|
||||
defer cancel()
|
||||
changed, err := changedDeclarations(comparing, open)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "plan-moved: which machines' declarations the walk changes cannot be read: %v\n", err)
|
||||
return
|
||||
}
|
||||
again, ok, err := withDeclared(body, changed)
|
||||
if err != nil || !ok {
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "plan-moved: the walk cannot be said again with its declared machines: %v\n", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
stating, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer stop()
|
||||
if err := bus.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeyPlanMoved, again); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "plan-moved: the walk could not be said again with its declared machines: %v\n", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0505 §2: the machines whose declaration a walk's merges change are said as declared touches.
|
||||
|
||||
// The machines behind what they were last sent, and those sent one before that cannot be worked out now, are changed;
|
||||
// one never sent and not worked out is not.
|
||||
func TestAMachineBehindOrNoLongerComposableIsChanged(t *testing.T) {
|
||||
nodes := []inventory.Node{{Name: "ace"}, {Name: "g14"}, {Name: "novox"}, {Name: "shanks"}, {Name: "new"}}
|
||||
would := map[string]string{"ace": "d1", "g14": "d2", "novox": "d3"}
|
||||
behind := []inventory.Machine{{Node: "ace"}}
|
||||
sent := map[string]bool{"ace": true, "g14": true, "novox": true, "shanks": true}
|
||||
if got := changedOf(nodes, would, behind, sent); !reflect.DeepEqual(got, []string{"ace", "shanks"}) {
|
||||
t.Fatalf("changed: %v, want ace (behind) and shanks (no longer composes)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A walk's start said by plan-moved (walk_said.go's body), said again with the changed machines it does not touch as
|
||||
// declared touches, once.
|
||||
func TestAWalksStartIsSaidAgainWithTheMachinesItsMergesChange(t *testing.T) {
|
||||
body, err := json.Marshal(planMovedBody{Plan: inventory.Plan{ID: "plan-1", State: inventory.PlanBuilding},
|
||||
walkSaid: &walkSaid{Walk: "plan-1", Phase: inventory.WalkStarted, BudgetClass: "leaf", BoundSeconds: 600,
|
||||
Touches: []walkTouch{{Machine: "novox", Modules: []string{"gitea"}}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again, ok, err := withDeclared(body, []string{"novox", "shanks"})
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("not said again: %v %v", ok, err)
|
||||
}
|
||||
var said struct {
|
||||
ID string `json:"id"`
|
||||
Phase string `json:"phase"`
|
||||
Touches []walkTouch `json:"touches"`
|
||||
DeclaredRead bool `json:"declared_read"`
|
||||
}
|
||||
if err := json.Unmarshal(again, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []walkTouch{{Machine: "novox", Modules: []string{"gitea"}}, {Machine: "shanks", Modules: []string{}, Declared: true}}
|
||||
if said.ID != "plan-1" || said.Phase != inventory.WalkStarted || !said.DeclaredRead || !reflect.DeepEqual(said.Touches, want) {
|
||||
t.Fatalf("said again: %+v", said)
|
||||
}
|
||||
if _, ok, _ := withDeclared(again, []string{"ace"}); ok {
|
||||
t.Fatalf("a start said again was said a third time")
|
||||
}
|
||||
if _, ok, _ := withDeclared(body, []string{"novox"}); ok {
|
||||
t.Fatalf("said again with nothing it does not touch")
|
||||
}
|
||||
moved, _ := json.Marshal(inventory.Plan{ID: "plan-2", State: inventory.PlanBuilding})
|
||||
if _, ok, _ := withDeclared(moved, []string{"shanks"}); ok {
|
||||
t.Fatalf("a move inside a walk was said as its start")
|
||||
}
|
||||
}
|
||||
@@ -59,6 +59,9 @@ type walkTouch struct {
|
||||
// Carried names those of Modules that are there only because a moved tool runner or node-engine carries
|
||||
// them: what restarts with it, not what the walk builds.
|
||||
Carried []string `json:"carried,omitempty"`
|
||||
// Declared marks a machine the walk sends nothing to whose declaration its merges change (novox/hq ADR 0505 §2):
|
||||
// said again by the serving controller once it has compared every machine (walk_declared.go).
|
||||
Declared bool `json:"declared,omitempty"`
|
||||
}
|
||||
|
||||
// planMovedBody is `plan-moved`'s body: the walk whole, as before, and on its start and end what it touches.
|
||||
|
||||
@@ -264,3 +264,46 @@ func TestAWalkStoppedByHandAndRetriedEndsFailedWhenItFailsAgain(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0505 §2: a walk's start is said again, over the real bus and against the real store, with a machine it
|
||||
// sends nothing to whose declaration differs from what it was last sent (here: assigned a module and never pushed),
|
||||
// marked declared.
|
||||
func TestAWalksStartIsSaidAgainWithAMachineWhoseDeclarationChanged(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
next := planMovedHeard(t)
|
||||
register(t, open, catalogue.Manifest{Module: "app", Version: "1"})
|
||||
if _, err := assign(ctx, open, "anchor", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
core, err := js.Conn().JetStream()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := json.Marshal(planMovedBody{Plan: inventory.Plan{ID: "plan-505", State: inventory.PlanBuilding},
|
||||
walkSaid: &walkSaid{Walk: "plan-505", Phase: inventory.WalkStarted, BudgetClass: inventory.ClassLeaf,
|
||||
BoundSeconds: 360, Touches: []walkTouch{}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sayDeclared(ctx, link.OverNATS{Conn: js.Conn(), JS: core}, open, body)
|
||||
again := next(inventory.WalkStarted)
|
||||
raw, _ := json.Marshal(again["touches"])
|
||||
var touches []walkTouch
|
||||
_ = json.Unmarshal(raw, &touches)
|
||||
anchor := false
|
||||
for _, tc := range touches {
|
||||
anchor = anchor || (tc.Machine == "anchor" && tc.Declared && len(tc.Modules) == 0)
|
||||
if !tc.Declared {
|
||||
t.Fatalf("a machine the walk sends nothing to was said sent: %s", raw)
|
||||
}
|
||||
}
|
||||
if again["walk"] != "plan-505" || again[walkDeclaredRead] != true || !anchor {
|
||||
t.Fatalf("said again: %v", again)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ module github.com/novox/mesh-controller
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021173436-07a65927c362
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161324-8892a1176800
|
||||
github.com/jackc/pgx/v5 v5.10.0
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5 h1:E5stfIeCN6uYgA9TCv+4K8JgaURg3zQppvUgJLaDMP4=
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5/go.mod h1:4wyoCr6IIIIarEUuROThWDj7DAvJvIidqMjWcpSnOwE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021173436-07a65927c362 h1:Ud/YHmhlm0qdB45LyDvxq7l4x6Bg2Ye0siHK5JFE+kk=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021173436-07a65927c362/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161324-8892a1176800 h1:Q29osS06HHcXH1WXURDyITbOLnJG7XJYgCzmZD++Nws=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161324-8892a1176800/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
|
||||
@@ -46,6 +46,7 @@ func CheckStreams() []Stream {
|
||||
Retention: RetentionWorkQueue,
|
||||
MaxAge: 7 * 24 * 60 * 60,
|
||||
MaxMsgsPerSubject: 1,
|
||||
Direct: true,
|
||||
Why: "the checks of pull requests marked ready, taken after a delivery build and before any other check " +
|
||||
"(novox/hq ADR 0406 §3); one subject a pull request, the newest request kept, so a newer one replaces a " +
|
||||
"request still waiting",
|
||||
@@ -56,6 +57,7 @@ func CheckStreams() []Stream {
|
||||
Retention: RetentionWorkQueue,
|
||||
MaxAge: 7 * 24 * 60 * 60,
|
||||
MaxMsgsPerSubject: 1,
|
||||
Direct: true,
|
||||
Why: "every other check, taken last and in the order asked (novox/hq ADR 0406 §3); one subject a pull " +
|
||||
"request, the newest request kept; one left behind for a head already judged is dropped when taken",
|
||||
},
|
||||
@@ -98,6 +100,13 @@ func holderCheckGrants() []string {
|
||||
"$KV."+CheckRunningBucket+".>",
|
||||
"$JS.API.STREAM.INFO.KV_"+CheckRunningBucket,
|
||||
"$JS.API.DIRECT.GET.KV_"+CheckRunningBucket+".>")
|
||||
// And, to hold checks while a delivery builds (ADR 0483, issues 1687 and 1688): the bucket of the plans building, **read and
|
||||
// never written** (the controller's alone, so no `$KV.` subject of it is granted), and the oldest waiting ask of each check
|
||||
// queue, read by a direct get that takes nothing from the queue and so counts against no delivery of the ask.
|
||||
pub = append(pub, readGrants(PlansBuildingBucket, "")...)
|
||||
for _, w := range CheckWorkers() {
|
||||
pub = append(pub, "$JS.API.DIRECT.GET."+w.Stream, "$JS.API.DIRECT.GET."+w.Stream+".>")
|
||||
}
|
||||
return pub
|
||||
}
|
||||
|
||||
|
||||
@@ -40,3 +40,37 @@ func TestTheBuildAgentReadsAndWatchesThePullRequestsKeys(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1687 (ADR 0483 clause 2): a holder of the build seat reads the bucket of the plans building and writes none of it, and
|
||||
// reads the oldest waiting ask of each check queue by a direct get, which takes nothing from the queue.
|
||||
func TestTheBuildSeatsHoldersReadThePlansBuildingAndWriteNone(t *testing.T) {
|
||||
seat := Seat{Name: TheBuildSeat, Scope: "node", Accepts: []string{"build"}, Emits: []string{"built", "started"}}
|
||||
p := Principal{Kind: KindModule, Node: "one", Module: "build-agent", Holds: []Seat{seat}, PasswordHash: "x"}
|
||||
perms, err := PermissionsFor(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stream := "KV_" + PlansBuildingBucket
|
||||
for _, s := range []string{
|
||||
"$JS.API.STREAM.INFO." + stream,
|
||||
"$JS.API.DIRECT.GET." + stream + ".>",
|
||||
"$JS.API.CONSUMER.CREATE." + stream + ".>",
|
||||
"$JS.API.DIRECT.GET.CHECKS_READY",
|
||||
"$JS.API.DIRECT.GET.CHECKS_OTHER",
|
||||
} {
|
||||
has(t, perms.Publish, s)
|
||||
}
|
||||
for _, s := range perms.Publish {
|
||||
if SubjectsOverlap(s, "$KV."+PlansBuildingBucket+".plan-1") {
|
||||
t.Errorf("a holder of the build seat may write %s through %q", PlansBuildingBucket, s)
|
||||
}
|
||||
}
|
||||
// And the controller alone writes it, as the writers table says.
|
||||
for _, row := range WritersTable {
|
||||
for _, s := range row.Subjects {
|
||||
if s == "$KV."+PlansBuildingBucket+".>" && row.Writes(p, "$KV."+PlansBuildingBucket+".plan-1") {
|
||||
t.Errorf("%q lets the build seat's holder write the plans building", row.State)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -63,6 +63,26 @@ var (
|
||||
// repository: the controller reads the ask's warrant from the router and accepts only the operator's Approve of
|
||||
// those heads. Written by the controller alone, from the announcement, when it holds the lease.
|
||||
DeliveryMergesBucket = BucketName(ControllerSeat, "delivery-merges")
|
||||
// PlansBuildingBucket holds one key per delivery plan in its build phase (novox/hq ADR 0483 clause 2, issue 1687): the key
|
||||
// is `<plan id>` (`plan-<digits>`), its value the plan, the time its build phase began and the time of the last refresh.
|
||||
// The controller (lease holder) puts it when the plan's first build is asked, puts it again about every thirty seconds
|
||||
// while the plan builds, and deletes it when the phase ends (every build of the last tier built, or the plan fails, stops
|
||||
// or waits on the operator); the bucket's age, PlansBuildingKeptFor, is how long a key outlives a controller that
|
||||
// stopped. **Read by the build seat's holders and the checker, written by the controller alone**: a holder takes no new
|
||||
// check while any key is there, but for one that waited past ADR 0483's bound. A reader lists the keys and reads none of
|
||||
// their values to decide that (`link.BuildingPlans`).
|
||||
PlansBuildingBucket = BucketName(ControllerSeat, PlansBuilding)
|
||||
)
|
||||
|
||||
// PlansBuilding is the local name of the bucket of the plans building, as a reader names it in `reads`
|
||||
// (`mesh-controller.plans-building`).
|
||||
const PlansBuilding = "plans-building"
|
||||
|
||||
// PlansBuildingKeptFor is the age of a plan-building key: the controller puts it again at PlansBuildingRefresh, so a
|
||||
// controller that stopped leaves checks held at most this long (ADR 0483 clause 2).
|
||||
const (
|
||||
PlansBuildingKeptFor = 2 * time.Minute
|
||||
PlansBuildingRefresh = 30 * time.Second
|
||||
)
|
||||
|
||||
// DeliveryMergesKeptFor is how long a merge mesh-delivery announced is kept: a week, as the events are.
|
||||
@@ -84,6 +104,9 @@ var (
|
||||
AssignmentsBucket = BucketName(ControllerSeat, MirrorAssignments)
|
||||
DeclaredConfigBucket = BucketName(ControllerSeat, MirrorDeclaredConfig)
|
||||
PredefinedBucket = BucketName(ControllerSeat, MirrorPredefined)
|
||||
// DeclaredPackagesBucket holds, keyed by the node, each package the modules assigned to it declare, present or absent,
|
||||
// with the declaring modules: what the pacman module's verbs read to say a package is declared (ADR 0507 §5).
|
||||
DeclaredPackagesBucket = BucketName(ControllerSeat, MirrorDeclaredPackages)
|
||||
)
|
||||
|
||||
// The mirrors' local names, as a reader names them in `reads` after `mesh-controller.`.
|
||||
@@ -91,6 +114,8 @@ const (
|
||||
MirrorAssignments = "assignments"
|
||||
MirrorDeclaredConfig = "declared-config"
|
||||
MirrorPredefined = "predefined"
|
||||
// MirrorDeclaredPackages is the local name of the packages each node's modules declare (ADR 0507 §5, issue 1840/S3).
|
||||
MirrorDeclaredPackages = "declared-packages"
|
||||
)
|
||||
|
||||
// The chain's buckets (novox/hq ADR 0408, issue 1089): builds, rollouts and assignments are one chain kept on the bus
|
||||
@@ -133,7 +158,7 @@ func ChainBuckets() []string { return []string{BuildsBucket, RolloutsBucket, Ass
|
||||
// ControllerMirrors are the local names of the controller's buckets a module may read: the record mirrors and the
|
||||
// chain's buckets.
|
||||
func ControllerMirrors() []string {
|
||||
return []string{MirrorAssignments, MirrorDeclaredConfig, MirrorPredefined, ChainBuilds, ChainRollouts, ChecksRunning}
|
||||
return []string{MirrorAssignments, MirrorDeclaredConfig, MirrorPredefined, ChainBuilds, ChainRollouts, ChecksRunning, PlansBuilding, MirrorDeclaredPackages}
|
||||
}
|
||||
|
||||
// MirrorsWrittenByAnother are the controller's mirrors whose writer is not the controller, each with its reason: the rule
|
||||
@@ -202,16 +227,17 @@ func IsControllerBucket(bucket string) bool {
|
||||
return bucket == CallsBucket || bucket == HandActsBucket || bucket == ConditionsBucket ||
|
||||
bucket == ConditionHistoryBucket || bucket == LeaseBucket || bucket == AskedBucket ||
|
||||
bucket == AssignmentsBucket || bucket == DeclaredConfigBucket || bucket == PredefinedBucket ||
|
||||
bucket == DeclaredPackagesBucket ||
|
||||
bucket == MergeWindowBucket || bucket == RolloutSilentBucket || bucket == CheckRunningBucket ||
|
||||
bucket == DeliveryMergesBucket || bucket == BuildsBucket || bucket == RolloutsBucket ||
|
||||
bucket == delivery.HaveBucket
|
||||
bucket == PlansBuildingBucket || bucket == delivery.HaveBucket
|
||||
}
|
||||
|
||||
// ControllerBuckets are the controller's own buckets, in the order they are asserted.
|
||||
func ControllerBuckets() []string {
|
||||
return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket, AskedBucket,
|
||||
AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket, MergeWindowBucket, RolloutSilentBucket, CheckRunningBucket,
|
||||
DeliveryMergesBucket, BuildsBucket, RolloutsBucket, delivery.HaveBucket}
|
||||
AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket, DeclaredPackagesBucket, MergeWindowBucket, RolloutSilentBucket, CheckRunningBucket,
|
||||
DeliveryMergesBucket, PlansBuildingBucket, BuildsBucket, RolloutsBucket, delivery.HaveBucket}
|
||||
}
|
||||
|
||||
// ControllerBucketsAsserter is what raising the controller's buckets needs of a connection.
|
||||
@@ -354,6 +380,22 @@ func (j *JetStream) EnsureControllerBuckets() error {
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", CheckRunningBucket, err)
|
||||
}
|
||||
// **The plans building: keys that age** (ADR 0483 clause 2, issue 1687). One message a key, each refresh a new one, so the
|
||||
// age of the bucket is how long a key outlives a controller that stopped refreshing it.
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: PlansBuildingBucket,
|
||||
Description: "the delivery plans in their build phase (novox/hq ADR 0483): `<plan id>` holds the plan and when its " +
|
||||
"build phase began; put when the plan's first build is asked and about every thirty seconds, deleted when the " +
|
||||
"phase ends, gone within two minutes of a controller that stopped; read by the build seat's holders, which " +
|
||||
"take no new check while a key is there; written by the controller alone",
|
||||
History: 1,
|
||||
TTL: PlansBuildingKeptFor,
|
||||
MaxValueSize: 4 << 10,
|
||||
MaxBytes: 8 << 20,
|
||||
Storage: jetstream.FileStorage,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", PlansBuildingBucket, err)
|
||||
}
|
||||
// **The merges mesh-delivery announced: keys that age** (ADR 0420 §7, issue 1025). One message a key; a week.
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: DeliveryMergesBucket,
|
||||
@@ -410,6 +452,9 @@ func (j *JetStream) EnsureControllerBuckets() error {
|
||||
{PredefinedBucket, "the list of predefined keys the mesh published, the newest trunk build's of the " +
|
||||
"catalogue, one key `list` (novox/hq ADR 0348 §7, design 57 K3): written by the controller alone from " +
|
||||
"its store when a newer list is kept, read through `reads`"},
|
||||
{DeclaredPackagesBucket, "each package the modules assigned to a node declare, present or absent, with the " +
|
||||
"declaring modules, keyed by the node (novox/hq ADR 0507 §5): written by the controller alone when it " +
|
||||
"composes the node, only on a change, read through `reads`"},
|
||||
} {
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: m.bucket,
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Issue 1840/S3 (ADR 0507 §5), 1840/AC5: a module whose manifest reads mesh-controller.declared-packages may name it in
|
||||
// `reads`, and reads the bucket and writes none; the controller writes it, so it is no mirror written by another, and it is
|
||||
// one of the controller's own buckets, asserted at its start.
|
||||
func TestAModuleReadingDeclaredPackagesReadsAndWritesNone(t *testing.T) {
|
||||
if !slices.Contains(ControllerMirrors(), "declared-packages") ||
|
||||
BucketName(ControllerSeat, MirrorDeclaredPackages) != DeclaredPackagesBucket {
|
||||
t.Fatalf("a module cannot name %s in `reads`: %v", DeclaredPackagesBucket, ControllerMirrors())
|
||||
}
|
||||
if _, other := MirrorsWrittenByAnother[MirrorDeclaredPackages]; other {
|
||||
t.Error("the controller writes the declared packages itself")
|
||||
}
|
||||
if !IsControllerBucket(DeclaredPackagesBucket) || !slices.Contains(ControllerBuckets(), DeclaredPackagesBucket) {
|
||||
t.Errorf("%s is not among the controller's buckets", DeclaredPackagesBucket)
|
||||
}
|
||||
perms, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "pacman", PasswordHash: "x",
|
||||
Reads: []string{"mesh-controller.declared-packages"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stream := "KV_" + DeclaredPackagesBucket
|
||||
has(t, perms.Publish, "$JS.API.STREAM.INFO."+stream)
|
||||
has(t, perms.Publish, "$JS.API.DIRECT.GET."+stream+".>")
|
||||
has(t, perms.Publish, "$JS.API.CONSUMER.CREATE."+stream+".>")
|
||||
for _, p := range perms.Publish {
|
||||
if SubjectsOverlap(p, "$KV."+DeclaredPackagesBucket+".one") {
|
||||
t.Errorf("the reading module may write %s through %q", DeclaredPackagesBucket, p)
|
||||
}
|
||||
}
|
||||
controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
has(t, controller.Publish, "$KV."+DeclaredPackagesBucket+".>")
|
||||
}
|
||||
@@ -2,10 +2,7 @@ package broker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
@@ -112,25 +109,6 @@ func DialPinned(url, fingerprint string, opts ...nats.Option) (*JetStream, error
|
||||
return Dial(url, opts...)
|
||||
}
|
||||
|
||||
// PinnedToFingerprint accepts exactly the certificate with this SHA-256 and no other.
|
||||
func PinnedToFingerprint(want string) *tls.Config {
|
||||
return &tls.Config{
|
||||
InsecureSkipVerify: true, //nolint:gosec // replaced by the pin below, which is stricter
|
||||
MinVersion: tls.VersionTLS12,
|
||||
VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
|
||||
if len(rawCerts) == 0 {
|
||||
return errors.New("the bus presented no certificate")
|
||||
}
|
||||
sum := sha256.Sum256(rawCerts[0])
|
||||
got := "sha256:" + hex.EncodeToString(sum[:])
|
||||
if got != want {
|
||||
return fmt.Errorf("the bus presented a certificate this mesh does not know (%s…), expected %s…", got[:23], want[:23])
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Conn is the connection itself, for what the mesh keeps off JetStream on purpose — a heartbeat,
|
||||
// a tool call — where a lost message is answered by the next one or by a timeout the caller
|
||||
// already handles (design 25 §3).
|
||||
|
||||
@@ -98,17 +98,6 @@ func (l Log) Config() jetstream.StreamConfig {
|
||||
}
|
||||
}
|
||||
|
||||
// LogIssued is one log an assignment may reach, by the name its module uses for it (novox/hq ADR
|
||||
// 0297): its stream, the subject its entries go under, and whether it may append. Only the owner's
|
||||
// instances are issued a log, so Writes is always true today; it is said so the runtime need not
|
||||
// assume it.
|
||||
type LogIssued struct {
|
||||
Name string `json:"name"`
|
||||
Stream string `json:"stream"`
|
||||
Subject string `json:"subject"`
|
||||
Writes bool `json:"writes"`
|
||||
}
|
||||
|
||||
// logsIssuedFor is every log a module's code may reach, as its membership lists them: its own.
|
||||
func logsIssuedFor(d Declared) []LogIssued {
|
||||
var out []LogIssued
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -16,81 +15,6 @@ import (
|
||||
// grant is the same composition read the other way. The shape issued today is the shape the mesh
|
||||
// already had, so nothing moves when a membership first arrives; only who decides it moves.
|
||||
|
||||
// Membership is one assignment's subjects: what this instance of a module on this machine serves,
|
||||
// and what it may reach.
|
||||
type Membership struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
// Serves is every address a tool of this instance answers on. `{tool}` stands for the tool's
|
||||
// own name, which the module knows and the mesh does not need to: the mesh issues the address,
|
||||
// the runtime fills the name. An address with a queue is shared with the module's other
|
||||
// instances, and the bus hands each call to one of them; an address without is this instance's.
|
||||
Serves []Served `json:"serves"`
|
||||
// Seats is every verb of a seat this instance holds, at the subject the seat's callers use.
|
||||
Seats []SeatServed `json:"seats,omitempty"`
|
||||
// Emits is where an event of this module lands; `{event}` stands for the event's name.
|
||||
Emits string `json:"emits"`
|
||||
// Reaches is each tool this module may call, `<module>.<tool>`, to the subjects that reach it:
|
||||
// the first is whichever instance answers, when the mesh issued one; the rest name a machine.
|
||||
Reaches map[string][]string `json:"reaches,omitempty"`
|
||||
// Tools is where this instance answers what it serves — the runtime's one verb of its own.
|
||||
Tools string `json:"tools"`
|
||||
// Receives is what this assignment is given for each requirement it receives, by requirement:
|
||||
// the contributions of every module that asked for it, as the catalogue composed them (novox/hq
|
||||
// ADR 0167). The same list its received file is written from, so the two cannot disagree; a
|
||||
// requirement nobody contributed to is an empty list, never absent. Kept as JSON because the
|
||||
// catalogue owns the shape of a contribution and the bus only carries it.
|
||||
Receives map[string]json.RawMessage `json:"receives,omitempty"`
|
||||
// Mesh is every machine's address on the private network — what a rule saying "from the mesh"
|
||||
// resolves to in the packet filter, issued here from the same list (novox/hq ADR 0167). A
|
||||
// module that must tell the mesh from the world, the route proxy serving an internal name, reads
|
||||
// it here rather than keeping a definition of its own.
|
||||
Mesh []string `json:"mesh,omitempty"`
|
||||
// State is every bucket this module's code may reach, by the name it uses for each, and whether
|
||||
// it may write it (novox/hq ADR 0201): the runtime answers a bundle's state verbs from this list
|
||||
// and refuses, with the reason, what is not on it — the bus enforces only the union over every
|
||||
// module on the machine.
|
||||
State []StateIssued `json:"state,omitempty"`
|
||||
// Logs is every log this module's code may reach, by the name it uses for each (novox/hq ADR 0297):
|
||||
// its own and no other's, since a log is its owner's alone. The runtime answers a bundle's log verbs
|
||||
// from this list and refuses, with the reason, a log not on it.
|
||||
Logs []LogIssued `json:"logs,omitempty"`
|
||||
// SeatTraffic is what this module's code may submit, say, hear, take, ask, answer and read on seats
|
||||
// that name their caller or their kind (novox/hq ADR 0259 §3). The runtime carrying the module
|
||||
// publishes, takes and answers for it only what is listed here: the bus enforces only the union
|
||||
// over every module on the machine, so one module's code reaching another's name or kind through
|
||||
// the runtime is the runtime's to refuse.
|
||||
SeatTraffic *SeatTraffic `json:"seat-traffic,omitempty"`
|
||||
// Streams are this module's own streams (novox/hq ADR 0363 §2): an event one of them carries is published
|
||||
// on that stream's subject, never on Emits. Hears are the module streams it hears, each through a durable
|
||||
// consumer on that stream the runtime binds beside the one on EVENTS (§8).
|
||||
Streams []StreamIssued `json:"streams,omitempty"`
|
||||
Hears []HeardIssued `json:"hears,omitempty"`
|
||||
// Waits are the event families a session may wait on through the mesh MCP server, the tool runner's
|
||||
// alone (novox/hq ADR 0367 §2): the subjects its account is granted, under the names a session asks for.
|
||||
// The tool runner refuses any family not listed here.
|
||||
Waits []WaitIssued `json:"waits,omitempty"`
|
||||
}
|
||||
|
||||
// Served is one address a tool is answered on.
|
||||
type Served struct {
|
||||
Subject string `json:"subject"`
|
||||
Queue string `json:"queue,omitempty"`
|
||||
}
|
||||
|
||||
// SeatServed is one verb of a held seat, where its callers ask.
|
||||
type SeatServed struct {
|
||||
Seat string `json:"seat"`
|
||||
Verb string `json:"verb"`
|
||||
Subject string `json:"subject"`
|
||||
}
|
||||
|
||||
// MembershipSubject is the one address a runtime derives for itself: where its own membership is
|
||||
// published, from the two names its credential carries. Everything else is in the membership.
|
||||
func MembershipSubject(node, module string) string {
|
||||
return "mesh.assignment." + node + "." + module
|
||||
}
|
||||
|
||||
// Placements is where every module runs, for deciding which instance answers for the module.
|
||||
type Placements struct {
|
||||
// Nodes is each module's machines.
|
||||
|
||||
@@ -260,21 +260,6 @@ func ownStreamGrants(module string, streams []ModuleStream) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// StreamIssued is one of a module's own streams as its membership names it (ADR 0363 §2): the events it
|
||||
// carries, as the bus spells them, and the subject an event of them is published on, `{event}` the event.
|
||||
type StreamIssued struct {
|
||||
Stream string `json:"stream"`
|
||||
Events []string `json:"events"`
|
||||
Subject string `json:"subject"`
|
||||
}
|
||||
|
||||
// HeardIssued is one module stream an assignment hears, as its membership names it (ADR 0363 §8): the
|
||||
// stream and the durable consumer on it the runtime binds.
|
||||
type HeardIssued struct {
|
||||
Stream string `json:"stream"`
|
||||
Consumer string `json:"consumer"`
|
||||
}
|
||||
|
||||
// streamsIssuedFor is every stream of its own a module's membership names.
|
||||
func streamsIssuedFor(d Declared) []StreamIssued {
|
||||
var out []StreamIssued
|
||||
|
||||
+13
-3
@@ -424,6 +424,16 @@ var DegradableGrants = []DegradableGrant{
|
||||
Reason: "refused: the controller asserts this bucket through the API and writes none of its keys — the build seat's " +
|
||||
"holders do (ADR 0406 §3, issue 974)",
|
||||
},
|
||||
{
|
||||
Subject: "$KV." + PlansBuildingBucket + ".>",
|
||||
Reason: "refused: the controller then says no plan builds, and no check is held while a delivery builds — a check run " +
|
||||
"wrongly costs a delivery a place, a check held wrongly costs a pull request its turn (ADR 0483 clause 2, issue 1687)",
|
||||
},
|
||||
{
|
||||
Subject: "$KV." + DeclaredPackagesBucket + ".>",
|
||||
Reason: "carried only at genesis: the controller composes its own permissions and pushes them to the bus at its " +
|
||||
"first push, and keeps the packages each node declares only after that, holding the lease (ADR 0507 §5, issue 1840)",
|
||||
},
|
||||
{
|
||||
Subject: "mesh.seat." + ControllerSeat + ".event.sent",
|
||||
Reason: "refused: the controller says a tier sent to a machine only for outsiders, and a send that could not " +
|
||||
@@ -709,7 +719,7 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// And asking the operator (novox/hq ADR 0259): an ask and its cancel under its own name, its warrants
|
||||
// heard under its own name, the record of its asks read under its own name — as any user of the seat,
|
||||
// derived the same way, from the seat its holder declares.
|
||||
tp, ts := SeatTrafficOf(ControllerSeat, nil, p.Uses, nil).grants()
|
||||
tp, ts := seatTrafficGrants(SeatTrafficOf(ControllerSeat, nil, p.Uses, nil))
|
||||
pub = append(pub, tp...)
|
||||
sub = append(sub, ts...)
|
||||
// And asks who answers (novox/hq to-be 45 §4, D3): the self-check finds every seat's holder by
|
||||
@@ -1042,7 +1052,7 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
|
||||
// 6. Its traffic on seats that name their caller or their kind, ask proofs or keep records
|
||||
// (novox/hq ADR 0259 §3).
|
||||
tp, ts := SeatTrafficOf(p.Module, p.Holds, p.Uses, p.Watches).grants()
|
||||
tp, ts := seatTrafficGrants(SeatTrafficOf(p.Module, p.Holds, p.Uses, p.Watches))
|
||||
pub = append(pub, tp...)
|
||||
sub = append(sub, ts...)
|
||||
|
||||
@@ -1166,7 +1176,7 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// not publish under another's name or kind through it is the runtime's to keep, from the seat
|
||||
// traffic each module's membership lists.
|
||||
for _, d := range p.Carries {
|
||||
tp, ts := SeatTrafficOf(d.Module, d.Holds, d.Uses, d.Watches).grants()
|
||||
tp, ts := seatTrafficGrants(SeatTrafficOf(d.Module, d.Holds, d.Uses, d.Watches))
|
||||
pub = append(pub, tp...)
|
||||
sub = append(sub, ts...)
|
||||
}
|
||||
|
||||
@@ -35,39 +35,6 @@ func OnNATS() (address string, on bool, err error) {
|
||||
return address, true, nil
|
||||
}
|
||||
|
||||
// CredentialIn reads the user and password out of a bus address, and the address without them.
|
||||
//
|
||||
// The controller's own credential arrives in its address, the way the old bus's does. Split out so the
|
||||
// controller can record a hash of what it is actually using: its user is created by the installer at a
|
||||
// bootstrap password, before the controller exists to mint one, and a composition that left itself out
|
||||
// would produce a bus the writer cannot connect to.
|
||||
func CredentialIn(address string) (user, password, bare string) {
|
||||
at := strings.LastIndex(address, "@")
|
||||
if at < 0 {
|
||||
return "", "", address
|
||||
}
|
||||
scheme := ""
|
||||
rest := address[:at]
|
||||
if i := strings.Index(rest, "://"); i >= 0 {
|
||||
scheme, rest = rest[:i+3], rest[i+3:]
|
||||
}
|
||||
user, password, _ = strings.Cut(rest, ":")
|
||||
return user, password, scheme + address[at+1:]
|
||||
}
|
||||
|
||||
// BareAddress is a bus address with any credential stripped, for something that only needs to know
|
||||
// whether a server is answering there.
|
||||
func BareAddress(address string) string {
|
||||
_, _, bare := CredentialIn(address)
|
||||
if bare == "" {
|
||||
return address
|
||||
}
|
||||
if strings.Contains(bare, "://") {
|
||||
return bare
|
||||
}
|
||||
return "nats://" + bare
|
||||
}
|
||||
|
||||
// BusAddress is where the mesh's bus is, with this process's credential, and refuses to be empty:
|
||||
// there is one bus, and a control plane without it can hold records and answer nothing (novox/hq
|
||||
// ADR 0131, design 28 task 5.5).
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Which bus the mesh is on is one fact, and being told about both is refused.
|
||||
//
|
||||
|
||||
// The controller's own credential arrives in its address, and has to be readable out of it — its user
|
||||
// is created by the installer at a bootstrap password, before the controller exists to mint one.
|
||||
func TestACredentialIsReadOutOfABusAddress(t *testing.T) {
|
||||
for _, c := range []struct{ in, user, password, bare string }{
|
||||
{"nats://controller:secret@127.0.0.1:4222", "controller", "secret", "nats://127.0.0.1:4222"},
|
||||
{"controller:secret@127.0.0.1:4222", "controller", "secret", "127.0.0.1:4222"},
|
||||
{"nats://127.0.0.1:4222", "", "", "nats://127.0.0.1:4222"},
|
||||
{"127.0.0.1:4222", "", "", "127.0.0.1:4222"},
|
||||
// A password containing an at-sign: split on the last one, or the address becomes part of the
|
||||
// credential and the connection goes somewhere nobody named.
|
||||
{"nats://controller:a@b@127.0.0.1:4222", "controller", "a@b", "nats://127.0.0.1:4222"},
|
||||
} {
|
||||
user, password, bare := CredentialIn(c.in)
|
||||
if user != c.user || password != c.password || bare != c.bare {
|
||||
t.Errorf("%q read as %q/%q at %q; wanted %q/%q at %q",
|
||||
c.in, user, password, bare, c.user, c.password, c.bare)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
//go:build integration
|
||||
|
||||
package broker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
)
|
||||
|
||||
// **A holder of the build seat reads the plans building and the server refuses its write** (novox/hq ADR 0483 clause 2, issue
|
||||
// 1687): on a real server whose users are composed as the controller composes them, the controller makes the bucket and puts a
|
||||
// plan's key; the build agent's credential lists the keys and gets one, and its put of a key is refused with a permissions
|
||||
// violation. It also reads the first waiting ask of a check queue by a direct get, which takes nothing from the queue.
|
||||
func TestNatsTheBuildSeatsHoldersReadThePlansBuildingAndTheServerRefusesTheirWrite(t *testing.T) { // 1687
|
||||
hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seat := Seat{Name: TheBuildSeat, Scope: "node", Accepts: []string{"build"}, Emits: []string{"built", "started"}}
|
||||
users, err := Users(Records{Nodes: []string{"alpha"}, Assigned: map[string][]Declared{
|
||||
"alpha": {{Module: RuntimeModule}, {Module: "build-agent", Holds: []Seat{seat}}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := range users {
|
||||
users[i].PasswordHash = string(hash)
|
||||
}
|
||||
accounts, err := ComposeAccounts(users)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conf := filepath.Join(t.TempDir(), "bus.conf")
|
||||
if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+
|
||||
`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts, err := server.ProcessConfigFile(conf)
|
||||
if err != nil {
|
||||
t.Fatalf("the composed accounts do not parse: %v", err)
|
||||
}
|
||||
opts.NoLog, opts.NoSigs = true, true
|
||||
s, err := server.NewServer(opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go s.Start()
|
||||
if !s.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(s.Shutdown)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
connect := func(user string) (*nats.Conn, chan error) {
|
||||
t.Helper()
|
||||
refused := make(chan error, 8)
|
||||
nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, "pw"), nats.CustomInboxPrefix("_INBOX."+user),
|
||||
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refused <- err }))
|
||||
if err != nil {
|
||||
t.Fatalf("%s cannot connect: %v", user, err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
return nc, refused
|
||||
}
|
||||
controller, _ := connect(ControllerName)
|
||||
cj := OnConn(controller)
|
||||
if err := AssertMeshStreams(cj); err != nil {
|
||||
t.Fatalf("the controller cannot make the streams: %v", err)
|
||||
}
|
||||
if err := cj.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatalf("the controller cannot make its buckets: %v", err)
|
||||
}
|
||||
cjs, _ := jetstream.New(controller)
|
||||
ckv, err := cjs.KeyValue(ctx, PlansBuildingBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ckv.Put(ctx, "plan-1", []byte(`{"plan":"plan-1"}`)); err != nil {
|
||||
t.Fatalf("the controller cannot put a plan's key: %v", err)
|
||||
}
|
||||
|
||||
holder, refused := connect("alpha.build-agent")
|
||||
hjs, err := jetstream.New(holder)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv, err := hjs.KeyValue(ctx, PlansBuildingBucket)
|
||||
if err != nil {
|
||||
t.Fatalf("a holder of the build seat cannot bind %s: %v", PlansBuildingBucket, err)
|
||||
}
|
||||
keys, err := kv.Keys(ctx)
|
||||
if err != nil || len(keys) != 1 || keys[0] != "plan-1" {
|
||||
t.Fatalf("the holder lists %v (%v), want plan-1", keys, err)
|
||||
}
|
||||
if e, err := kv.Get(ctx, "plan-1"); err != nil || e == nil {
|
||||
t.Fatalf("the holder cannot get the key: %v", err)
|
||||
}
|
||||
// Its write is refused by the server.
|
||||
if err := holder.Publish("$KV."+PlansBuildingBucket+".plan-2", []byte(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = holder.Flush()
|
||||
select {
|
||||
case err := <-refused:
|
||||
if !errors.Is(err, nats.ErrPermissionViolation) {
|
||||
t.Errorf("the server said %v, want a permissions violation", err)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Error("the server did not refuse the holder's put of a plan's key")
|
||||
}
|
||||
if _, err := ckv.Get(ctx, "plan-2"); !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
t.Errorf("the refused put left a key: %v", err)
|
||||
}
|
||||
|
||||
// And the direct get of a check queue's first waiting ask, which is granted.
|
||||
reply, err := holder.RequestWithContext(ctx, "$JS.API.DIRECT.GET."+ChecksOtherStream, []byte(`{"seq":1,"next_by_subj":"`+
|
||||
CheckSubjectPrefix(TheBuildSeat, false)+`>"}`))
|
||||
if err != nil {
|
||||
t.Fatalf("the holder's direct get of %s was not answered: %v", ChecksOtherStream, err)
|
||||
}
|
||||
if reply.Header.Get("Status") != "404" {
|
||||
t.Errorf("an empty queue answered %q, want 404", reply.Header.Get("Status"))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Issue 1687 (ADR 0483 clause 2): a module whose manifest reads mesh-controller.plans-building may name it in `reads`, and
|
||||
// reads the bucket and writes none; the controller writes it, so it is no mirror written by another.
|
||||
func TestAModuleReadingPlansBuildingReadsAndWritesNone(t *testing.T) {
|
||||
if !slices.Contains(ControllerMirrors(), "plans-building") || BucketName(ControllerSeat, PlansBuilding) != PlansBuildingBucket {
|
||||
t.Fatalf("a module cannot name %s in `reads`: %v", PlansBuildingBucket, ControllerMirrors())
|
||||
}
|
||||
if _, other := MirrorsWrittenByAnother[PlansBuilding]; other {
|
||||
t.Error("the controller writes the plans building itself")
|
||||
}
|
||||
perms, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "mesh-delivery-checker", PasswordHash: "x",
|
||||
Reads: []string{"mesh-controller.plans-building"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stream := "KV_" + PlansBuildingBucket
|
||||
has(t, perms.Publish, "$JS.API.STREAM.INFO."+stream)
|
||||
has(t, perms.Publish, "$JS.API.DIRECT.GET."+stream+".>")
|
||||
for _, p := range perms.Publish {
|
||||
if SubjectsOverlap(p, "$KV."+PlansBuildingBucket+".plan-1") {
|
||||
t.Errorf("the reading module may write %s through %q", PlansBuildingBucket, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package broker
|
||||
|
||||
import "testing"
|
||||
|
||||
// 1840/AC9: a process account declaring the state `x` may watch and list the keys of its module's own bucket (a consumer
|
||||
// created and deleted on KV_<module>_x), read and write it, and has no grant on another module's bucket or on a mirror of the
|
||||
// controller's, whatever it is named after.
|
||||
func TestAProcessMayWatchItsOwnStateAndNoOthers(t *testing.T) {
|
||||
p := Principal{Kind: KindProcess, Node: "novox", Module: "mesh-mail", Process: "send", State: []string{"x"}}
|
||||
got := perms(t, p)
|
||||
for _, s := range []string{
|
||||
"$JS.API.CONSUMER.CREATE.KV_mesh-mail_x",
|
||||
"$JS.API.CONSUMER.CREATE.KV_mesh-mail_x.watcher",
|
||||
"$JS.API.CONSUMER.CREATE.KV_mesh-mail_x.watcher.$KV.mesh-mail_x.>",
|
||||
"$JS.API.CONSUMER.DELETE.KV_mesh-mail_x.watcher",
|
||||
"$JS.API.DIRECT.GET.KV_mesh-mail_x.$KV.mesh-mail_x.key",
|
||||
"$JS.API.STREAM.INFO.KV_mesh-mail_x",
|
||||
"$JS.FC.KV_mesh-mail_x.watcher",
|
||||
"$KV.mesh-mail_x.key",
|
||||
} {
|
||||
if !allowed(got.Publish, s) {
|
||||
t.Errorf("the process may not publish %s, so it cannot watch or list its own state", s)
|
||||
}
|
||||
}
|
||||
for _, other := range []string{"mesh-other_x", "mesh-mail_y", DeclaredPackagesBucket, AssignmentsBucket} {
|
||||
for _, s := range []string{
|
||||
"$JS.API.CONSUMER.CREATE.KV_" + other,
|
||||
"$JS.API.CONSUMER.CREATE.KV_" + other + ".watcher",
|
||||
"$JS.API.CONSUMER.CREATE.KV_" + other + ".watcher.$KV." + other + ".>",
|
||||
"$JS.API.CONSUMER.DELETE.KV_" + other + ".watcher",
|
||||
"$JS.API.DIRECT.GET.KV_" + other + ".$KV." + other + ".key",
|
||||
"$JS.API.STREAM.INFO.KV_" + other,
|
||||
"$JS.FC.KV_" + other + ".watcher",
|
||||
"$KV." + other + ".key",
|
||||
} {
|
||||
if allowed(got.Publish, s) {
|
||||
t.Errorf("the process may publish %s, on a bucket that is not its own state", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// The membership the mesh issues an assignment (novox/hq ADR 0160), its subject, its stream, the bare address and the
|
||||
// pin are mesh-sdk's, public so that a module's program outside this Go module, the route proxy, reads them too
|
||||
// (novox/hq ADR 0480 clause 4, issue 1633). This package keeps no second copy: what follows only names them where the
|
||||
// controller already did.
|
||||
type (
|
||||
Membership = membership.Membership
|
||||
Served = membership.Served
|
||||
SeatServed = membership.SeatServed
|
||||
StateIssued = membership.StateIssued
|
||||
LogIssued = membership.LogIssued
|
||||
Worker = membership.Worker
|
||||
SeatTraffic = membership.SeatTraffic
|
||||
KindHeld = membership.KindHeld
|
||||
StreamIssued = membership.StreamIssued
|
||||
HeardIssued = membership.HeardIssued
|
||||
WaitIssued = membership.WaitIssued
|
||||
)
|
||||
|
||||
// AssignmentsStream holds every assignment's membership, the newest per subject.
|
||||
const AssignmentsStream = membership.AssignmentsStream
|
||||
|
||||
var (
|
||||
// MembershipSubject is where an assignment's own membership is published.
|
||||
MembershipSubject = membership.Subject
|
||||
// CredentialIn reads the user and password out of a bus address, and the address without them.
|
||||
CredentialIn = membership.CredentialIn
|
||||
// BareAddress is a bus address with any credential stripped.
|
||||
BareAddress = membership.BareAddress
|
||||
// PinnedToFingerprint accepts exactly the certificate with this SHA-256 and no other.
|
||||
PinnedToFingerprint = process.PinnedToFingerprint
|
||||
)
|
||||
@@ -24,44 +24,6 @@ import "sort"
|
||||
// And one read: **a holder's records**, a bucket the seat names, read by each user under its own name only
|
||||
// (`$KV.<bucket>.<module>.>`), so an asker reads the state of its own asks and no other asker's.
|
||||
|
||||
// Worker is one durable consumer a holder pulls a seat's work from.
|
||||
type Worker struct {
|
||||
Stream string
|
||||
Consumer string
|
||||
Filter string
|
||||
}
|
||||
|
||||
// SeatTraffic is one module's seat traffic beyond tools. Publish and Subscribe are subject patterns, in the
|
||||
// server's wildcards; the runtime that carries the module checks a bundle's request against them, since
|
||||
// the runtime's own principal holds the union of every module it carries.
|
||||
type SeatTraffic struct {
|
||||
// Publish is what it submits (accepts of seats it uses), says (events of seats it holds) and asks
|
||||
// (proofs of seats it holds a kind of).
|
||||
Publish []string `json:"publish,omitempty"`
|
||||
// Subscribe is what it hears: events of seats it uses that are named by caller, events of seats it
|
||||
// watches, and accepts of seats it holds.
|
||||
Subscribe []string `json:"subscribe,omitempty"`
|
||||
// Answers is the proof subjects it answers, as a watcher of a kinded seat.
|
||||
Answers []string `json:"answers,omitempty"`
|
||||
// Workers are the work queues it takes from, as a holder.
|
||||
Workers []Worker `json:"workers,omitempty"`
|
||||
// Records is the direct-get subjects of the records it reads under its own name.
|
||||
Records []string `json:"records,omitempty"`
|
||||
// Kinds are the holders of every kinded bench it uses or watches, with what each promises: the one
|
||||
// account of which channel is which, and what it can carry, that the router judges an answer by. The
|
||||
// controller's, from the claims, never a channel's word (novox/hq ADR 0259 §5).
|
||||
Kinds []KindHeld `json:"kinds,omitempty"`
|
||||
}
|
||||
|
||||
// KindHeld is one kind of a kinded bench and who holds it.
|
||||
type KindHeld struct {
|
||||
Seat string `json:"seat"`
|
||||
Kind string `json:"kind"`
|
||||
Module string `json:"module"`
|
||||
Node string `json:"node"`
|
||||
Capabilities []string `json:"capabilities,omitempty"`
|
||||
}
|
||||
|
||||
// KindedBenches are the seats that may be kinded (ADR 0234 §2): making another is a decision, recorded.
|
||||
var KindedBenches = map[string]bool{"channel": true, "intake": true}
|
||||
|
||||
@@ -175,9 +137,9 @@ func SeatTrafficOf(module string, holds, uses, watches []Seat) SeatTraffic {
|
||||
return t
|
||||
}
|
||||
|
||||
// grants is the bus permissions seat traffic needs: the subjects themselves, and the JetStream API a
|
||||
// seatTrafficGrants is the bus permissions seat traffic needs: the subjects themselves, and the JetStream API a
|
||||
// worker is pulled and acknowledged through and a record is read through.
|
||||
func (t SeatTraffic) grants() (pub, sub []string) {
|
||||
func seatTrafficGrants(t SeatTraffic) (pub, sub []string) {
|
||||
pub = append(pub, t.Publish...)
|
||||
sub = append(sub, t.Subscribe...)
|
||||
sub = append(sub, t.Answers...)
|
||||
|
||||
@@ -186,16 +186,6 @@ func perMachineNames(buckets []Bucket) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// StateIssued is one bucket an assignment may reach, by the name its module uses for it: its own
|
||||
// state by the local name, another's as `<module>.<name>` (novox/hq ADR 0201). Key, when set, is the
|
||||
// one key it may reach (novox/hq ADR 0260): this machine's.
|
||||
type StateIssued struct {
|
||||
Name string `json:"name"`
|
||||
Bucket string `json:"bucket"`
|
||||
Writes bool `json:"writes,omitempty"`
|
||||
Key string `json:"key,omitempty"`
|
||||
}
|
||||
|
||||
// stateIssuedFor is every bucket a module's code may reach on a machine, as its membership lists them.
|
||||
func stateIssuedFor(d Declared, node string) []StateIssued {
|
||||
var out []StateIssued
|
||||
|
||||
@@ -80,9 +80,6 @@ const RestrictedMetadata = "mesh-restricted"
|
||||
// it under: never set by a mesh-internal work-queue stream (a seat's, a check's), only by Stream.WorkQueue.
|
||||
const WorkQueueMetadata = "mesh-work-queue"
|
||||
|
||||
// AssignmentsStream holds every assignment's membership, the newest per subject.
|
||||
const AssignmentsStream = "ASSIGNMENTS"
|
||||
|
||||
// What a durable consumer gave up on is kept (novox/hq issue 330, design 25 §3).
|
||||
//
|
||||
// **The server says it and keeps it; the controller copies it.** A consumer that handed a message over
|
||||
|
||||
+1
-1
@@ -29,7 +29,7 @@ accounts {
|
||||
]
|
||||
users = [
|
||||
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-controller.tool.push", "mesh.seat.mesh-controller.tool.upgrade", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_declared-packages.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_plans-building.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-controller.tool.push", "mesh.seat.mesh-controller.tool.upgrade", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
|
||||
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
|
||||
@@ -31,12 +31,6 @@ type Wait struct {
|
||||
Last string
|
||||
}
|
||||
|
||||
// WaitIssued is one family as a membership issues it: the name a session asks for, and its subject.
|
||||
type WaitIssued struct {
|
||||
Event string `json:"event"`
|
||||
Subject string `json:"subject"`
|
||||
}
|
||||
|
||||
// WaitSubject is where a family's events land. Each token is checked before it becomes a subject: a
|
||||
// wildcard, a dot inside a name or an empty token would widen the grant.
|
||||
func WaitSubject(w Wait) (string, error) {
|
||||
|
||||
@@ -177,6 +177,10 @@ var WritersTable = []WriterRow{
|
||||
KeptIn: "key-value " + CheckRunningBucket + ", aged out when not renewed", Others: "the build seat's holders read it to drop a duplicate request",
|
||||
Subjects: kvOf(CheckRunningBucket), Writes: holdsTheBuildSeat,
|
||||
Shared: "every machine holding the build seat puts the key of the check it runs and deletes it when the check ends"},
|
||||
{State: "the plans-building mirror", Writer: "controller (lease holder), at the first build's ask and at each refresh",
|
||||
KeptIn: "key-value " + PlansBuildingBucket + ", aged out when not refreshed",
|
||||
Others: "the build seat's holders read it and take no new check while a key is there (ADR 0483)",
|
||||
Subjects: kvOf(PlansBuildingBucket), Writes: isController},
|
||||
{State: "a merge announced", Writer: "one announcer per forge (the hook, or the poll when the hook is absent — never both)",
|
||||
KeptIn: "the bus", Others: "—", Subjects: []string{"mesh.mod.*.event.pull.merged"}, Writes: ownModule},
|
||||
// A pull request's head, before it merges (novox/hq to-be 45 §9): the same one announcer.
|
||||
@@ -232,6 +236,9 @@ var WritersTable = []WriterRow{
|
||||
{State: "the predefined-keys mirror", Writer: "controller (lease holder), from its store, only on a change",
|
||||
KeptIn: "key-value " + PredefinedBucket, Others: "read through `reads` as mesh-controller." + MirrorPredefined,
|
||||
Subjects: kvOf(PredefinedBucket), Writes: isController},
|
||||
{State: "the declared-packages mirror", Writer: "controller (lease holder), from the composition of each node, only on a change",
|
||||
KeptIn: "key-value " + DeclaredPackagesBucket, Others: "read through `reads` as mesh-controller." + MirrorDeclaredPackages,
|
||||
Subjects: kvOf(DeclaredPackagesBucket), Writes: isController},
|
||||
// What a machine's provisioner client placed of a module's files (novox/hq ADR 0349 §4, design 57 P6): its own
|
||||
// machine's word alone, which a walk's end and a gate wait for (ADR 0362, ADR 0349 §15). Kept newest per module
|
||||
// in the stream ProvAppliedStream, and by the client in the provisioner server's `applied` bucket.
|
||||
|
||||
@@ -29,6 +29,8 @@ var designRows = []string{
|
||||
"builds and their outcomes",
|
||||
"a pull request's check asked",
|
||||
"a pull request's check running",
|
||||
// The plans in their build phase, a bucket the build seat's holders read (ADR 0483 clause 2, issue 1687).
|
||||
"the plans-building mirror",
|
||||
"a merge announced",
|
||||
"a pull request's head announced",
|
||||
"a pull request's merge check",
|
||||
@@ -47,6 +49,7 @@ var designRows = []string{
|
||||
"a declaration's large files",
|
||||
"the declared-config mirror",
|
||||
"the predefined-keys mirror",
|
||||
"the declared-packages mirror",
|
||||
// The provisioner client's word on what it placed (novox/hq ADR 0349 §4, design 57 P6).
|
||||
"a module's files placed on a machine",
|
||||
}
|
||||
|
||||
@@ -86,11 +86,14 @@ func TestAnImageCompilingGoIsHandedItsBuildSourceAndSaysIt(t *testing.T) {
|
||||
}
|
||||
for file, want := range map[string]bool{"proxy/main.go": true, "internal/shared/s.go": true, "go.mod": true,
|
||||
"cmd/ctl/main.go": false, "internal/only/o.go": false, "README.md": false} {
|
||||
_, err := os.Stat(filepath.Join(workspace, "narrow-server", filepath.FromSlash(file)))
|
||||
if (err == nil) != want {
|
||||
t.Errorf("%s handed to the recipe: %v, wanted %v", file, err == nil, want)
|
||||
held := slices.Contains(r.builtFrom, file)
|
||||
if held != want {
|
||||
t.Errorf("%s handed to the recipe: %v, wanted %v", file, held, want)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(workspace, "narrow-server")); err == nil {
|
||||
t.Error("the narrowed build source was left in the workspace")
|
||||
}
|
||||
|
||||
// Said per repository: its own, the manifest and the recipe; the context's, the closure.
|
||||
if len(got.Sources) != 2 {
|
||||
|
||||
@@ -219,6 +219,11 @@ func build(ctx context.Context, run Runner, publish Publisher,
|
||||
if err := os.MkdirAll(workspace, 0o755); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
// **A node with no room takes no build** (novox/hq issue 1834).
|
||||
if err := RoomToWork("/", workspace); err != nil {
|
||||
say("clone", "refused: %v", err)
|
||||
return Result{}, err
|
||||
}
|
||||
// The credential is a file git reads, never an argument: a URL carrying a password in argv
|
||||
// would be readable by anything that can list processes for as long as a clone runs.
|
||||
credentials, forget, err := storeCredential(workspace, forge)
|
||||
@@ -738,11 +743,41 @@ func logging(log Log) func(step, format string, args ...any) {
|
||||
}
|
||||
}
|
||||
|
||||
// keepsImageLocally says the publisher leaves the image in the machine's runtime as the artifact itself.
|
||||
func keepsImageLocally(publish Publisher) bool {
|
||||
switch publish.(type) {
|
||||
case Local, *Local:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// PushedNames is implemented by a publisher that tags the image under other names to push it: the names it leaves in the
|
||||
// machine's runtime for a repository, which go with the build's own tag (novox/hq issue 1834).
|
||||
type PushedNames interface {
|
||||
PushedNames(repository string) []string
|
||||
}
|
||||
|
||||
// releaseImage removes the names an image was given on the node, in one command: its layers go only with the last. It runs
|
||||
// with a context of its own, the build's may be cancelled. A name never made (the build failed before it) is no fault;
|
||||
// one that cannot be removed is said, and removal is said only when the image is gone.
|
||||
func releaseImage(ctx context.Context, run Runner, names []string, say func(step, format string, args ...any)) {
|
||||
removing, done := context.WithTimeout(context.WithoutCancel(ctx), time.Minute)
|
||||
defer done()
|
||||
if _, err := run(removing, "", "docker", append([]string{"rmi"}, names...)...); err != nil {
|
||||
if !strings.Contains(err.Error(), "No such image") {
|
||||
say("image", "the local image %s could not be removed, and goes with the week's sweep: %v", strings.Join(names, ", "), err)
|
||||
}
|
||||
return
|
||||
}
|
||||
say("image", "the local image %s is removed: the registry holds it", strings.Join(names, ", "))
|
||||
}
|
||||
|
||||
// contextFrom clones an image artifact's own build context, when it names one apart from this
|
||||
// module's own repository — a fresh tree, the same way the module's own is, keyed by artifact
|
||||
// name so two artifacts of one module naming different contexts do not collide.
|
||||
func contextFrom(ctx context.Context, run Runner, workspace, artifact, credentials string,
|
||||
from catalogue.ArtifactContext, seats map[string]string, say func(step, format string, args ...any)) (string, error) {
|
||||
from catalogue.ArtifactContext, seats map[string]string, say func(step, format string, args ...any)) (_ string, err error) {
|
||||
url, err := contextURL(from, seats)
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -752,6 +787,12 @@ func contextFrom(ctx context.Context, run Runner, workspace, artifact, credentia
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
return "", err
|
||||
}
|
||||
// A clone that failed half-way is no context to leave behind (novox/hq issue 1834).
|
||||
defer func() {
|
||||
if err != nil {
|
||||
os.RemoveAll(dir)
|
||||
}
|
||||
}()
|
||||
if _, err := run(ctx, workspace, "git", cloneWith(credentials, "clone", "--quiet", url, dir)...); err != nil {
|
||||
return "", fmt.Errorf("cannot clone %s: %w", url, err)
|
||||
}
|
||||
@@ -1143,6 +1184,8 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
||||
src.contexts[a.Name] = t
|
||||
}
|
||||
buildDir = cloned
|
||||
// Let go of when the image is built, whatever became of it (novox/hq issue 1834).
|
||||
defer os.RemoveAll(cloned)
|
||||
if t, _ := trunkOf(ctx, run, cloned, "HEAD"); t == "" || !atTrunkHead(ctx, run, cloned, "HEAD", t) {
|
||||
src.contextOffHead()
|
||||
}
|
||||
@@ -1180,6 +1223,7 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
||||
src.ownHas(paths...)
|
||||
}
|
||||
buildDir = narrowed
|
||||
defer os.RemoveAll(narrowed)
|
||||
} else if a.Context != nil {
|
||||
src.readWhole(*a.Context)
|
||||
}
|
||||
@@ -1205,6 +1249,16 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
||||
}
|
||||
invocation = append(invocation, ".")
|
||||
say("image", "docker build -f %s", recipePath)
|
||||
// **The local tag goes with the build** (novox/hq issue 1834): an image published to a registry is held there, and
|
||||
// a tag per module and commit left on the node was the 96 GB of a full disk. Only a node publishing nowhere (a
|
||||
// mesh being raised) keeps it, since the local image is then the artifact.
|
||||
if !keepsImageLocally(publish) {
|
||||
names := []string{local}
|
||||
if p, ok := publish.(PushedNames); ok {
|
||||
names = append(names, p.PushedNames(module+"/"+a.Name)...)
|
||||
}
|
||||
defer releaseImage(ctx, run, names, say)
|
||||
}
|
||||
if _, err := run(ctx, buildDir, "docker", invocation...); err != nil {
|
||||
return catalogue.Built{}, fmt.Errorf("%s: building %s failed: %w", module, a.Name, err)
|
||||
}
|
||||
|
||||
@@ -38,6 +38,9 @@ type recorded struct {
|
||||
stamped time.Time
|
||||
// tree is what git names as the tree of the module's directory; empty answers as for a commit.
|
||||
tree string
|
||||
// builtFrom is the files (slash-separated, relative) of the folder the last `docker build` ran in, read as it ran:
|
||||
// the build removes that folder when it ends (novox/hq issue 1834).
|
||||
builtFrom []string
|
||||
}
|
||||
|
||||
func (r *recorded) run(_ context.Context, dir, name string, args ...string) (string, error) {
|
||||
@@ -53,6 +56,16 @@ func (r *recorded) run(_ context.Context, dir, name string, args ...string) (str
|
||||
break
|
||||
}
|
||||
}
|
||||
if name == "docker" && len(args) > 0 && args[0] == "build" {
|
||||
r.builtFrom = nil
|
||||
_ = filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error {
|
||||
if err == nil && !d.IsDir() {
|
||||
rel, _ := filepath.Rel(dir, p)
|
||||
r.builtFrom = append(r.builtFrom, filepath.ToSlash(rel))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
switch {
|
||||
case name == "git" && isClone:
|
||||
repository := args[len(args)-2]
|
||||
|
||||
@@ -123,8 +123,41 @@ type CheckSpec struct {
|
||||
// controller could not learn them, and the verdict says so.
|
||||
Siblings []Sibling
|
||||
SiblingsUnknown string
|
||||
// Held are the approved held values of the check's pull requests (novox/hq ADR 0502 clause 4): written to a file and
|
||||
// handed to the gate, which lays them over the change's composition only. None: no flag, no file.
|
||||
Held []HeldValue
|
||||
}
|
||||
|
||||
// HeldValue is one approved config value held for a key an open pull request introduces: the value of Key for Module on
|
||||
// Node (a machine's real name; the gate finds it in the snapshot by its pseudonym). The gate reads this document.
|
||||
type HeldValue struct {
|
||||
Pull string `json:"pull"`
|
||||
Head string `json:"head,omitempty"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
Approved string `json:"approved,omitempty"`
|
||||
By string `json:"by,omitempty"`
|
||||
At string `json:"at,omitempty"`
|
||||
}
|
||||
|
||||
// writeHeld writes the held values to held.json under root and answers its path; empty, and nothing written, for none.
|
||||
func writeHeld(root string, held []HeldValue) (string, error) {
|
||||
if len(held) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
raw, err := json.Marshal(held)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
file := filepath.Join(root, "held.json")
|
||||
return file, os.WriteFile(file, raw, 0o644)
|
||||
}
|
||||
|
||||
// heldFlag is how the gate is given the held values: only when there are some, so a judge from before --held still runs.
|
||||
const heldFlag = `${MESH_CHECK_HELD:+--held "$MESH_CHECK_HELD"}`
|
||||
|
||||
// Sibling is one other open pull request of a delivery group (novox/hq ADR 0382): the repository it is in as
|
||||
// owner/repo, its number, its whole head, the branch it merges into, and Dir, the directory the check finds that
|
||||
// repository under beside it (an entry of Beside).
|
||||
@@ -272,6 +305,8 @@ const (
|
||||
// EnvGroup is a delivery group's other heads, as JSON, for the gate (novox/hq ADR 0239); empty for a
|
||||
// pull request checked alone.
|
||||
EnvGroup = "MESH_CHECK_GROUP"
|
||||
// EnvHeld is the file of the approved held values of the check's pull requests (novox/hq ADR 0502); empty for none.
|
||||
EnvHeld = "MESH_CHECK_HELD"
|
||||
// EnvTests is a `go test -run` pattern and EnvStoreOnly is 1 when a branch tried before its pull request
|
||||
// asks for only those tests, or only the store tests (novox/hq issue 686). Never set for a pull request's
|
||||
// check. A script that reads neither runs whole.
|
||||
@@ -357,6 +392,12 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
||||
ctx, stop := context.WithTimeout(ctx, CheckTimeout)
|
||||
defer stop()
|
||||
|
||||
// **A node with no room takes no check** (novox/hq issue 1834): said as the check's verdict, never a throwaway store
|
||||
// that dies at once on a full disk.
|
||||
if err := RoomToWork("/", workspace); err != nil {
|
||||
say("check", "refused: %v", err)
|
||||
return CheckVerdict{}, err
|
||||
}
|
||||
root := filepath.Join(workspace, "check")
|
||||
if err := os.RemoveAll(root); err != nil {
|
||||
return CheckVerdict{}, err
|
||||
@@ -572,6 +613,15 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
||||
}
|
||||
}
|
||||
|
||||
// The approved held values of the check's pull requests, for the gate (novox/hq ADR 0502 clause 4).
|
||||
heldFile, err := writeHeld(root, spec.Held)
|
||||
if err != nil {
|
||||
return CheckVerdict{}, err
|
||||
}
|
||||
if heldFile != "" {
|
||||
say("check", "with %d config value(s) held for its pull request(s)", len(spec.Held))
|
||||
}
|
||||
|
||||
// Every clone is made: the credential is gone before anything of the check runs (novox/hq issue 462).
|
||||
forget()
|
||||
|
||||
@@ -733,7 +783,7 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
||||
EnvTestStore + "=" + storeURL, EnvTestBus + "=nats://" + bus, EnvRepository + "=" + spec.Owner + "/" + spec.Repo,
|
||||
EnvBeside + "=" + root,
|
||||
EnvModules + "=" + strings.Join(append(append([]string{}, spec.Modules...), spec.New...), ","),
|
||||
EnvGroup + "=" + groupFile,
|
||||
EnvGroup + "=" + groupFile, EnvHeld + "=" + heldFile,
|
||||
"GOCACHE=" + filepath.Join(workspace, "go-cache"), "GOMODCACHE=" + filepath.Join(workspace, "go-modules")}
|
||||
env = append(env, changedEnv...)
|
||||
env = append(env, selection(spec)...)
|
||||
@@ -1100,7 +1150,7 @@ func gateLayer(ctx context.Context, spec CheckSpec, tree, root, gate, verdictFil
|
||||
if err := running(inToolchain(tree, append(env, EnvVerdict+"="+verdictFile), "sh", "-c",
|
||||
`"$MESH_GATE" merge-gate --facts "$MESH_FACTS" --store "$MESH_GATE_POSTGRES" `+
|
||||
`--repository "$MESH_CHECK_REPOSITORY" --tree . `+gateChangedFlag(env)+` `+
|
||||
`${MESH_CHECK_GROUP:+--group "$MESH_CHECK_GROUP"} --json > "$MESH_CHECK_VERDICT"`)); err != nil {
|
||||
`${MESH_CHECK_GROUP:+--group "$MESH_CHECK_GROUP"} `+heldFlag+` --json > "$MESH_CHECK_VERDICT"`)); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return "error", "the check was ended during the merge gate"
|
||||
}
|
||||
@@ -1587,7 +1637,9 @@ func BusImage(version string) string {
|
||||
|
||||
// throwaway starts a container publishing one port on loopback, and answers where it is reached.
|
||||
func throwaway(ctx context.Context, run, plain Runner, name, image string, port int, opts, args []string) (string, error) {
|
||||
invocation := []string{"run", "-d", "--rm", "--name", name, "-p", fmt.Sprintf("127.0.0.1::%d", port)}
|
||||
// Never with --rm (novox/hq issue 1834): a store that dies at once took its log with it. The check removes what it
|
||||
// raised, with its volumes, after it has read why one stopped (RemoveContainersOf).
|
||||
invocation := []string{"run", "-d", "--name", name, "-p", fmt.Sprintf("127.0.0.1::%d", port)}
|
||||
invocation = append(invocation, opts...)
|
||||
invocation = append(invocation, image)
|
||||
invocation = append(invocation, args...)
|
||||
@@ -1608,17 +1660,45 @@ func throwaway(ctx context.Context, run, plain Runner, name, image string, port
|
||||
|
||||
// waitFor runs a readiness command in a container until it answers, for a minute.
|
||||
func waitFor(ctx context.Context, run Runner, name string, ready []string) error {
|
||||
for i := 0; i < 60; i++ {
|
||||
return waitReady(ctx, run, name, ready, 60)
|
||||
}
|
||||
|
||||
// waitReady is waitFor for a number of tries, a second apart. A container that never answers is not left a mystery
|
||||
// (novox/hq issue 1834): its state and exit code and the last 20 lines of its log are in the error, read before the
|
||||
// check removes it.
|
||||
func waitReady(ctx context.Context, run Runner, name string, ready []string, attempts int) error {
|
||||
for i := 0; i < attempts; i++ {
|
||||
if _, err := run(ctx, "", "docker", append([]string{"exec", name}, ready...)...); err == nil {
|
||||
return nil
|
||||
}
|
||||
if i == attempts-1 {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("%s never became ready", name)
|
||||
return fmt.Errorf("%s never became ready%s", name, whyItStopped(ctx, run, name))
|
||||
}
|
||||
|
||||
// whyItStopped is ": <state>, exit code N; its last log lines: …" for a container, or "" when docker says nothing.
|
||||
func whyItStopped(ctx context.Context, run Runner, name string) string {
|
||||
reading, done := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
|
||||
defer done()
|
||||
var said []string
|
||||
if out, err := run(reading, "", "docker", "inspect", "--format",
|
||||
"{{.State.Status}}, exit code {{.State.ExitCode}}", name); err == nil && strings.TrimSpace(out) != "" {
|
||||
said = append(said, strings.TrimSpace(out))
|
||||
}
|
||||
if out, err := run(reading, "", "docker", "logs", "--tail", "20", name); err == nil && strings.TrimSpace(out) != "" {
|
||||
said = append(said, "its last log lines: "+strings.TrimSpace(out))
|
||||
}
|
||||
if len(said) == 0 {
|
||||
return ""
|
||||
}
|
||||
return ": " + strings.Join(said, "; ")
|
||||
}
|
||||
|
||||
// dialable waits for an address to take a connection, for a minute.
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 1826/AC2: the held values are written whole to held.json, and the gate is given the flag only when there are some.
|
||||
func TestHeldValuesAreWrittenForTheGateAndNoneWriteNothing(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if file, err := writeHeld(root, nil); err != nil || file != "" {
|
||||
t.Fatalf("none wrote %q %v", file, err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "held.json")); err == nil {
|
||||
t.Fatal("none wrote a file")
|
||||
}
|
||||
file, err := writeHeld(root, []HeldValue{{Pull: "novox/mesh-catalog#434", Node: "ace", Module: "n8n", Key: "media-gid",
|
||||
Value: []byte("2000"), Approved: "the operator"}})
|
||||
if err != nil || file != filepath.Join(root, "held.json") {
|
||||
t.Fatalf("wrote %q %v", file, err)
|
||||
}
|
||||
raw, _ := os.ReadFile(file)
|
||||
for _, want := range []string{`"node":"ace"`, `"module":"n8n"`, `"key":"media-gid"`, `"value":2000`} {
|
||||
if !strings.Contains(string(raw), want) {
|
||||
t.Fatalf("%s lacks %s", raw, want)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(heldFlag, `--held "$MESH_CHECK_HELD"`) || !strings.HasPrefix(heldFlag, "${MESH_CHECK_HELD:+") {
|
||||
t.Fatalf("the flag is %s", heldFlag)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Issue 1834: a build node's root disk filled and every check there ended "the store never became ready". A node
|
||||
// frees what each build and check leaves, refuses to take work under a floor of free space, and a store that does
|
||||
// not come up says why.
|
||||
|
||||
func init() {
|
||||
// The disk the tests run on is none of their business; the tests of the floor say their own.
|
||||
FreeBytes = func(string) (uint64, error) { return 500 << 30, nil }
|
||||
}
|
||||
|
||||
func withFree(t *testing.T, bytes uint64) {
|
||||
t.Helper()
|
||||
was := FreeBytes
|
||||
FreeBytes = func(string) (uint64, error) { return bytes, nil }
|
||||
t.Cleanup(func() { FreeBytes = was })
|
||||
}
|
||||
|
||||
// 1834/AC2: given 3 GiB free, when a check or a build is asked, then it refuses at once, saying so, running nothing.
|
||||
func TestACheckUnderTheFloorOfFreeSpaceRefusesSayingSo(t *testing.T) {
|
||||
withFree(t, 3<<30)
|
||||
var ran []string
|
||||
run := func(_ context.Context, _ string, name string, args ...string) (string, error) {
|
||||
ran = append(ran, name+" "+strings.Join(args, " "))
|
||||
return "", nil
|
||||
}
|
||||
_, err := Check(context.Background(), run, CheckSpec{ID: "build-1"}, t.TempDir(), "registry.invalid", GitCredential{}, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "the build node has 3 GiB free on /, under the floor of 20 GiB") {
|
||||
t.Fatalf("a check on a full disk said %v", err)
|
||||
}
|
||||
if len(ran) != 0 {
|
||||
t.Errorf("a refused check ran %v", ran)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABuildUnderTheFloorOfFreeSpaceRefusesSayingSo(t *testing.T) {
|
||||
withFree(t, 3<<30)
|
||||
r := &recorded{}
|
||||
_, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", t.TempDir(), nil, Npmrc{}, GitCredential{}, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "under the floor of 20 GiB") {
|
||||
t.Fatalf("a build on a full disk said %v", err)
|
||||
}
|
||||
if len(r.ran) != 0 {
|
||||
t.Errorf("a refused build ran %v", r.ran)
|
||||
}
|
||||
}
|
||||
|
||||
// The workspace's filesystem is read too, and a workspace not made yet is read at its nearest folder.
|
||||
func TestTheWorkspacesFilesystemIsReadToo(t *testing.T) {
|
||||
was := FreeBytes
|
||||
t.Cleanup(func() { FreeBytes = was })
|
||||
missing := filepath.Join(t.TempDir(), "not", "yet")
|
||||
FreeBytes = func(p string) (uint64, error) {
|
||||
if p == "/" {
|
||||
return 100 << 30, nil
|
||||
}
|
||||
if _, err := os.Stat(p); err != nil {
|
||||
t.Errorf("read the free space of %s, which is not there", p)
|
||||
}
|
||||
return 1 << 30, nil
|
||||
}
|
||||
err := RoomToWork("/", missing)
|
||||
if err == nil || !strings.Contains(err.Error(), "1 GiB free on "+missing) {
|
||||
t.Errorf("the workspace's disk, under the floor, said %v", err)
|
||||
}
|
||||
FreeBytes = func(string) (uint64, error) { return 0, errors.New("no") }
|
||||
if RoomToWork("/") == nil {
|
||||
t.Error("a disk that cannot be read was taken as having room")
|
||||
}
|
||||
}
|
||||
|
||||
// 1834/AC3: given a store that stops at once, when the check waits for it, then its exit code and last log lines are
|
||||
// in the error and the container is removed afterwards, not by --rm before anyone could read it.
|
||||
func TestAStoreThatDiesSaysWhyItDied(t *testing.T) {
|
||||
var ran []string
|
||||
run := func(_ context.Context, _ string, name string, args ...string) (string, error) {
|
||||
line := name + " " + strings.Join(args, " ")
|
||||
ran = append(ran, line)
|
||||
switch {
|
||||
case args[0] == "exec":
|
||||
return "", errors.New("Error response from daemon: container is not running")
|
||||
case args[0] == "inspect":
|
||||
return "exited exit code 1\n", nil
|
||||
case args[0] == "logs":
|
||||
return "initdb: error: could not create directory: No space left on device\n", nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
err := waitReady(context.Background(), run, "build-1-store", []string{"pg_isready"}, 1)
|
||||
for _, want := range []string{"build-1-store never became ready", "exit code 1", "No space left on device"} {
|
||||
if err == nil || !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the verdict %q lacks %q", err, want)
|
||||
}
|
||||
}
|
||||
logs := ""
|
||||
for _, l := range ran {
|
||||
if strings.HasPrefix(l, "docker logs") {
|
||||
logs = l
|
||||
}
|
||||
}
|
||||
if logs != "docker logs --tail 20 build-1-store" {
|
||||
t.Errorf("the log was read as %q", logs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheThrowawayStoreIsNotStartedWithRm(t *testing.T) {
|
||||
var started []string
|
||||
run := func(_ context.Context, _ string, name string, args ...string) (string, error) {
|
||||
if args[0] == "run" {
|
||||
started = args
|
||||
}
|
||||
return "127.0.0.1:40000\n", nil
|
||||
}
|
||||
if _, err := throwaway(context.Background(), run, run, "build-1-store", "postgres:17-alpine", 5432, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, a := range started {
|
||||
if a == "--rm" {
|
||||
t.Errorf("started with --rm, which erases why it died: %v", started)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1834/AC1: given an image built for a check or a walk and published to the registry, when the build ends, then the
|
||||
// local image tag, the narrowed source and the context clone it left are gone; an image published nowhere (a mesh
|
||||
// being raised) is kept, since it is the artifact.
|
||||
// pushedNamed is a publisher that tags the image as <registry>/<repository> to push it, as the registry one does.
|
||||
type pushedNamed struct{ *recorded }
|
||||
|
||||
func (pushedNamed) PushedNames(repository string) []string {
|
||||
return []string{"registry.invalid/" + repository}
|
||||
}
|
||||
|
||||
func TestABuildRemovesWhatItLeftOnTheNode(t *testing.T) {
|
||||
r := &recorded{
|
||||
contents: map[string]string{"modules/m/" + ManifestName: anImageWithAContext, "modules/m/Dockerfile": "FROM scratch\n"},
|
||||
secondary: map[string]map[string]string{"https://forge.invalid/ctl.git": {"go.mod": "module x\n"}},
|
||||
}
|
||||
workspace := t.TempDir()
|
||||
if _, err := Build(context.Background(), r.run, pushedNamed{r}, "https://forge.invalid/c.git", "modules/m", "", workspace, nil, Npmrc{},
|
||||
GitCredential{}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Both names go in one rmi: the layers stay while either is left (the push tagged the image with the registry's).
|
||||
if !strings.Contains(strings.Join(r.ran, "\n"), "docker rmi m-server:c0ffeec0 registry.invalid/m/server") {
|
||||
t.Errorf("the image built for it and its pushed name were not removed: %v", r.ran)
|
||||
}
|
||||
left, _ := filepath.Glob(filepath.Join(workspace, "*"))
|
||||
if len(left) != 0 {
|
||||
t.Errorf("the build left %v in its workspace", left)
|
||||
}
|
||||
}
|
||||
|
||||
const anImageWithAContext = `{"module":"m","version":"1",
|
||||
"build":{"artifacts":[{"name":"server","kind":"image","from":"Dockerfile",
|
||||
"context":{"repository":"https://forge.invalid/ctl.git","ref":"main"}}]}}`
|
||||
|
||||
func TestAnImageKeptLocallyIsNotRemoved(t *testing.T) {
|
||||
r := &recorded{contents: map[string]string{"modules/m/" + ManifestName: anImageWithAContext, "modules/m/Dockerfile": "FROM scratch\n"}}
|
||||
local := Local{Run: func(ctx context.Context, dir, name string, args ...string) (string, error) {
|
||||
if args[0] == "image" {
|
||||
return "sha256:" + strings.Repeat("a", 64) + "\n", nil
|
||||
}
|
||||
return r.run(ctx, dir, name, args...)
|
||||
}}
|
||||
if _, err := Build(context.Background(), local.Run, local, "https://forge.invalid/c.git", "modules/m", "", t.TempDir(), nil, Npmrc{},
|
||||
GitCredential{}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(strings.Join(r.ran, "\n"), "docker rmi") {
|
||||
t.Errorf("the artifact of a mesh being raised was removed: %v", r.ran)
|
||||
}
|
||||
}
|
||||
|
||||
// The held image the comparison pulls to read is let go of once read.
|
||||
func TestTheHeldImageAComparisonPullsIsRemovedAfterReading(t *testing.T) {
|
||||
var ran []string
|
||||
run := func(_ context.Context, _ string, name string, args ...string) (string, error) {
|
||||
ran = append(ran, name+" "+strings.Join(args, " "))
|
||||
if args[0] == "create" {
|
||||
return "abcdef0123456789\n", nil
|
||||
}
|
||||
if args[0] == "image" {
|
||||
return "", errors.New("stop here")
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
_, _ = imageContent(context.Background(), run, t.TempDir(), "reg.invalid/m/server@sha256:"+strings.Repeat("a", 64))
|
||||
if got := ran[len(ran)-1]; !strings.HasPrefix(got, "docker rmi reg.invalid/m/server@sha256:") {
|
||||
t.Errorf("the last command was %q, not the removal of the pulled image: %v", got, ran)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnImageThatCannotBeRemovedIsNotSaidRemoved(t *testing.T) {
|
||||
var said []string
|
||||
run := func(context.Context, string, string, ...string) (string, error) {
|
||||
return "", errors.New("image is being used")
|
||||
}
|
||||
releaseImage(context.Background(), run, []string{"a:1"}, func(step, format string, args ...any) {
|
||||
said = append(said, format)
|
||||
})
|
||||
for _, s := range said {
|
||||
if strings.Contains(s, "is removed") {
|
||||
t.Errorf("said removed: %q", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// FreeSpaceFloor is the free space a build node must have, on / and on its workspace's filesystem, before it takes
|
||||
// a build or a check (novox/hq issue 1834): under it, a throwaway store cannot initialise its data directory and
|
||||
// dies at once, and the check ended "never became ready" instead of saying the disk was full.
|
||||
const FreeSpaceFloor = 20 << 30
|
||||
|
||||
// FreeBytes reads the bytes an unprivileged writer can still use on the filesystem holding path. A variable so a
|
||||
// test answers for a disk it does not have.
|
||||
var FreeBytes = func(path string) (uint64, error) {
|
||||
var s syscall.Statfs_t
|
||||
if err := syscall.Statfs(path, &s); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.Bavail * uint64(s.Bsize), nil
|
||||
}
|
||||
|
||||
// RoomToWork refuses, saying how much is free, when any of the paths' filesystems is under FreeSpaceFloor. A path
|
||||
// that does not exist yet (a workspace not made) is read at its nearest folder that does.
|
||||
func RoomToWork(paths ...string) error {
|
||||
for _, p := range paths {
|
||||
at := p
|
||||
for {
|
||||
if _, err := os.Stat(at); err == nil || filepath.Dir(at) == at {
|
||||
break
|
||||
}
|
||||
at = filepath.Dir(at)
|
||||
}
|
||||
free, err := FreeBytes(at)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the free space on %s cannot be read: %w", p, err)
|
||||
}
|
||||
if free < FreeSpaceFloor {
|
||||
return fmt.Errorf("the build node has %d GiB free on %s, under the floor of %d GiB",
|
||||
free>>30, p, FreeSpaceFloor>>30)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -172,11 +172,11 @@ func TestANarrowedTreeHoldsTheBuildSourceAndNothingElse(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// This repository's own programs: the route proxy's build source is not the controller's, and neither
|
||||
// holds the other's command.
|
||||
// This repository's own programs: the route proxy lives in its module's own Go module, so its build source is
|
||||
// not the controller's, and neither holds the other's command.
|
||||
func TestThisRepositorysProgramsHaveBuildSourcesOfTheirOwn(t *testing.T) {
|
||||
root := filepath.Join("..", "..")
|
||||
proxy, err := GoBuildSource(root, "examples/route-proxy")
|
||||
proxy, err := GoBuildSource(filepath.Join(root, "modules", "route-proxy"), "cmd/route-proxy")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -194,15 +194,15 @@ func TestThisRepositorysProgramsHaveBuildSourcesOfTheirOwn(t *testing.T) {
|
||||
file string
|
||||
held bool
|
||||
}{
|
||||
{proxy, "the route proxy", "examples/route-proxy/main.go", true},
|
||||
{proxy, "the route proxy", "internal/broker/broker.go", true},
|
||||
{proxy, "the route proxy", "cmd/mesh-controller/main.go", false},
|
||||
{proxy, "the route proxy", "internal/conditions/condition.go", false},
|
||||
{proxy, "the route proxy", "cmd/route-proxy/main.go", true},
|
||||
{proxy, "the route proxy", "cmd/route-proxy/bus.go", true},
|
||||
{proxy, "the route proxy", "go.mod", true},
|
||||
{proxy, "the route proxy", "cmd/route-provider/main.go", false},
|
||||
{proxy, "the route proxy", "README.md", false},
|
||||
{controller, "the controller", "cmd/mesh-controller/main.go", true},
|
||||
{controller, "the controller", "internal/conditions/condition.go", true},
|
||||
{controller, "the controller", "internal/inventory/migrations/0001-nodes.sql", true},
|
||||
{controller, "the controller", "examples/route-proxy/main.go", false},
|
||||
{controller, "the controller", "modules/route-proxy/cmd/route-proxy/main.go", false},
|
||||
{controller, "the controller", "cmd/mesh-builder/main.go", false},
|
||||
{controller, "the controller", "README.md", false},
|
||||
{builder, "the build seat's program", "cmd/mesh-builder/main.go", true},
|
||||
|
||||
@@ -3,6 +3,7 @@ package builder
|
||||
import (
|
||||
"context"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
@@ -41,6 +42,11 @@ func inItsOwnGroup(cmd *exec.Cmd) {
|
||||
cmd.WaitDelay = KillWait
|
||||
}
|
||||
|
||||
// CgroupParent is the slice every container a build or check starts joins (novox/hq issue 1798), so the node's own
|
||||
// weights and cap bind it: the build-agent module declares the slice with the machine's CPU weight, IO weight and
|
||||
// memory limits, and names it in MESH_BUILD_CGROUP_PARENT. Empty, a container joins the runtime's default.
|
||||
var CgroupParent string
|
||||
|
||||
// Labelled is a Runner that marks every container a build starts with the build's id, so a kill
|
||||
// finds what outlived the docker client (novox/hq ADR 0219). Applied at the one place every command
|
||||
// passes rather than at each `docker run` the builder composes: a run added later is labelled too.
|
||||
@@ -55,8 +61,11 @@ func LabelledArgs(name string, args []string, id string) []string {
|
||||
if name != "docker" || len(args) == 0 || args[0] != "run" || id == "" {
|
||||
return args
|
||||
}
|
||||
out := make([]string, 0, len(args)+2)
|
||||
out := make([]string, 0, len(args)+4)
|
||||
out = append(out, "run", "--label", BuildLabel+"="+id)
|
||||
if CgroupParent != "" {
|
||||
out = append(out, "--cgroup-parent", CgroupParent)
|
||||
}
|
||||
return append(out, args[1:]...)
|
||||
}
|
||||
|
||||
@@ -71,12 +80,45 @@ func RemoveContainersOf(ctx context.Context, run Runner, id string) (int, error)
|
||||
if len(ids) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if _, err := run(ctx, "", "docker", append([]string{"rm", "-f"}, ids...)...); err != nil {
|
||||
if err := removeWithVolumes(ctx, run, ids); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return len(ids), nil
|
||||
}
|
||||
|
||||
// MountedVolumes is the template `docker inspect --format` reads the names of a container's volumes with.
|
||||
const MountedVolumes = `{{range .Mounts}}{{if eq .Type "volume"}}{{println .Name}}{{end}}{{end}}`
|
||||
|
||||
// anonymousVolume is the name docker gives a volume it makes for an image's VOLUME line: 64 hex digits.
|
||||
var anonymousVolume = regexp.MustCompile(`^[0-9a-f]{64}$`)
|
||||
|
||||
// removeWithVolumes removes containers together with their anonymous volumes (novox/hq issue 1772). A throwaway
|
||||
// store declares its data directory a VOLUME, docker makes an anonymous volume for it, and `docker rm -f`
|
||||
// without -v leaves that volume for ever: 1,629 of them, 107 GB, on one build node. So the volumes the
|
||||
// containers mount are read first, the containers are removed with -v, and an anonymous volume still there
|
||||
// afterwards is removed by name. Only a volume one of these containers mounted, and only one with an anonymous
|
||||
// volume's name, is touched: a named volume — an operator's own data — never is. A volume already gone is
|
||||
// no failure.
|
||||
func removeWithVolumes(ctx context.Context, run Runner, ids []string) error {
|
||||
var volumes []string
|
||||
if out, err := run(ctx, "", "docker", append([]string{"inspect", "--format", MountedVolumes}, ids...)...); err == nil {
|
||||
seen := map[string]bool{}
|
||||
for _, name := range strings.Fields(out) {
|
||||
if anonymousVolume.MatchString(name) && !seen[name] {
|
||||
seen[name] = true
|
||||
volumes = append(volumes, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
if _, err := run(ctx, "", "docker", append([]string{"rm", "-f", "-v"}, ids...)...); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(volumes) > 0 {
|
||||
_, _ = run(ctx, "", "docker", append([]string{"volume", "rm"}, volumes...)...)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// A holder that stops mid-build leaves its containers running (novox/hq issue 306).
|
||||
//
|
||||
// A build's containers are removed by the build itself, by a kill, or by the next delivery of the same
|
||||
@@ -106,7 +148,7 @@ func RemoveLeftBehind(ctx context.Context, run Runner) (int, error) {
|
||||
if len(ids) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if _, err := run(ctx, "", "docker", append([]string{"rm", "-f"}, ids...)...); err != nil {
|
||||
if err := removeWithVolumes(ctx, run, ids); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return len(ids), nil
|
||||
|
||||
@@ -2,6 +2,7 @@ package builder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -49,7 +50,8 @@ func TestAKillRemovesTheContainersLabelledWithTheBuild(t *testing.T) {
|
||||
if err != nil || n != 2 {
|
||||
t.Fatalf("%d %v", n, err)
|
||||
}
|
||||
if want := []string{"docker ps -aq --filter label=mesh.build=build-3", "docker rm -f c1 c2"}; !reflect.DeepEqual(ran, want) {
|
||||
if want := []string{"docker ps -aq --filter label=mesh.build=build-3", "docker inspect --format " + MountedVolumes + " c1 c2",
|
||||
"docker rm -f -v c1 c2"}; !reflect.DeepEqual(ran, want) {
|
||||
t.Errorf("ran %v", ran)
|
||||
}
|
||||
}
|
||||
@@ -73,7 +75,7 @@ func TestAStartingHolderRemovesWhatEarlierHoldersLeftAndNothingElse(t *testing.T
|
||||
if n != 3 {
|
||||
t.Errorf("removed %d", n)
|
||||
}
|
||||
if len(ran) != 2 || !reflect.DeepEqual(ran[1], []string{"docker", "rm", "-f", "c1", "c2", "c7"}) {
|
||||
if len(ran) != 3 || !reflect.DeepEqual(ran[2], []string{"docker", "rm", "-f", "-v", "c1", "c2", "c7"}) {
|
||||
t.Fatalf("ran %v", ran)
|
||||
}
|
||||
if !strings.Contains(strings.Join(ran[0], " "), "--filter label=mesh.build") {
|
||||
@@ -90,3 +92,75 @@ func TestAStartingHolderRemovesWhatEarlierHoldersLeftAndNothingElse(t *testing.T
|
||||
t.Errorf("with nothing of the seat's: %d %v %v", n, err, ran)
|
||||
}
|
||||
}
|
||||
|
||||
// **Issue 1772**: a throwaway store declares its data directory a VOLUME, so docker makes an anonymous volume for
|
||||
// it, and `docker rm -f` without -v leaves that volume behind (1,629 of them, 107 GB, on one build node). Every
|
||||
// container a build or check removes is removed with its anonymous volumes (-v), and any volume the container
|
||||
// mounted that is still there afterwards is removed by name — those, and only those, never a named volume.
|
||||
func TestARemovedContainerTakesItsAnonymousVolumesWithIt(t *testing.T) {
|
||||
anon := strings.Repeat("a", 64)
|
||||
other := strings.Repeat("b", 64)
|
||||
var ran []string
|
||||
run := func(_ context.Context, _ string, name string, args ...string) (string, error) {
|
||||
ran = append(ran, name+" "+strings.Join(args, " "))
|
||||
switch args[0] {
|
||||
case "ps":
|
||||
return "c1\n", nil
|
||||
case "inspect":
|
||||
return anon + "\n" + "compose_data\n" + other + "\n", nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
if n, err := RemoveContainersOf(context.Background(), run, "build-4"); err != nil || n != 1 {
|
||||
t.Fatalf("%d %v", n, err)
|
||||
}
|
||||
last := ran[len(ran)-1]
|
||||
if want := "docker volume rm " + anon + " " + other; last != want {
|
||||
t.Errorf("last command %q, want %q", last, want)
|
||||
}
|
||||
if !strings.Contains(strings.Join(ran, "\n"), "docker rm -f -v c1") {
|
||||
t.Errorf("ran %v", ran)
|
||||
}
|
||||
if strings.Contains(last, "compose_data") {
|
||||
t.Errorf("a named volume was removed: %q", last)
|
||||
}
|
||||
}
|
||||
|
||||
// A volume already gone when the sweep looks (rm -v took it) is no failure.
|
||||
func TestAVolumeAlreadyGoneIsNoFailure(t *testing.T) {
|
||||
run := func(_ context.Context, _ string, name string, args ...string) (string, error) {
|
||||
switch args[0] {
|
||||
case "ps":
|
||||
return "c1\n", nil
|
||||
case "inspect":
|
||||
return strings.Repeat("c", 64) + "\n", nil
|
||||
case "volume":
|
||||
return "", errors.New("no such volume")
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
if n, err := RemoveContainersOf(context.Background(), run, "build-5"); err != nil || n != 1 {
|
||||
t.Fatalf("%d %v", n, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A node's build slice limits everything a build or check starts (novox/hq issue 1798): with a parent
|
||||
// set, every `docker run` joins it, so the node's CPU and IO weights and memory cap bind the container;
|
||||
// with none set the arguments are as they always were.
|
||||
func TestEveryContainerJoinsTheNodesBuildSlice(t *testing.T) {
|
||||
was := CgroupParent
|
||||
defer func() { CgroupParent = was }()
|
||||
CgroupParent = "meshbuild.slice"
|
||||
got := LabelledArgs("docker", []string{"run", "--rm", "img", "sh"}, "build-1")
|
||||
want := []string{"run", "--label", "mesh.build=build-1", "--cgroup-parent", "meshbuild.slice", "--rm", "img", "sh"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("1798/AC1: docker run became %v", got)
|
||||
}
|
||||
if got := LabelledArgs("docker", []string{"build", "."}, "build-1"); !reflect.DeepEqual(got, []string{"build", "."}) {
|
||||
t.Errorf("a docker build became %v", got)
|
||||
}
|
||||
CgroupParent = ""
|
||||
if got := LabelledArgs("docker", []string{"run", "img"}, "build-1"); !reflect.DeepEqual(got, []string{"run", "--label", "mesh.build=build-1", "img"}) {
|
||||
t.Errorf("1798/AC1: with no slice, docker run became %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -43,6 +43,11 @@ type Registry struct {
|
||||
HTTP *http.Client
|
||||
}
|
||||
|
||||
// PushedNames are the names PublishImage leaves the image under on the node.
|
||||
func (r Registry) PushedNames(repository string) []string {
|
||||
return []string{r.Address + "/" + repository}
|
||||
}
|
||||
|
||||
// PublishImage pushes a locally built image and returns a reference pinned by digest.
|
||||
//
|
||||
// The digest is read back from the registry's own answer rather than computed here. What matters
|
||||
|
||||
@@ -253,6 +253,13 @@ func (s scratchPublisher) PublishImage(ctx context.Context, localTag, repository
|
||||
return s.to.PublishImage(ctx, localTag, ScratchPrefix+repository)
|
||||
}
|
||||
|
||||
func (s scratchPublisher) PushedNames(repository string) []string {
|
||||
if p, ok := s.to.(PushedNames); ok {
|
||||
return p.PushedNames(ScratchPrefix + repository)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s scratchPublisher) PublishArchive(ctx context.Context, repository string, body []byte, digest string) (string, error) {
|
||||
return s.to.PublishArchive(ctx, ScratchPrefix+repository, body, digest)
|
||||
}
|
||||
|
||||
@@ -88,6 +88,9 @@ func imageContent(ctx context.Context, run Runner, dir, ref string) (string, err
|
||||
if _, err := run(ctx, dir, "docker", "pull", "--quiet", ref); err != nil {
|
||||
return "", fmt.Errorf("pulling %s: %w", ref, err)
|
||||
}
|
||||
// The image is on the node only for the reading (novox/hq issue 1834); one a container still runs is refused by
|
||||
// docker and stays.
|
||||
defer func() { _, _ = run(context.WithoutCancel(ctx), dir, "docker", "rmi", ref) }()
|
||||
config, err := run(ctx, dir, "docker", "image", "inspect", "--format",
|
||||
"{{.Os}}/{{.Architecture}}/{{.Variant}} {{json .Config}}", ref)
|
||||
if err != nil {
|
||||
@@ -105,7 +108,7 @@ func imageContent(ctx context.Context, run Runner, dir, ref string) (string, err
|
||||
if id == "" {
|
||||
return "", errors.New("the container engine named no container for " + ref)
|
||||
}
|
||||
defer func() { _, _ = run(context.WithoutCancel(ctx), dir, "docker", "rm", "--force", id) }()
|
||||
defer func() { _, _ = run(context.WithoutCancel(ctx), dir, "docker", "rm", "--force", "--volumes", id) }()
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -114,6 +114,16 @@ func (i *Inventory) AnswerMerge(ctx context.Context, repository, commit, plan st
|
||||
return i.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: repository, Commit: commit, Plan: plan, Alone: alone}})
|
||||
}
|
||||
|
||||
// HoldMerge puts a merge back to waiting, no plan answering it, with the event it is kept under replaced (novox/hq issue 1173:
|
||||
// a first merge held for a build-agent that reads its manifest, the hold said in its event).
|
||||
func (i *Inventory) HoldMerge(ctx context.Context, repository, commit string, event json.RawMessage) error {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return i.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: repository, Commit: commit, Event: event}})
|
||||
}
|
||||
|
||||
// Batches is every batch not yet cut: assembling or queued, oldest first. One at a time is the rule; more
|
||||
// are read so a fault is seen.
|
||||
func (i *Inventory) Batches(ctx context.Context) ([]Plan, error) {
|
||||
|
||||
@@ -185,7 +185,7 @@ func TestABuildsBuildSourceComesBackOverWhatItRead(t *testing.T) {
|
||||
said.Read = []ReadRepository{{Repository: "novox/mesh-controller", Ref: "main"}, {Repository: "novox/other"}}
|
||||
said.Sources = []BuildSource{
|
||||
{Paths: []string{"modules/route-proxy/Dockerfile", "modules/route-proxy/module.json"}},
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"examples/route-proxy/", "go.mod"}},
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"modules/route-proxy/cmd/route-proxy/", "go.mod"}},
|
||||
{Repository: "novox/unread", Paths: []string{"x"}},
|
||||
}
|
||||
unsaid := aBuild("unsaid", "build-agent", "")
|
||||
@@ -200,7 +200,7 @@ func TestABuildsBuildSourceComesBackOverWhatItRead(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []ReadRepository{
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"examples/route-proxy/", "go.mod"}},
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"modules/route-proxy/cmd/route-proxy/", "go.mod"}},
|
||||
{Repository: "novox/other"},
|
||||
{Paths: []string{"modules/route-proxy/Dockerfile", "modules/route-proxy/module.json"}, Own: true},
|
||||
}
|
||||
@@ -238,7 +238,7 @@ func TestAFailedNewerBuildLeavesTheBuildSourceStale(t *testing.T) {
|
||||
worked.Asked = time.Now().Add(-time.Hour)
|
||||
worked.Read = []ReadRepository{{Repository: "novox/mesh-controller", Ref: "main"}}
|
||||
worked.Sources = []BuildSource{{Paths: []string{"modules/route-proxy/module.json"}},
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"examples/route-proxy/"}}}
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"modules/route-proxy/cmd/route-proxy/"}}}
|
||||
failed := aBuild("failed", "route-proxy", "compile error")
|
||||
failed.Asked = time.Now()
|
||||
for _, b := range []Build{worked, failed} {
|
||||
|
||||
@@ -634,6 +634,10 @@ func (e *recordEdit) apply(held []chain.HeldMerge) []chain.HeldMerge {
|
||||
}
|
||||
if f, has := e.flag[k]; has {
|
||||
h.Waiting, h.Alone = f.Waiting, f.Alone
|
||||
// A merge put back to waiting may carry a new announcement (HoldMerge); every other flag copies its own.
|
||||
if len(f.Event) > 0 {
|
||||
h.Event = f.Event
|
||||
}
|
||||
}
|
||||
out = append(out, h)
|
||||
}
|
||||
@@ -758,6 +762,9 @@ func (i *Inventory) chainSavePlans(ctx context.Context, c *chain.Chain, plans []
|
||||
if m.Plan == "" {
|
||||
f := w.m
|
||||
f.Waiting, f.Alone = true, m.Alone
|
||||
if m.Event != nil {
|
||||
f.Event = m.Event
|
||||
}
|
||||
editOf(w.plan).flag[key] = f
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// aNodesPackages is the resources of a node whose modules declare packages: one package two modules declare present, one
|
||||
// declared absent, and a resource that is no package.
|
||||
func aNodesPackages() ([]map[string]any, map[string]string) {
|
||||
return []map[string]any{
|
||||
{"id": "web.pkg-curl", "type": "package", "package": "curl"},
|
||||
{"id": "backup.pkg-curl", "type": "package", "package": "curl"},
|
||||
{"id": "web.pkg-nano", "type": "package", "package": "nano", "absent": true},
|
||||
{"id": "web.conf", "type": "file", "path": "/etc/web.conf"},
|
||||
},
|
||||
map[string]string{"web.pkg-curl": "web", "backup.pkg-curl": "backup", "web.pkg-nano": "web", "web.conf": "web"}
|
||||
}
|
||||
|
||||
// 1840/AC5: the declared-packages mirror's value for a node names each package with the modules that declare it and whether
|
||||
// they declare it absent, and the same declaration is the same bytes however its resources are ordered, so a pass over an
|
||||
// unchanged node writes nothing.
|
||||
func TestTheDeclaredPackagesValueNamesEachPackageWithItsModules(t *testing.T) {
|
||||
resources, owner := aNodesPackages()
|
||||
got, err := DeclaredPackagesValue("one", resources, owner)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := `{"node":"one","packages":[{"package":"curl","modules":["backup","web"],"absent":false},` +
|
||||
`{"package":"nano","modules":["web"],"absent":true}]}`
|
||||
if string(got) != want {
|
||||
t.Errorf("the value is\n%s\nwant\n%s", got, want)
|
||||
}
|
||||
reversed := slices.Clone(resources)
|
||||
slices.Reverse(reversed)
|
||||
again, err := DeclaredPackagesValue("one", reversed, owner)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(got, again) {
|
||||
t.Errorf("the same declaration in another order is other bytes:\n%s\n%s", got, again)
|
||||
}
|
||||
if got := jsonFields(reflect.TypeOf(DeclaredPackagesMirrored{})); !slices.Equal(got, []string{"node", "packages"}) {
|
||||
t.Errorf("the value carries %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A node declaring no package has a value that says so, an empty list and not null.
|
||||
func TestANodeDeclaringNoPackageHasAnEmptyList(t *testing.T) {
|
||||
got, err := DeclaredPackagesValue("one", nil, nil)
|
||||
if err != nil || string(got) != `{"node":"one","packages":[]}` {
|
||||
t.Errorf("the value is %s (%v)", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1840/S3: a node's value stays under the 64 KiB a mirror's key holds at far more packages than a node declares: 600
|
||||
// packages, each declared by four modules, is a few times what the largest node declares today.
|
||||
func TestTheDeclaredPackagesValueOfABigNodeFitsAKey(t *testing.T) {
|
||||
var resources []map[string]any
|
||||
owner := map[string]string{}
|
||||
for p := 0; p < 600; p++ {
|
||||
for m := 0; m < 4; m++ {
|
||||
id := fmt.Sprintf("module-%d.pkg-%d", m, p)
|
||||
resources = append(resources, map[string]any{"id": id, "type": "package", "package": fmt.Sprintf("package-%d", p)})
|
||||
owner[id] = fmt.Sprintf("module-%d", m)
|
||||
}
|
||||
}
|
||||
got, err := DeclaredPackagesValue("one", resources, owner)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var v DeclaredPackagesMirrored
|
||||
if err := json.Unmarshal(got, &v); err != nil || len(v.Packages) != 600 {
|
||||
t.Fatalf("the value holds %d packages (%v)", len(v.Packages), err)
|
||||
}
|
||||
if len(got) >= 64<<10 {
|
||||
t.Errorf("the value is %d bytes, over the 64 KiB a key holds", len(got))
|
||||
}
|
||||
}
|
||||
@@ -5,7 +5,10 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// The values of the controller's record mirrors, built from the store (novox/hq ADR 0289, design 54 §1,
|
||||
@@ -179,3 +182,75 @@ func (i *Inventory) PredefinedMirror(ctx context.Context) (map[string][]byte, er
|
||||
}
|
||||
return map[string][]byte{PredefinedMirrorKey: value}, nil
|
||||
}
|
||||
|
||||
// DeclaredOf is a manifest's config declarations as the mirror holds them (DeclaredMirrored), so a pull request's
|
||||
// declarations are told to mesh-config in the one shape it reads the trunk's in (novox/hq ADR 0502 clause 2).
|
||||
func DeclaredOf(settings map[string]catalogue.SettingDeclaration) (map[string]DeclaredMirrored, error) {
|
||||
raw, err := json.Marshal(settings)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return declaredMirrored(raw)
|
||||
}
|
||||
|
||||
// PackageDeclared is one package a node's modules declare, as the declared-packages mirror holds it (ADR 0507 §5): its
|
||||
// name, the modules that declare it that way, sorted, and whether they declare it absent. A package some modules declare
|
||||
// present and others absent is two entries; the composition refuses that pair, so it is seen only while it is refused.
|
||||
type PackageDeclared struct {
|
||||
Package string `json:"package"`
|
||||
Modules []string `json:"modules"`
|
||||
Absent bool `json:"absent"`
|
||||
}
|
||||
|
||||
// DeclaredPackagesMirrored is one node's value of the declared-packages mirror, keyed by the node: its packages sorted by
|
||||
// name (absent after present). Fixed fields and no time, so the same declaration is the same bytes (ADR 0289 §1).
|
||||
type DeclaredPackagesMirrored struct {
|
||||
Node string `json:"node"`
|
||||
Packages []PackageDeclared `json:"packages"`
|
||||
}
|
||||
|
||||
// DeclaredPackagesValue is the mirror's value for one node, from the resources its composition declares and the module that
|
||||
// owns each (Composed.Owner, by resource id). A resource that is no package, or names none, is left out.
|
||||
func DeclaredPackagesValue(node string, resources []map[string]any, owner map[string]string) ([]byte, error) {
|
||||
type key struct {
|
||||
name string
|
||||
absent bool
|
||||
}
|
||||
by := map[key]map[string]bool{}
|
||||
for _, r := range resources {
|
||||
if fmt.Sprint(r["type"]) != "package" {
|
||||
continue
|
||||
}
|
||||
name, _ := r["package"].(string)
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
k := key{name, r["absent"] == true}
|
||||
if by[k] == nil {
|
||||
by[k] = map[string]bool{}
|
||||
}
|
||||
if module := owner[fmt.Sprint(r["id"])]; module != "" {
|
||||
by[k][module] = true
|
||||
}
|
||||
}
|
||||
keys := make([]key, 0, len(by))
|
||||
for k := range by {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if keys[i].name != keys[j].name {
|
||||
return keys[i].name < keys[j].name
|
||||
}
|
||||
return !keys[i].absent && keys[j].absent
|
||||
})
|
||||
v := DeclaredPackagesMirrored{Node: node, Packages: []PackageDeclared{}}
|
||||
for _, k := range keys {
|
||||
modules := make([]string, 0, len(by[k]))
|
||||
for m := range by[k] {
|
||||
modules = append(modules, m)
|
||||
}
|
||||
sort.Strings(modules)
|
||||
v.Packages = append(v.Packages, PackageDeclared{Package: k.name, Modules: modules, Absent: k.absent})
|
||||
}
|
||||
return json.Marshal(v)
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
@@ -677,6 +678,8 @@ func (i *Inventory) SavePlan(ctx context.Context, p *Plan) error {
|
||||
type MergeMove struct {
|
||||
Repository, Commit, Plan string
|
||||
Alone bool
|
||||
// Event replaces the merge's kept announcement when it is put back to waiting (Plan empty); nil leaves it.
|
||||
Event json.RawMessage
|
||||
}
|
||||
|
||||
// SavePlansMoving writes plans, as SavePlan does each, and gives merges to the plans that answer them, all in one
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// 1768/AC1: a request dropped as a duplicate of one another builder runs marks nothing in hand: the claim is not made,
|
||||
// so no mark is left behind to fail every later claim.
|
||||
func TestADuplicateDropIsJudgedBeforeTheClaimSoLeavesNoMark(t *testing.T) {
|
||||
claims := 0
|
||||
m := &natsMachine{
|
||||
opts: MachineOptions{Claim: func() bool { claims++; return true }},
|
||||
duplicate: func(context.Context, BuildRequest) string { return "ace is running it" },
|
||||
}
|
||||
if m.admit(context.Background(), &nats.Msg{}, BuildRequest{ID: "build-1"}) {
|
||||
t.Fatal("a duplicate was admitted")
|
||||
}
|
||||
if claims != 0 {
|
||||
t.Fatalf("a dropped duplicate was claimed %d times, which leaves the in-hand mark behind", claims)
|
||||
}
|
||||
m.duplicate = func(context.Context, BuildRequest) string { return "" }
|
||||
if !m.admit(context.Background(), &nats.Msg{}, BuildRequest{ID: "build-2"}) || claims != 1 {
|
||||
t.Fatalf("a request that is no duplicate was not claimed once: claims=%d", claims)
|
||||
}
|
||||
}
|
||||
@@ -218,6 +218,25 @@ type CheckRequest struct {
|
||||
Try bool `json:"try,omitempty"`
|
||||
Tests string `json:"tests,omitempty"`
|
||||
StoreOnly bool `json:"store-only,omitempty"`
|
||||
// HeldConfig are the approved held values of the pull requests this check is of (novox/hq ADR 0502 clause 4): each
|
||||
// a config value the operator approved for a key an open pull request introduces. The merge gate lays them over
|
||||
// their module's values on their node in the composition with the change only, never in the mesh as it is. Empty
|
||||
// from a controller from before it, or when mesh-config held none or did not answer.
|
||||
HeldConfig []HeldConfig `json:"held_config,omitempty"`
|
||||
}
|
||||
|
||||
// HeldConfig is one approved held value (novox/hq ADR 0502), as mesh-config's `held` action answers it.
|
||||
type HeldConfig struct {
|
||||
Pull string `json:"pull"`
|
||||
Head string `json:"head,omitempty"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
// Approved says who approved it, in words, as mesh-config states it; By and At are who held it and when.
|
||||
Approved string `json:"approved,omitempty"`
|
||||
By string `json:"by,omitempty"`
|
||||
At string `json:"at,omitempty"`
|
||||
}
|
||||
|
||||
// StepTried is the step a build seat says a try's outcome under, in the build's log: one line, a Tried as JSON
|
||||
|
||||
+195
-65
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
@@ -162,11 +163,23 @@ type natsMachine struct {
|
||||
// build waits, then the checks of pull requests marked ready, then every other check (novox/hq ADR 0406 §3).
|
||||
subs []*takeQueue
|
||||
opts MachineOptions
|
||||
// duplicate stands in for runningElsewhere in tests; nil is the real one.
|
||||
duplicate func(context.Context, BuildRequest) string
|
||||
// gate is what the loop asks of the bus to hold checks while a delivery builds; nil is the bus's (a test sets a fake).
|
||||
gate checkGate
|
||||
// said is the faults of the hold already said, letKey whether this holder created the let-through key for the check it
|
||||
// took last (ADR 0483 clause 3).
|
||||
said map[string]bool
|
||||
letKey bool
|
||||
// mu guards the subscriptions of subs, bound and let go of by the taking loop and closed by Close.
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// takeQueue is one queue a machine pulls from.
|
||||
type takeQueue struct {
|
||||
sub *nats.Subscription
|
||||
// pull is what the queue is fetched through: the subscription once bound (a test sets a fake).
|
||||
pull puller
|
||||
// kind is what its asks are: a build, a ready check, or another check.
|
||||
kind string
|
||||
// worker is the consumer bound; bound is false until a bind succeeded, for a queue the mesh may not have made yet.
|
||||
@@ -174,6 +187,11 @@ type takeQueue struct {
|
||||
bound bool
|
||||
}
|
||||
|
||||
// puller is the fetch of a queue.
|
||||
type puller interface {
|
||||
Fetch(batch int, opts ...nats.PullOpt) ([]*nats.Msg, error)
|
||||
}
|
||||
|
||||
// The kinds of queue a machine takes from, in the order it takes from them.
|
||||
const (
|
||||
queueBuilds = "builds"
|
||||
@@ -195,6 +213,9 @@ type MachineOptions struct {
|
||||
// the request in hand first and reads its drain after, so an ask of `busy` racing the fetch sees one or
|
||||
// the other, never neither. Nil claims every one.
|
||||
Claim func() bool
|
||||
// LetThroughAfter is how long a check waits while a delivery builds before it is let through; zero is ADR 0483's thirty
|
||||
// minutes. Set by a test.
|
||||
LetThroughAfter time.Duration
|
||||
}
|
||||
|
||||
// MachineOverNATS takes build work from the current build role.
|
||||
@@ -222,6 +243,8 @@ const (
|
||||
)
|
||||
|
||||
func (m *natsMachine) Close() {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
for _, q := range m.subs {
|
||||
if q.sub != nil {
|
||||
_ = q.sub.Unsubscribe()
|
||||
@@ -244,6 +267,9 @@ func (m *natsMachine) Close() {
|
||||
// **In this order** (novox/hq ADR 0406 §3): a delivery build, then a check of a pull request marked ready, then any
|
||||
// other check, each queue tried without waiting; when all three are empty the first is waited on for a second at most
|
||||
// and the three are tried again, so a delivery build is never much more than a second behind and nothing spins.
|
||||
//
|
||||
// **Unless a delivery builds** (novox/hq ADR 0483): while the controller's bucket of the plans building holds a key, no check queue
|
||||
// is fetched from, but for the one check that waited LetThroughAfter, which goes first (see next).
|
||||
func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build)) error {
|
||||
worker, found := broker.HolderConsumerFor(m.on, "build-agent",
|
||||
broker.DeclaredSeat{Name: m.seat, Accepts: []string{"build"}})
|
||||
@@ -288,6 +314,19 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
|
||||
}
|
||||
continue
|
||||
}
|
||||
// The seat's own queue is bound again when a running ask made the machine let go of it (quiesce): a machine that cannot
|
||||
// is not taking, which is said and tried again.
|
||||
if !m.subs[0].bound {
|
||||
if err := m.bind(m.subs[0]); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "this machine cannot bind %s again after a build, and tries again: %v\n", m.seat, err)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-time.After(pausedPoll):
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
if time.Now().After(bindAgain) {
|
||||
for _, q := range m.subs[1:] {
|
||||
if !q.bound {
|
||||
@@ -312,7 +351,7 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
|
||||
// say why (2026-10-03, the first build agents). Asked again.
|
||||
continue
|
||||
default:
|
||||
if m.subs[0].sub.IsValid() {
|
||||
if m.subs[0].sub != nil && m.subs[0].sub.IsValid() {
|
||||
// A transient fault in asking — a reconnect, a slow server — is asked past rather
|
||||
// than ending the machine; one that outlasts the ack wait redelivers nothing lost.
|
||||
time.Sleep(time.Second)
|
||||
@@ -320,70 +359,125 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
|
||||
}
|
||||
return fmt.Errorf("the bus stopped delivering build work: %w", err)
|
||||
}
|
||||
// **Paused while the pull was answered** (ADR 0219): "takes no new build" holds even for
|
||||
// the one that arrived in that moment. Handed back at once for another holder — counted as
|
||||
// a delivery, which a pause landing exactly then costs and nothing else does.
|
||||
if m.opts.Paused != nil && m.opts.Paused() {
|
||||
_ = msg.Nak()
|
||||
continue
|
||||
}
|
||||
var request BuildRequest
|
||||
if err := json.Unmarshal(msg.Data, &request); err != nil {
|
||||
// Unreadable: terminated rather than retried, because the next attempt reads the same
|
||||
// bytes. Nobody waiting hears an answer, which is right — there was no request.
|
||||
_ = msg.Term()
|
||||
continue
|
||||
}
|
||||
// **Cancelled in the moment it was fetched** (novox/hq ADR 0219): the controller wrote the
|
||||
// id into the seat's cancelled set before deleting the ask, so one taken in between is
|
||||
// ended here, terminated rather than redelivered, and its outcome said as failed — never
|
||||
// built. A set that cannot be read is said and the ask built: a cancel is a person's
|
||||
// exception, and a holder that refused every build while its grant was missing would
|
||||
// stop the mesh building for it.
|
||||
build := &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat, delivery: deliveryOf(msg)}
|
||||
if cancelled, err := IsCancelled(m.js.Conn(), m.seat, request.ID); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read whether %s was cancelled, so it is built: %v\n", request.ID, err)
|
||||
} else if cancelled {
|
||||
endCancelled(ctx, build)
|
||||
continue
|
||||
}
|
||||
// **A check left behind is dropped when it is taken** (novox/hq ADR 0406 §3): a request on the third queue for a
|
||||
// head and a trunk commit its pull request's key already says were judged is acknowledged and not run.
|
||||
if q.kind == queueOther {
|
||||
if why := m.judgedAlready(ctx, request); why != "" {
|
||||
fmt.Fprintf(os.Stderr, "%s was left behind in the check queue and is dropped: %s\n", request.ID, why)
|
||||
_ = msg.Ack()
|
||||
continue
|
||||
}
|
||||
}
|
||||
// **Drained in the moment it was fetched** (novox/hq ADR 0335): handed back at once, as a pause
|
||||
// landing then is.
|
||||
if m.opts.Claim != nil && !m.opts.Claim() {
|
||||
_ = msg.Nak()
|
||||
continue
|
||||
}
|
||||
// **A duplicate of a check another builder is running is dropped** (issue 1649): an ask redelivered to this machine
|
||||
// because its first holder did not say, in time, that it had it. The running key names that builder and this same
|
||||
// ask; running it a second time would spend a second place on one check.
|
||||
if why := m.runningElsewhere(ctx, request); why != "" {
|
||||
fmt.Fprintf(os.Stderr, "%s is handed out again while another builder runs it and is dropped: %s\n", request.ID, why)
|
||||
_ = msg.Ack()
|
||||
continue
|
||||
}
|
||||
// A build outlives the acknowledgement window many times over; said while it runs,
|
||||
// as the controller says it for its own long handlers, so the server neither hands
|
||||
// the ask to a second machine nor counts the wait against its deliveries.
|
||||
working := make(chan struct{})
|
||||
go stillWorking(msg, working, heartbeatWhileWorking)
|
||||
// And a check says it is running, to the other builders (ADR 0406 §3): its key put now and at each heartbeat,
|
||||
// deleted when it ends, whatever the outcome.
|
||||
stopRunning := keepRunning(ctx, m.js.Conn(), request, m.on, heartbeatWhileWorking)
|
||||
do(ctx, build)
|
||||
stopRunning()
|
||||
close(working)
|
||||
m.handOver(ctx, msg, q, do)
|
||||
// And the queues it let go of are bound again at once, not after the half minute.
|
||||
bindAgain = time.Time{}
|
||||
}
|
||||
}
|
||||
|
||||
// handOver settles one fetched ask: dropped, handed back or built. **A check let through keeps its key for as long as it
|
||||
// stays here** (ADR 0483 clause 3): refreshed while it runs and deleted when it ends, however it ends.
|
||||
func (m *natsMachine) handOver(ctx context.Context, msg *nats.Msg, q *takeQueue, do func(context.Context, Build)) {
|
||||
if m.letKey {
|
||||
defer m.keepLetThrough(ctx, heartbeatWhileWorking)()
|
||||
}
|
||||
// **Paused while the pull was answered** (ADR 0219): "takes no new build" holds even for
|
||||
// the one that arrived in that moment. Handed back at once for another holder — counted as
|
||||
// a delivery, which a pause landing exactly then costs and nothing else does.
|
||||
if m.opts.Paused != nil && m.opts.Paused() {
|
||||
_ = msg.Nak()
|
||||
return
|
||||
}
|
||||
var request BuildRequest
|
||||
if err := json.Unmarshal(msg.Data, &request); err != nil {
|
||||
// Unreadable: terminated rather than retried, because the next attempt reads the same
|
||||
// bytes. Nobody waiting hears an answer, which is right — there was no request.
|
||||
_ = msg.Term()
|
||||
return
|
||||
}
|
||||
// **Cancelled in the moment it was fetched** (novox/hq ADR 0219): the controller wrote the
|
||||
// id into the seat's cancelled set before deleting the ask, so one taken in between is
|
||||
// ended here, terminated rather than redelivered, and its outcome said as failed — never
|
||||
// built. A set that cannot be read is said and the ask built: a cancel is a person's
|
||||
// exception, and a holder that refused every build while its grant was missing would
|
||||
// stop the mesh building for it.
|
||||
build := &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat, delivery: deliveryOf(msg)}
|
||||
if cancelled, err := IsCancelled(m.js.Conn(), m.seat, request.ID); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read whether %s was cancelled, so it is built: %v\n", request.ID, err)
|
||||
} else if cancelled {
|
||||
endCancelled(ctx, build)
|
||||
return
|
||||
}
|
||||
// **A check left behind is dropped when it is taken** (novox/hq ADR 0406 §3): a request on the third queue for a
|
||||
// head and a trunk commit its pull request's key already says were judged is acknowledged and not run.
|
||||
if q.kind == queueOther {
|
||||
if why := m.judgedAlready(ctx, request); why != "" {
|
||||
fmt.Fprintf(os.Stderr, "%s was left behind in the check queue and is dropped: %s\n", request.ID, why)
|
||||
_ = msg.Ack()
|
||||
return
|
||||
}
|
||||
}
|
||||
if !m.admit(ctx, msg, request) {
|
||||
return
|
||||
}
|
||||
// A build outlives the acknowledgement window many times over; said while it runs,
|
||||
// as the controller says it for its own long handlers, so the server neither hands
|
||||
// the ask to a second machine nor counts the wait against its deliveries.
|
||||
working := make(chan struct{})
|
||||
go stillWorking(msg, working, heartbeatWhileWorking)
|
||||
// And a check says it is running, to the other builders (ADR 0406 §3): its key put now and at each heartbeat,
|
||||
// deleted when it ends, whatever the outcome.
|
||||
stopRunning := keepRunning(ctx, m.js.Conn(), request, m.on, heartbeatWhileWorking)
|
||||
// **And holds nothing else while it runs** (issue 1747): see quiesce.
|
||||
m.quiesce(q)
|
||||
do(ctx, build)
|
||||
stopRunning()
|
||||
close(working)
|
||||
// **The queue it came from too, once the ask is settled**: a fetch that returned a late message leaves its pull request
|
||||
// open, and the server can deliver one more into the subscription (issue 1747). Take binds every queue again first.
|
||||
m.quiesce(nil)
|
||||
}
|
||||
|
||||
// quiesce lets go of every queue but the one the running ask came from, for as long as it runs. **A builder never holds a
|
||||
// message it does not start** (issue 1747). A message the server sends after a fetch gave up, within the few milliseconds its
|
||||
// pull request still lived, lands in that queue's subscription and stays there until the next fetch of it: held, never
|
||||
// acknowledged as in progress, for as long as the builder runs its current ask, five or ten minutes for a check, while the
|
||||
// holders that could build it are all busy and a delivery waits behind it. The legacy pull subscription cannot be read
|
||||
// without sending a new pull request, which could take one more, so it cannot be searched for such a message. It is dropped
|
||||
// instead: the subscription is closed, a message in it or on its way is lost to this builder and the server hands it out again
|
||||
// when its worker's wait for an acknowledgement ends (a minute), to any holder that is free, and the queue is bound again
|
||||
// before the next fetch (Take). The queue the running ask came from keeps its subscription, which its acknowledgements use.
|
||||
func (m *natsMachine) quiesce(keep *takeQueue) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
for _, q := range m.subs {
|
||||
if q == keep || !q.bound {
|
||||
continue
|
||||
}
|
||||
if q.sub != nil {
|
||||
_ = q.sub.Unsubscribe()
|
||||
}
|
||||
q.sub, q.pull, q.bound = nil, nil, false
|
||||
}
|
||||
}
|
||||
|
||||
// admit decides whether a fetched request is run here, and answers the message when it is not.
|
||||
//
|
||||
// **The duplicate is judged before the request is claimed** (issue 1768): a claim marks the build in hand, and a request
|
||||
// dropped after it left the mark behind, so every later claim failed and every ask was handed back at once until all
|
||||
// had spent their deliveries. A request dropped here has marked nothing.
|
||||
//
|
||||
// **Drained in the moment it was fetched** (novox/hq ADR 0335): handed back at once, as a pause landing then is.
|
||||
func (m *natsMachine) admit(ctx context.Context, msg *nats.Msg, request BuildRequest) bool {
|
||||
// **A duplicate of a check another builder is running is dropped** (issue 1649): an ask redelivered to this machine
|
||||
// because its first holder did not say, in time, that it had it. The running key names that builder and this same
|
||||
// ask; running it a second time would spend a second place on one check.
|
||||
elsewhere := m.runningElsewhere
|
||||
if m.duplicate != nil {
|
||||
elsewhere = m.duplicate
|
||||
}
|
||||
if why := elsewhere(ctx, request); why != "" {
|
||||
fmt.Fprintf(os.Stderr, "%s is handed out again while another builder runs it and is dropped: %s\n", request.ID, why)
|
||||
_ = msg.Ack()
|
||||
return false
|
||||
}
|
||||
if m.opts.Claim != nil && !m.opts.Claim() {
|
||||
_ = msg.Nak()
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// runningElsewhere says why an ask need not run here, or nothing: the bucket of the checks running says another builder
|
||||
// runs this very ask (its id, its head and its trunk commit). A key that cannot be read runs the check.
|
||||
func (m *natsMachine) runningElsewhere(ctx context.Context, r BuildRequest) string {
|
||||
@@ -420,25 +514,39 @@ func duplicateOf(running CheckRunning, found bool, r BuildRequest, me string) st
|
||||
// subscribing `…accept.build` against a consumer filtered on `…accept.>` is rejected even though it is narrower. Learned
|
||||
// twice now, on two different consumers, which is why it is written down here.
|
||||
func (m *natsMachine) bind(q *takeQueue) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
sub, err := m.js.Context().PullSubscribe(q.worker.Filters[0], q.worker.Name,
|
||||
nats.Bind(q.worker.Stream, q.worker.Name), nats.ManualAck())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
q.sub, q.bound = sub, true
|
||||
q.sub, q.pull, q.bound = sub, sub, true
|
||||
return nil
|
||||
}
|
||||
|
||||
// next takes the next request in the order of the queues: each tried for a moment, and when all are empty the first waited
|
||||
// on for a second. Nothing, with no error, when there was nothing.
|
||||
func (m *natsMachine) next(ctx context.Context) (*nats.Msg, *takeQueue, error) {
|
||||
// **While a delivery builds, only delivery builds are taken, but for one check that waited** (novox/hq ADR 0483): the
|
||||
// bucket of the plans building is read before either check queue is fetched from, and a key there holds both. The check
|
||||
// that waited LetThroughAfter goes first, ahead of the builds that wait; the rest of the holders take builds only.
|
||||
if m.seat == broker.TheBuildSeat && len(m.subs) > 1 {
|
||||
if plans := m.building(ctx); len(plans) > 0 {
|
||||
if msg, q := m.letOneThrough(ctx); msg != nil {
|
||||
fmt.Fprintf(os.Stderr, "a check that waited %s is let through while %s builds\n", m.letThroughAfter(), whyHeld(plans))
|
||||
return msg, q, nil
|
||||
}
|
||||
return m.nextBuild(ctx)
|
||||
}
|
||||
}
|
||||
var last error
|
||||
for _, q := range m.subs {
|
||||
if !q.bound {
|
||||
continue
|
||||
}
|
||||
trying, end := context.WithTimeout(ctx, checkTakeWait)
|
||||
fetched, err := q.sub.Fetch(1, nats.Context(trying))
|
||||
fetched, err := q.pull.Fetch(1, nats.Context(trying))
|
||||
end()
|
||||
if len(fetched) > 0 {
|
||||
return fetched[0], q, nil
|
||||
@@ -454,9 +562,18 @@ func (m *natsMachine) next(ctx context.Context) (*nats.Msg, *takeQueue, error) {
|
||||
if last != nil {
|
||||
return nil, nil, last
|
||||
}
|
||||
return m.nextBuild(ctx)
|
||||
}
|
||||
|
||||
// nextBuild waits for a second at most for a delivery build, the seat's own queue.
|
||||
func (m *natsMachine) nextBuild(ctx context.Context) (*nats.Msg, *takeQueue, error) {
|
||||
if !m.subs[0].bound {
|
||||
// Let go of while an ask ran and not bound again yet (Take binds it before it asks): nothing to take from.
|
||||
return nil, nil, nil
|
||||
}
|
||||
waiting, end := context.WithTimeout(ctx, takeIdleWait)
|
||||
defer end()
|
||||
fetched, err := m.subs[0].sub.Fetch(1, nats.Context(waiting))
|
||||
fetched, err := m.subs[0].pull.Fetch(1, nats.Context(waiting))
|
||||
if len(fetched) > 0 {
|
||||
return fetched[0], m.subs[0], nil
|
||||
}
|
||||
@@ -619,3 +736,16 @@ func endCancelled(ctx context.Context, b *natsBuild) {
|
||||
}
|
||||
_ = b.msg.Term()
|
||||
}
|
||||
|
||||
// boundKinds are the kinds of queue this machine holds a subscription to now.
|
||||
func (m *natsMachine) boundKinds() []string {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
var out []string
|
||||
for _, q := range m.subs {
|
||||
if q.bound {
|
||||
out = append(out, q.kind)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -287,12 +287,16 @@ type CheckRunning struct {
|
||||
}
|
||||
|
||||
// RunningKeyOf is the key a check is said running under: its pull request's tokens, or `group.<name>` for a group's
|
||||
// composed check; none for a try, which no other request replaces.
|
||||
// composed check; `try.<id>` for a try, which no other request replaces.
|
||||
func RunningKeyOf(r BuildRequest) (string, bool) {
|
||||
c := r.Check
|
||||
switch {
|
||||
case c == nil, c.Try:
|
||||
case c == nil:
|
||||
return "", false
|
||||
case c.Try:
|
||||
// A try says it runs under its own id, which no other request shares: so a reader of the bucket can tell a try that runs
|
||||
// from a check left waiting for a redelivery (ADR 0483 clause 3).
|
||||
return "try." + tokenOf(r.ID), true
|
||||
case c.Group != "":
|
||||
return "group." + tokenOf(c.Group), true
|
||||
case c.Number <= 0:
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -595,21 +594,12 @@ func TestABusyMachineTakesNoWaitingAskWhateverItsPullsHold(t *testing.T) {
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("builder a took nothing")
|
||||
}
|
||||
var other *takeQueue
|
||||
for _, q := range a.subs {
|
||||
if q.kind == queueOther && q.bound {
|
||||
other = q
|
||||
}
|
||||
}
|
||||
if other == nil {
|
||||
t.Fatal("builder a has no pull on the other checks")
|
||||
}
|
||||
// The queue is empty, so this request is answered at its expiry with a status message that lands in the pull.
|
||||
reply := strings.TrimSuffix(other.sub.Subject, "*") + "late"
|
||||
if err := js.Conn().PublishRequest("$JS.API.CONSUMER.MSG.NEXT."+other.worker.Stream+"."+other.worker.Name, reply,
|
||||
[]byte(`{"batch":1,"expires":1000000}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
// **A busy machine holds no pull at all** (issues 1649 and 1747): the queues the running ask did not come from are let go of,
|
||||
// so no message, a status message or a late delivery, can sit in one while it runs.
|
||||
if kinds := a.boundKinds(); len(kinds) != 1 || kinds[0] != queueBuilds {
|
||||
t.Fatalf("builder a holds %v while it builds; want only the queue its build came from", kinds)
|
||||
}
|
||||
other := &takeQueue{kind: queueOther, worker: broker.CheckWorkers()[1]}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
for i, id := range []string{"wait-1", "wait-2"} {
|
||||
if err := asks.Ask(ctx, aPullCheck(id, 20+i, "head-x-000000000", "trunk-1-000000", false)); err != nil {
|
||||
|
||||
@@ -0,0 +1,479 @@
|
||||
//go:build integration
|
||||
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/nats-io/nats.go"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// Checks wait while a delivery builds (novox/hq ADR 0483, issues 1652, 1687, 1688), against a real bus.
|
||||
|
||||
// aBusWithTheBuckets is the bus with the build role and the controller's buckets, the bucket of the checks running with an age
|
||||
// of two seconds so a key a holder stopped refreshing is seen to expire.
|
||||
func aBusWithTheBuckets(t *testing.T) (*broker.JetStream, jetstream.KeyValue) {
|
||||
t.Helper()
|
||||
js := aBusWithTheBuildRole(t)
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := t.Context()
|
||||
if _, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: broker.CheckRunningBucket, History: 1, TTL: 2 * time.Second}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plans, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: broker.PlansBuildingBucket, History: 1, TTL: broker.PlansBuildingKeptFor})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return js, plans
|
||||
}
|
||||
|
||||
func aCheckAskedAgo(ago time.Duration, number int, ready bool) BuildRequest {
|
||||
r := aPullCheck(fmt.Sprintf("build-%d", time.Now().Add(-ago).UnixNano()), number, fmt.Sprintf("head-%d-0000000", number), "trunk-0-000000", ready)
|
||||
return r
|
||||
}
|
||||
|
||||
// 1652/AC6: a plan's builds asked in two tiers with a check waiting, the check is handed out only after the last tier's builds,
|
||||
// and not in the gap between the tiers.
|
||||
func TestNatsACheckIsHandedOutOnlyAfterTheLastTierOfAPlan(t *testing.T) {
|
||||
js, plans := aBusWithTheBuckets(t)
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
defer stop()
|
||||
asks := BuildsOn(js, TheBuildMachine)
|
||||
|
||||
if _, err := plans.Put(ctx, "plan-1", []byte(`{"plan":"plan-1"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := asks.Ask(ctx, aCheckAskedAgo(time.Minute, 7, true)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := asks.Ask(ctx, BuildRequest{ID: "tier-1-a", Repository: "/a"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
took := make(chan string, 16)
|
||||
machine := MachineOverNATS(js, "anchor")
|
||||
defer machine.Close()
|
||||
go func() {
|
||||
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
|
||||
took <- work.Request().ID
|
||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
|
||||
_ = work.Done()
|
||||
})
|
||||
}()
|
||||
expect := func(want string) {
|
||||
t.Helper()
|
||||
select {
|
||||
case got := <-took:
|
||||
if got != want {
|
||||
t.Fatalf("taken %s, want %s", got, want)
|
||||
}
|
||||
case <-time.After(15 * time.Second):
|
||||
t.Fatalf("%s was not taken", want)
|
||||
}
|
||||
}
|
||||
expect("tier-1-a")
|
||||
// The gap between two tiers: the delivery's queue is empty, the plan's key is there, and the check is not taken.
|
||||
select {
|
||||
case got := <-took:
|
||||
t.Fatalf("%s was taken in the gap between two tiers", got)
|
||||
case <-time.After(3 * time.Second):
|
||||
}
|
||||
if err := asks.Ask(ctx, BuildRequest{ID: "tier-2-a", Repository: "/b"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expect("tier-2-a")
|
||||
select {
|
||||
case got := <-took:
|
||||
t.Fatalf("%s was taken before the plan's last build was built and its key deleted", got)
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
// The phase ends: the key is deleted, and the check is handed out.
|
||||
if err := plans.Delete(ctx, "plan-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case got := <-took:
|
||||
if got == "tier-1-a" || got == "tier-2-a" {
|
||||
t.Fatalf("a build was taken twice: %s", got)
|
||||
}
|
||||
case <-time.After(15 * time.Second):
|
||||
t.Fatal("the check was not handed out once the plan's key was deleted")
|
||||
}
|
||||
}
|
||||
|
||||
// 1688: two holders racing for checks that waited past the bound: exactly one is let through at a time, by a create that fails
|
||||
// while the key is there, and the key is deleted when its check ends, after which the next oldest goes.
|
||||
func TestNatsTwoHoldersRacingForOldChecksLetExactlyOneThrough(t *testing.T) {
|
||||
js, plans := aBusWithTheBuckets(t)
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
defer stop()
|
||||
// The buckets' age is two seconds here, so the keys are refreshed well within it.
|
||||
was := heartbeatWhileWorking
|
||||
heartbeatWhileWorking = 400 * time.Millisecond
|
||||
t.Cleanup(func() { heartbeatWhileWorking = was })
|
||||
asks := BuildsOn(js, TheBuildMachine)
|
||||
if _, err := plans.Put(ctx, "plan-1", []byte(`{"plan":"plan-1"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, ago := range []time.Duration{50 * time.Minute, 40 * time.Minute, 35 * time.Minute} {
|
||||
if err := asks.Ask(ctx, aCheckAskedAgo(ago, i+1, false)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var mu sync.Mutex
|
||||
running, most := 0, 0
|
||||
started := make(chan string, 8)
|
||||
release := make(chan struct{})
|
||||
do := func(ctx context.Context, work Build) {
|
||||
mu.Lock()
|
||||
running++
|
||||
most = max(most, running)
|
||||
mu.Unlock()
|
||||
started <- work.Request().Check.Repo + fmt.Sprint(work.Request().Check.Number)
|
||||
<-release
|
||||
mu.Lock()
|
||||
running--
|
||||
mu.Unlock()
|
||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "x", Failed: "no"})
|
||||
_ = work.Done()
|
||||
}
|
||||
for _, on := range []string{"ace", "shanks"} {
|
||||
machine := MachineOverNATS(js, on)
|
||||
defer machine.Close()
|
||||
go func() { _ = machine.Take(ctx, do) }()
|
||||
}
|
||||
var order []string
|
||||
for range 3 {
|
||||
select {
|
||||
case id := <-started:
|
||||
order = append(order, id)
|
||||
case <-time.After(20 * time.Second):
|
||||
t.Fatalf("started so far: %v", order)
|
||||
}
|
||||
// While it runs, no second check starts and the key is there.
|
||||
select {
|
||||
case id := <-started:
|
||||
t.Fatalf("%s started beside a check let through", id)
|
||||
case <-time.After(2500 * time.Millisecond):
|
||||
}
|
||||
kv := runningKV(t, js)
|
||||
if _, err := kv.Get(ctx, LetThroughKey); err != nil {
|
||||
t.Fatalf("the let-through key is not there while the check runs: %v", err)
|
||||
}
|
||||
release <- struct{}{}
|
||||
// It is deleted when the check ends: the next check let through creates the key again at once, and a create fails
|
||||
// while the key is there, so its start (the next round) is the proof; after the last one the key stays gone.
|
||||
if len(order) == 3 {
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for {
|
||||
if _, err := kv.Get(ctx, LetThroughKey); err != nil {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the let-through key was not deleted when the last check ended")
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
}
|
||||
if want := []string{"a1", "a2", "a3"}; !reflect.DeepEqual(order, want) {
|
||||
t.Fatalf("checks went %v, want the oldest first %v", order, want)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if most != 1 {
|
||||
t.Fatalf("%d checks ran at once while a delivery built", most)
|
||||
}
|
||||
}
|
||||
|
||||
func runningKV(t *testing.T, js *broker.JetStream) jetstream.KeyValue {
|
||||
t.Helper()
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv, err := api.KeyValue(t.Context(), broker.CheckRunningBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return kv
|
||||
}
|
||||
|
||||
// 1688: the create of the let-through key succeeds for exactly one of holders asking at once, and a key its holder stopped
|
||||
// refreshing expires, after which the next create succeeds.
|
||||
func TestNatsOneCreateOfTheLetThroughKeySucceedsAndAStoppedHoldersKeyExpires(t *testing.T) {
|
||||
js, _ := aBusWithTheBuckets(t)
|
||||
gate := busGate{conn: js.Conn()}
|
||||
ctx := t.Context()
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
won := 0
|
||||
for i := range 8 {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
ok, err := gate.TakeLetThrough(ctx, fmt.Sprint("holder-", i))
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
if ok {
|
||||
mu.Lock()
|
||||
won++
|
||||
mu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
if won != 1 {
|
||||
t.Fatalf("%d of 8 holders created the let-through key, want exactly one", won)
|
||||
}
|
||||
// The holder stopped: nothing refreshes the key, and the bucket's age (two seconds here) removes it.
|
||||
deadline := time.Now().Add(15 * time.Second)
|
||||
for {
|
||||
ok, err := gate.TakeLetThrough(ctx, "next")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ok {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the key of a stopped holder did not expire")
|
||||
}
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
}
|
||||
// And a refreshed key does not expire.
|
||||
for range 6 {
|
||||
if err := gate.KeepLetThrough(ctx, "next"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(700 * time.Millisecond)
|
||||
}
|
||||
if ok, _ := gate.TakeLetThrough(ctx, "other"); ok {
|
||||
t.Fatal("a refreshed key expired")
|
||||
}
|
||||
var v LetThrough
|
||||
e, err := runningKV(t, js).Get(ctx, LetThroughKey)
|
||||
if err != nil || json.Unmarshal(e.Value(), &v) != nil || v.Builder != "next" {
|
||||
t.Fatalf("the key says %+v (%v)", v, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1687: the oldest waiting ask is read without being taken: no delivery is counted against it.
|
||||
func TestNatsTheOldestWaitingAskIsReadWithoutTakingIt(t *testing.T) {
|
||||
js, _ := aBusWithTheBuckets(t)
|
||||
ctx := t.Context()
|
||||
asks := BuildsOn(js, TheBuildMachine)
|
||||
old := aCheckAskedAgo(45*time.Minute, 1, false)
|
||||
if err := asks.Ask(ctx, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gate := busGate{conn: js.Conn()}
|
||||
q := &takeQueue{kind: queueOther, worker: broker.CheckWorkers()[1]}
|
||||
at, found, err := gate.OldestWaiting(ctx, q)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("the first waiting ask was not read: %v %v", found, err)
|
||||
}
|
||||
if want, _ := BuildAskedAt(old.ID); !at.Equal(want) {
|
||||
t.Fatalf("read %v, asked %v", at, want)
|
||||
}
|
||||
for range 3 {
|
||||
if _, _, err := gate.OldestWaiting(ctx, q); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
info, err := js.Context().ConsumerInfo(q.worker.Stream, q.worker.Name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.Delivered.Consumer != 0 || info.NumAckPending != 0 || info.NumPending != 1 {
|
||||
t.Fatalf("reading the age took the ask: delivered %d, pending acks %d, waiting %d", info.Delivered.Consumer, info.NumAckPending, info.NumPending)
|
||||
}
|
||||
// And an empty queue reads as none.
|
||||
if _, found, err := gate.OldestWaiting(ctx, &takeQueue{kind: queueReady, worker: broker.CheckWorkers()[0]}); err != nil || found {
|
||||
t.Fatalf("an empty queue read as %v (%v)", found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1747: a message that reaches a builder's subscription outside a fetch is not held while the builder runs another ask. A
|
||||
// stray check lands in the other queue's subscription of builder A, which then runs a long check; the server hands the stray
|
||||
// out again when its wait for an acknowledgement ends (two seconds here), builder B runs it, and A, finished, does not run it a
|
||||
// second time from the subscription it held.
|
||||
func TestNatsAMessageDeliveredOutsideAFetchIsNotHeldWhileTheBuilderRunsAnotherAsk(t *testing.T) { // 1747
|
||||
js := aBusWithTheBuildRole(t)
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
defer stop()
|
||||
for _, w := range broker.CheckWorkers() {
|
||||
w.AckWaitSeconds = 2
|
||||
if err := js.EnsureConsumer(w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// The long check says it is alive well within the ack wait of two seconds.
|
||||
was := heartbeatWhileWorking
|
||||
heartbeatWhileWorking = 400 * time.Millisecond
|
||||
t.Cleanup(func() { heartbeatWhileWorking = was })
|
||||
asks := BuildsOn(js, TheBuildMachine)
|
||||
long := aPullCheck("build-"+fmt.Sprint(time.Now().UnixNano()), 1, "head-1-0000000", "trunk-0-000000", true)
|
||||
stray := aPullCheck("build-"+fmt.Sprint(time.Now().UnixNano()+1), 2, "head-2-0000000", "trunk-0-000000", false)
|
||||
for _, r := range []BuildRequest{long, stray} {
|
||||
if err := asks.Ask(ctx, r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
var mu sync.Mutex
|
||||
runs := map[string][]string{} // ask id -> the machines that ran it
|
||||
release := make(chan struct{})
|
||||
started := make(chan string, 8)
|
||||
do := func(on string) func(context.Context, Build) {
|
||||
return func(ctx context.Context, work Build) {
|
||||
id := work.Request().ID
|
||||
mu.Lock()
|
||||
runs[id] = append(runs[id], on)
|
||||
mu.Unlock()
|
||||
started <- on + ":" + id
|
||||
if id == long.ID {
|
||||
<-release
|
||||
}
|
||||
_ = work.Announce(ctx, BuildResult{ID: id, On: on, Failed: "no"})
|
||||
_ = work.Done()
|
||||
}
|
||||
}
|
||||
|
||||
var a *natsMachine
|
||||
strayLanded := make(chan error, 1)
|
||||
a = MachineOverNATSWith(js, "ace", TheBuildMachine, MachineOptions{Claim: func() bool {
|
||||
// The moment the long check is claimed: the server hands the other queue's waiting check to A's subscription there, as
|
||||
// one that arrives after a fetch gave up does. Done once.
|
||||
select {
|
||||
case <-strayLanded:
|
||||
return true
|
||||
default:
|
||||
}
|
||||
other := a.subs[2]
|
||||
inbox := strings.TrimSuffix(other.sub.Subject, "*") + "stray"
|
||||
body, _ := json.Marshal(map[string]any{"batch": 1, "expires": int64(30 * time.Second)})
|
||||
if err := js.Conn().PublishRequest("$JS.API.CONSUMER.MSG.NEXT."+other.worker.Stream+"."+other.worker.Name, inbox, body); err != nil {
|
||||
strayLanded <- err
|
||||
return true
|
||||
}
|
||||
_ = js.Conn().Flush()
|
||||
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) {
|
||||
if n, _, _ := other.sub.Pending(); n > 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
strayLanded <- nil
|
||||
return true
|
||||
}}).(*natsMachine)
|
||||
defer a.Close()
|
||||
go func() { _ = a.Take(ctx, do("ace")) }()
|
||||
|
||||
select {
|
||||
case got := <-started:
|
||||
if got != "ace:"+long.ID {
|
||||
t.Fatalf("started %s, want the long check on ace first", got)
|
||||
}
|
||||
case <-time.After(15 * time.Second):
|
||||
t.Fatal("the long check did not start")
|
||||
}
|
||||
if err := <-strayLanded; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// B, free, runs the stray once the server hands it out again.
|
||||
b := MachineOverNATS(js, "shanks")
|
||||
defer b.Close()
|
||||
go func() { _ = b.Take(ctx, do("shanks")) }()
|
||||
select {
|
||||
case got := <-started:
|
||||
if got != "shanks:"+stray.ID {
|
||||
t.Fatalf("started %s, want the stray on shanks", got)
|
||||
}
|
||||
case <-time.After(20 * time.Second):
|
||||
t.Fatal("the stray was held by the builder that ran the long check and nobody ran it")
|
||||
}
|
||||
// A finishes, and does not run the stray from the subscription it held.
|
||||
release <- struct{}{}
|
||||
select {
|
||||
case got := <-started:
|
||||
t.Fatalf("%s started after the long check: the stray ran twice", got)
|
||||
case <-time.After(6 * time.Second):
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(runs[stray.ID]) != 1 {
|
||||
t.Fatalf("the stray ran on %v, want once", runs[stray.ID])
|
||||
}
|
||||
}
|
||||
|
||||
// Review of 1688: a young check pending redelivery (handed back by a Nak) stands ahead of a check older than 30 minutes in the
|
||||
// same queue while a plan builds. The let-through fetch would return the young one first, so nothing is fetched: the young
|
||||
// check's delivery count stays 1, it is not handed back every second until it is dead-lettered, and it is not run.
|
||||
func TestNatsAYoungRedeliveryAheadOfAnOldCheckIsNeverFetchedByTheLetThrough(t *testing.T) {
|
||||
js, plans := aBusWithTheBuckets(t)
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
defer stop()
|
||||
asks := BuildsOn(js, TheBuildMachine)
|
||||
young := aCheckAskedAgo(time.Minute, 1, false)
|
||||
if err := asks.Ask(ctx, young); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Someone took it and handed it back: it is pending redelivery.
|
||||
worker := broker.CheckWorkers()[1]
|
||||
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := sub.Fetch(1, nats.MaxWait(5*time.Second))
|
||||
if err != nil || len(got) != 1 {
|
||||
t.Fatalf("the young check was not fetched to be handed back: %v", err)
|
||||
}
|
||||
if err := got[0].Nak(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = sub.Unsubscribe()
|
||||
old := aCheckAskedAgo(45*time.Minute, 2, false)
|
||||
if err := asks.Ask(ctx, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := plans.Put(ctx, "plan-1", []byte(`{"plan":"plan-1"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
started := make(chan string, 4)
|
||||
machine := MachineOverNATS(js, "ace")
|
||||
defer machine.Close()
|
||||
go func() {
|
||||
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
|
||||
started <- work.Request().ID
|
||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "ace", Failed: "no"})
|
||||
_ = work.Done()
|
||||
})
|
||||
}()
|
||||
select {
|
||||
case id := <-started:
|
||||
t.Fatalf("%s was run while the young redelivery stood ahead of it", id)
|
||||
case <-time.After(6 * time.Second):
|
||||
}
|
||||
info, err := js.Context().ConsumerInfo(worker.Stream, worker.Name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.Delivered.Consumer != 1 || info.NumRedelivered != 0 {
|
||||
t.Fatalf("the young check was handed out %d time(s), %d redelivered: the let-through fetched it", info.Delivered.Consumer, info.NumRedelivered)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,384 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// Checks wait while a delivery builds (novox/hq ADR 0483, issues 1652, 1687 and 1688).
|
||||
//
|
||||
// **The layout other parts read** (ADR 0484 clause 12 has the checker read the same key; a layout shared between modules
|
||||
// belongs in mesh-sdk, and this one is not there yet):
|
||||
//
|
||||
// - bucket `mesh-controller_plans-building` (broker.PlansBuildingBucket): one key per delivery plan in its build phase, named
|
||||
// by the plan's id (`plan-<digits>`), value a PlanBuilding as JSON. Written by the controller (lease holder) alone: put when
|
||||
// the plan's first build is asked, put again about every thirty seconds, deleted when the phase ends; the bucket's age
|
||||
// (two minutes) removes the key of a controller that stopped. A reader lists the keys and needs no value to decide.
|
||||
// - key `let-through` in bucket `mesh-controller_checks-running` (broker.CheckRunningBucket), value a LetThrough as JSON:
|
||||
// created by the holder that takes a check which waited LetThroughAfter, refreshed at each heartbeat of that check, deleted
|
||||
// when it ends. At most one such check runs at a time on the build seat, because the create fails while the key is there.
|
||||
|
||||
// LetThroughAfter is how long a check waits, while a delivery builds, before it is let through (ADR 0483 clause 3).
|
||||
const LetThroughAfter = 30 * time.Minute
|
||||
|
||||
// LetThroughKey is the key of the one check let through, in the bucket of the checks running. It has no dot, so it is no
|
||||
// pull request's key (`<owner>.<repo>.<number>`) and no group's (`group.<name>`).
|
||||
const LetThroughKey = "let-through"
|
||||
|
||||
// PlanBuilding is the value of a plan's key in the bucket of the plans building.
|
||||
type PlanBuilding struct {
|
||||
Plan string `json:"plan"`
|
||||
Since time.Time `json:"since"`
|
||||
Seen time.Time `json:"seen"`
|
||||
}
|
||||
|
||||
// LetThrough is the value of the let-through key: the builder that runs the check and when it took it.
|
||||
type LetThrough struct {
|
||||
Builder string `json:"builder"`
|
||||
Since time.Time `json:"since"`
|
||||
}
|
||||
|
||||
// checkGate is what a taking loop asks of the bus to hold checks while a delivery builds; a variable of the machine so a test
|
||||
// hands out against fakes.
|
||||
type checkGate interface {
|
||||
// Building is the ids of the plans in their build phase: the keys of the bucket. An error is a bucket that cannot be read.
|
||||
Building(ctx context.Context) ([]string, error)
|
||||
// OldestWaiting is when the first waiting ask of a check queue was asked, read without taking it.
|
||||
OldestWaiting(ctx context.Context, q *takeQueue) (time.Time, bool, error)
|
||||
// TakeLetThrough creates the let-through key for a builder: false when it is there (another builder runs a check let through).
|
||||
TakeLetThrough(ctx context.Context, builder string) (bool, error)
|
||||
// KeepLetThrough puts the key again, and DropLetThrough deletes it.
|
||||
KeepLetThrough(ctx context.Context, builder string) error
|
||||
DropLetThrough(ctx context.Context) error
|
||||
}
|
||||
|
||||
// busGate is the gate over a connection.
|
||||
type busGate struct{ conn *nats.Conn }
|
||||
|
||||
func (g busGate) Building(ctx context.Context) ([]string, error) {
|
||||
return BuildingPlans(ctx, g.conn)
|
||||
}
|
||||
|
||||
// BuildingPlans is the ids of the delivery plans in their build phase, from the controller's bucket. A reader that is not the
|
||||
// build seat's (the checker, ADR 0484 clause 12) calls this. A bucket that cannot be read is an error; none is none building.
|
||||
func BuildingPlans(ctx context.Context, conn *nats.Conn) ([]string, error) {
|
||||
api, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, broker.PlansBuildingBucket)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the plans building: %w", err)
|
||||
}
|
||||
keys, err := kv.Keys(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the plans building: %w", err)
|
||||
}
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
// OldestWaiting is when the check a fetch of this queue would return was asked, read without taking anything.
|
||||
//
|
||||
// **It sees redeliveries** (review of issue 1688). A work queue keeps only what is not acknowledged: what the worker handed out
|
||||
// (sequence up to its delivered mark) is running or waiting to be handed out again, and a fetch returns those before anything
|
||||
// new. An ask there that is not said running (a key of the checks running names its build) is such a redelivery. If one is
|
||||
// younger than the bound, the queue's next fetch would return it, and a young check is never let through: the queue is not a
|
||||
// candidate, and nothing is fetched, so nothing is handed back and no delivery is spent. Otherwise the candidate is the oldest
|
||||
// of the redeliveries and the first ask after the delivered mark.
|
||||
func (g busGate) OldestWaiting(ctx context.Context, q *takeQueue) (time.Time, bool, error) {
|
||||
info, err := jsOf(g.conn).ConsumerInfo(q.worker.Stream, q.worker.Name, nats.Context(ctx))
|
||||
if err != nil {
|
||||
return time.Time{}, false, fmt.Errorf("the %s worker: %w", q.kind, err)
|
||||
}
|
||||
var oldest time.Time
|
||||
note := func(at time.Time) {
|
||||
if oldest.IsZero() || at.Before(oldest) {
|
||||
oldest = at
|
||||
}
|
||||
}
|
||||
for seq := uint64(1); seq <= info.Delivered.Stream; {
|
||||
data, stored, got, err := g.directGet(ctx, q, seq)
|
||||
if err != nil {
|
||||
return time.Time{}, false, err
|
||||
}
|
||||
if got == 0 || got > info.Delivered.Stream {
|
||||
break
|
||||
}
|
||||
seq = got + 1
|
||||
var r BuildRequest
|
||||
if json.Unmarshal(data, &r) != nil {
|
||||
continue
|
||||
}
|
||||
running, ok, err := ReadRunning(ctx, g.conn, r)
|
||||
if err != nil {
|
||||
return time.Time{}, false, fmt.Errorf("the checks running: %w", err)
|
||||
}
|
||||
if ok && running.Build == r.ID {
|
||||
continue // running now
|
||||
}
|
||||
at := askedAtOf(data, stored)
|
||||
if time.Since(at) < LetThroughAfter {
|
||||
return time.Time{}, false, nil // a young redelivery is next in line
|
||||
}
|
||||
note(at)
|
||||
}
|
||||
data, stored, got, err := g.directGet(ctx, q, info.Delivered.Stream+1)
|
||||
if err != nil {
|
||||
return time.Time{}, false, err
|
||||
}
|
||||
if got != 0 {
|
||||
note(askedAtOf(data, stored))
|
||||
}
|
||||
return oldest, !oldest.IsZero(), nil
|
||||
}
|
||||
|
||||
// directGet reads the first ask at or after a sequence of a check queue's stream, without taking it: its body, the time the
|
||||
// stream stored it, and its sequence (0 when there is none).
|
||||
func (g busGate) directGet(ctx context.Context, q *takeQueue, seq uint64) ([]byte, string, uint64, error) {
|
||||
body, _ := json.Marshal(map[string]any{"seq": seq, "next_by_subj": q.worker.Filters[0]})
|
||||
reply, err := g.conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET."+q.worker.Stream, body)
|
||||
if err != nil {
|
||||
return nil, "", 0, fmt.Errorf("the asks of the %s queue: %w", q.kind, err)
|
||||
}
|
||||
if st := reply.Header.Get("Status"); st == "404" || st == "408" {
|
||||
return nil, "", 0, nil
|
||||
}
|
||||
got, _ := strconv.ParseUint(reply.Header.Get("Nats-Sequence"), 10, 64)
|
||||
return reply.Data, reply.Header.Get("Nats-Time-Stamp"), got, nil
|
||||
}
|
||||
|
||||
// askedAtOf is when a request was asked: its id says (NewBuildID), the stream's own time otherwise, and a request that says
|
||||
// neither is taken as asked now, so it is never let through on a guess.
|
||||
func askedAtOf(data []byte, stored string) time.Time {
|
||||
var r BuildRequest
|
||||
if json.Unmarshal(data, &r) == nil {
|
||||
if at, ok := BuildAskedAt(r.ID); ok {
|
||||
return at
|
||||
}
|
||||
}
|
||||
if at, err := time.Parse(time.RFC3339Nano, stored); err == nil {
|
||||
return at
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
func jsOf(conn *nats.Conn) nats.JetStreamContext {
|
||||
js, _ := conn.JetStream()
|
||||
return js
|
||||
}
|
||||
|
||||
func (g busGate) TakeLetThrough(ctx context.Context, builder string) (bool, error) {
|
||||
kv, err := runningBucket(ctx, g.conn)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
body, _ := json.Marshal(LetThrough{Builder: builder, Since: time.Now().UTC()})
|
||||
if _, err := kv.Create(ctx, LetThroughKey, body); err != nil {
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (g busGate) KeepLetThrough(ctx context.Context, builder string) error {
|
||||
kv, err := runningBucket(ctx, g.conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body, _ := json.Marshal(LetThrough{Builder: builder, Since: time.Now().UTC()})
|
||||
_, err = kv.Put(ctx, LetThroughKey, body)
|
||||
return err
|
||||
}
|
||||
|
||||
func (g busGate) DropLetThrough(ctx context.Context) error {
|
||||
kv, err := runningBucket(ctx, g.conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := kv.Delete(ctx, LetThroughKey); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CheckHold is what a reader of the queue (the controller's own verb) says of why checks wait: the plans building and the check
|
||||
// let through now, if one.
|
||||
type CheckHold struct {
|
||||
Plans []string
|
||||
LetThrough *LetThrough
|
||||
}
|
||||
|
||||
// ReadCheckHold reads the plans building and the let-through key. A bucket that cannot be read is an error the caller says.
|
||||
func ReadCheckHold(ctx context.Context, conn *nats.Conn) (CheckHold, error) {
|
||||
var h CheckHold
|
||||
plans, err := BuildingPlans(ctx, conn)
|
||||
if err != nil {
|
||||
return h, err
|
||||
}
|
||||
h.Plans = plans
|
||||
kv, err := runningBucket(ctx, conn)
|
||||
if err != nil {
|
||||
return h, err
|
||||
}
|
||||
e, err := kv.Get(ctx, LetThroughKey)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return h, nil
|
||||
}
|
||||
if err != nil {
|
||||
return h, err
|
||||
}
|
||||
var l LetThrough
|
||||
if json.Unmarshal(e.Value(), &l) == nil {
|
||||
h.LetThrough = &l
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// gateOf is the machine's gate: the one a test set, or the bus's.
|
||||
func (m *natsMachine) gateOf() checkGate {
|
||||
if m.gate != nil {
|
||||
return m.gate
|
||||
}
|
||||
return busGate{conn: m.js.Conn()}
|
||||
}
|
||||
|
||||
// sayOnce says a fault of the hold the first time it happens and again once it has recovered and happened anew, so a bucket
|
||||
// that cannot be read is in the log once and not every second.
|
||||
func (m *natsMachine) sayOnce(kind string, err error) {
|
||||
if m.said == nil {
|
||||
m.said = map[string]bool{}
|
||||
}
|
||||
if err == nil {
|
||||
delete(m.said, kind)
|
||||
return
|
||||
}
|
||||
if !m.said[kind] {
|
||||
m.said[kind] = true
|
||||
fmt.Fprintf(os.Stderr, "%s; read as empty, so no check is held for it: %v\n", kind, err)
|
||||
}
|
||||
}
|
||||
|
||||
// building says whether a delivery builds: any key in the bucket of the plans building. **A bucket that cannot be read is said
|
||||
// and read as empty** (ADR 0483 clause 2): a check wrongly held costs a pull request its turn, a fault that held every check
|
||||
// would stop the mesh checking.
|
||||
func (m *natsMachine) building(ctx context.Context) []string {
|
||||
plans, err := m.gateOf().Building(ctx)
|
||||
m.sayOnce("the bucket of the plans building cannot be read", err)
|
||||
if err != nil {
|
||||
// empty-on-error: ADR 0483 clause 2 — a bucket that cannot be read is said (sayOnce) and read as empty, because a check wrongly held costs a pull request its turn and a fault that held every check would stop the mesh checking.
|
||||
return nil
|
||||
}
|
||||
return plans
|
||||
}
|
||||
|
||||
// letOneThrough takes the oldest waiting check of the two check queues once it waited LetThroughAfter, by the one let-through
|
||||
// key: nil when none waited that long, when another holder runs the check let through, or when the check was taken by
|
||||
// another holder in the moment. The caller owns the key while it returns a message and releases it with dropLetThrough.
|
||||
func (m *natsMachine) letOneThrough(ctx context.Context) (*nats.Msg, *takeQueue) {
|
||||
gate := m.gateOf()
|
||||
var oldest *takeQueue
|
||||
var oldestAt time.Time
|
||||
for _, q := range m.subs[1:] {
|
||||
if !q.bound {
|
||||
continue
|
||||
}
|
||||
at, found, err := gate.OldestWaiting(ctx, q)
|
||||
m.sayOnce("the first waiting check of the "+q.kind+" queue cannot be read", err)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
if oldest == nil || at.Before(oldestAt) {
|
||||
oldest, oldestAt = q, at
|
||||
}
|
||||
}
|
||||
if oldest == nil || time.Since(oldestAt) < m.letThroughAfter() {
|
||||
return nil, nil
|
||||
}
|
||||
created, err := gate.TakeLetThrough(ctx, m.on)
|
||||
m.sayOnce("the let-through key cannot be taken", err)
|
||||
if err != nil || !created {
|
||||
return nil, nil
|
||||
}
|
||||
trying, end := context.WithTimeout(ctx, checkTakeWait)
|
||||
defer end()
|
||||
fetched, err := oldest.pull.Fetch(1, nats.Context(trying))
|
||||
if len(fetched) == 0 {
|
||||
if err != nil && !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, nats.ErrTimeout) && !errors.Is(err, context.Canceled) {
|
||||
m.sayOnce("the "+oldest.kind+" check queue cannot be taken from", err)
|
||||
}
|
||||
m.dropLetThrough(ctx)
|
||||
return nil, nil
|
||||
}
|
||||
// Another holder took the one that was oldest in the moment between the read and the fetch: what came is a younger check,
|
||||
// handed back and the key released, never run during a delivery.
|
||||
if askedAtOf(fetched[0].Data, "").Add(m.letThroughAfter()).After(time.Now()) {
|
||||
_ = fetched[0].NakWithDelay(time.Minute) // a delay keeps it off the queue of redeliveries a fetch reads first
|
||||
m.dropLetThrough(ctx)
|
||||
return nil, nil
|
||||
}
|
||||
m.letKey = true
|
||||
return fetched[0], oldest
|
||||
}
|
||||
|
||||
func (m *natsMachine) letThroughAfter() time.Duration {
|
||||
if m.opts.LetThroughAfter > 0 {
|
||||
return m.opts.LetThroughAfter
|
||||
}
|
||||
return LetThroughAfter
|
||||
}
|
||||
|
||||
// dropLetThrough deletes the let-through key this holder created.
|
||||
func (m *natsMachine) dropLetThrough(ctx context.Context) {
|
||||
m.letKey = false
|
||||
if err := m.gateOf().DropLetThrough(context.WithoutCancel(ctx)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not take the let-through key away; it ages out: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
// keepLetThrough refreshes the key at every heartbeat while the check it was created for runs, and returns the end of that: it
|
||||
// stops the refreshing and deletes the key.
|
||||
func (m *natsMachine) keepLetThrough(ctx context.Context, every time.Duration) (end func()) {
|
||||
done := make(chan struct{})
|
||||
finished := make(chan struct{})
|
||||
go func() {
|
||||
defer close(finished)
|
||||
tick := time.NewTicker(every)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
case <-tick.C:
|
||||
putting, stop := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||
if err := m.gateOf().KeepLetThrough(putting, m.on); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not keep the let-through key: %v\n", err)
|
||||
}
|
||||
stop()
|
||||
}
|
||||
}
|
||||
}()
|
||||
return func() {
|
||||
close(done)
|
||||
<-finished
|
||||
m.dropLetThrough(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
// whyHeld words the plans a check is held for, for the log.
|
||||
func whyHeld(plans []string) string { return strings.Join(plans, ", ") }
|
||||
@@ -0,0 +1,307 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// Checks wait while a delivery builds (novox/hq ADR 0483, issues 1687 and 1688): the order the taking loop hands out in, against
|
||||
// fake queues and a fake bus.
|
||||
|
||||
// fakeQueue is a queue the loop fetches from: the asks waiting, handed out in order, and how often it was fetched from.
|
||||
type fakeQueue struct {
|
||||
mu sync.Mutex
|
||||
waiting []*nats.Msg
|
||||
fetched int
|
||||
}
|
||||
|
||||
func (f *fakeQueue) Fetch(int, ...nats.PullOpt) ([]*nats.Msg, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.fetched++
|
||||
if len(f.waiting) == 0 {
|
||||
return nil, nats.ErrTimeout
|
||||
}
|
||||
m := f.waiting[0]
|
||||
f.waiting = f.waiting[1:]
|
||||
return []*nats.Msg{m}, nil
|
||||
}
|
||||
|
||||
func (f *fakeQueue) fetches() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.fetched
|
||||
}
|
||||
|
||||
// askedAgo is a request asked this long ago, as the id says, for a check of this pull request.
|
||||
func askedAgo(ago time.Duration, number int, ready bool) *nats.Msg {
|
||||
id := fmt.Sprintf("build-%d", time.Now().Add(-ago).UnixNano())
|
||||
body, _ := json.Marshal(BuildRequest{ID: id, Repository: "r", Check: &CheckRequest{Owner: "o", Repo: "r", Number: number, Ready: ready}})
|
||||
return &nats.Msg{Data: body}
|
||||
}
|
||||
|
||||
func buildAsk() *nats.Msg {
|
||||
body, _ := json.Marshal(BuildRequest{ID: fmt.Sprintf("build-%d", time.Now().UnixNano()), Repository: "r"})
|
||||
return &nats.Msg{Data: body}
|
||||
}
|
||||
|
||||
// fakeGate is the bus as the loop sees it for the hold.
|
||||
type fakeGate struct {
|
||||
mu sync.Mutex
|
||||
plans []string
|
||||
plansErr error
|
||||
letKey bool
|
||||
taken int
|
||||
kept int
|
||||
dropped int
|
||||
oldestFrom map[*takeQueue]*fakeQueue
|
||||
}
|
||||
|
||||
func (g *fakeGate) Building(context.Context) ([]string, error) {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
return g.plans, g.plansErr
|
||||
}
|
||||
|
||||
// OldestWaiting reads the first waiting ask of the fake queue without taking it.
|
||||
func (g *fakeGate) OldestWaiting(_ context.Context, q *takeQueue) (time.Time, bool, error) {
|
||||
f := g.oldestFrom[q]
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if len(f.waiting) == 0 {
|
||||
return time.Time{}, false, nil
|
||||
}
|
||||
return askedAtOf(f.waiting[0].Data, ""), true, nil
|
||||
}
|
||||
|
||||
func (g *fakeGate) TakeLetThrough(context.Context, string) (bool, error) {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
if g.letKey {
|
||||
return false, nil
|
||||
}
|
||||
g.letKey = true
|
||||
g.taken++
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (g *fakeGate) KeepLetThrough(context.Context, string) error {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.kept++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *fakeGate) DropLetThrough(context.Context) error {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.letKey = false
|
||||
g.dropped++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *fakeGate) setPlans(plans ...string) {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.plans = plans
|
||||
}
|
||||
|
||||
// aMachine is a loop on the build seat with a delivery queue and the two check queues, all fake.
|
||||
func aMachine() (m *natsMachine, builds, ready, other *fakeQueue, gate *fakeGate) {
|
||||
builds, ready, other = &fakeQueue{}, &fakeQueue{}, &fakeQueue{}
|
||||
subs := []*takeQueue{
|
||||
{kind: queueBuilds, pull: builds, bound: true},
|
||||
{kind: queueReady, pull: ready, bound: true},
|
||||
{kind: queueOther, pull: other, bound: true},
|
||||
}
|
||||
gate = &fakeGate{oldestFrom: map[*takeQueue]*fakeQueue{subs[1]: ready, subs[2]: other}}
|
||||
m = &natsMachine{on: "ace", seat: broker.TheBuildSeat, subs: subs, gate: gate}
|
||||
return m, builds, ready, other, gate
|
||||
}
|
||||
|
||||
func nextAsk(t *testing.T, m *natsMachine) (*nats.Msg, *takeQueue) {
|
||||
t.Helper()
|
||||
msg, q, err := m.next(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return msg, q
|
||||
}
|
||||
|
||||
// 1652/AC6, 1687: a waiting check is not taken while a plan key is there, also when the delivery's queue is empty for the moment
|
||||
// (between two tiers).
|
||||
func TestNoCheckIsTakenWhileAPlanBuilds(t *testing.T) {
|
||||
m, builds, ready, other, gate := aMachine()
|
||||
gate.setPlans("plan-1")
|
||||
ready.waiting = []*nats.Msg{askedAgo(2*time.Minute, 1, true)}
|
||||
other.waiting = []*nats.Msg{askedAgo(5*time.Minute, 2, false)}
|
||||
// The delivery's queue is empty: between two tiers.
|
||||
if msg, _ := nextAsk(t, m); msg != nil {
|
||||
t.Fatalf("a check was taken while plan-1 builds: %s", msg.Data)
|
||||
}
|
||||
if ready.fetches() != 0 || other.fetches() != 0 {
|
||||
t.Fatalf("the check queues were fetched from while plan-1 builds: ready %d, other %d", ready.fetches(), other.fetches())
|
||||
}
|
||||
if builds.fetches() == 0 {
|
||||
t.Fatal("the delivery's queue was not looked at")
|
||||
}
|
||||
}
|
||||
|
||||
// A delivery build is still taken first, and the check waits behind it.
|
||||
func TestADeliveryBuildIsTakenWhileAPlanBuilds(t *testing.T) {
|
||||
m, builds, ready, _, gate := aMachine()
|
||||
gate.setPlans("plan-1")
|
||||
builds.waiting = []*nats.Msg{buildAsk()}
|
||||
ready.waiting = []*nats.Msg{askedAgo(2*time.Minute, 1, true)}
|
||||
msg, q := nextAsk(t, m)
|
||||
if msg == nil || q.kind != queueBuilds {
|
||||
t.Fatalf("the delivery build was not taken first: %v %v", msg, q)
|
||||
}
|
||||
if ready.fetches() != 0 {
|
||||
t.Fatal("a check queue was fetched from")
|
||||
}
|
||||
}
|
||||
|
||||
// It is taken once the key is gone (deleted at the phase's end, or expired).
|
||||
func TestACheckIsTakenOnceTheKeyIsGone(t *testing.T) {
|
||||
m, _, ready, _, gate := aMachine()
|
||||
gate.setPlans("plan-1")
|
||||
ready.waiting = []*nats.Msg{askedAgo(2*time.Minute, 1, true)}
|
||||
if msg, _ := nextAsk(t, m); msg != nil {
|
||||
t.Fatal("taken while the key is there")
|
||||
}
|
||||
gate.setPlans()
|
||||
msg, q := nextAsk(t, m)
|
||||
if msg == nil || q.kind != queueReady {
|
||||
t.Fatalf("the check was not taken once the key was gone: %v", q)
|
||||
}
|
||||
if m.letKey {
|
||||
t.Fatal("a check taken without a delivery building took the let-through key")
|
||||
}
|
||||
}
|
||||
|
||||
// 1687: a bucket that cannot be read is said and read as empty, so the checks are taken.
|
||||
func TestAnUnreadableBucketHoldsNoCheck(t *testing.T) {
|
||||
m, _, ready, _, gate := aMachine()
|
||||
gate.plansErr = errors.New("permissions violation")
|
||||
gate.plans = []string{"plan-1"}
|
||||
ready.waiting = []*nats.Msg{askedAgo(2*time.Minute, 1, true)}
|
||||
msg, q := nextAsk(t, m)
|
||||
if msg == nil || q.kind != queueReady {
|
||||
t.Fatal("a check was held because the bucket could not be read")
|
||||
}
|
||||
if !m.said["the bucket of the plans building cannot be read"] {
|
||||
t.Fatalf("the fault was not said: %v", m.said)
|
||||
}
|
||||
}
|
||||
|
||||
// 1688: a check asked 30 minutes ago is let through while a key is there; the oldest across both queues by ask time, ahead of a
|
||||
// waiting delivery build; a younger one is never taken before it.
|
||||
func TestTheOldestCheckPastTheBoundGoesFirstAcrossBothQueues(t *testing.T) {
|
||||
m, builds, ready, other, gate := aMachine()
|
||||
gate.setPlans("plan-1")
|
||||
builds.waiting = []*nats.Msg{buildAsk()}
|
||||
ready.waiting = []*nats.Msg{askedAgo(31*time.Minute, 1, true)}
|
||||
other.waiting = []*nats.Msg{askedAgo(50*time.Minute, 2, false)}
|
||||
msg, q := nextAsk(t, m)
|
||||
if msg == nil || q.kind != queueOther || !strings.Contains(string(msg.Data), `"number":2`) {
|
||||
t.Fatalf("the oldest check (50 minutes, other queue) was not let through first: %v", q)
|
||||
}
|
||||
if !m.letKey || gate.taken != 1 {
|
||||
t.Fatalf("the let-through key was not created for it: held %v, created %d", m.letKey, gate.taken)
|
||||
}
|
||||
}
|
||||
|
||||
// 1688: several checks past the bound go one at a time, oldest first: while the key is there every other holder takes builds.
|
||||
func TestChecksPastTheBoundGoOneAtATimeOldestFirst(t *testing.T) {
|
||||
m, builds, ready, other, gate := aMachine()
|
||||
gate.setPlans("plan-1")
|
||||
ready.waiting = []*nats.Msg{askedAgo(33*time.Minute, 1, true), askedAgo(31*time.Minute, 3, true)}
|
||||
other.waiting = []*nats.Msg{askedAgo(40*time.Minute, 2, false)}
|
||||
var order []string
|
||||
for range 3 {
|
||||
msg, q := nextAsk(t, m)
|
||||
if msg == nil {
|
||||
t.Fatalf("no check let through after %v", order)
|
||||
}
|
||||
var r BuildRequest
|
||||
_ = json.Unmarshal(msg.Data, &r)
|
||||
order = append(order, fmt.Sprint(r.Check.Number, q.kind))
|
||||
// While it runs another holder (the same fake gate, key held) takes builds only.
|
||||
builds.waiting = []*nats.Msg{buildAsk()}
|
||||
other2 := &natsMachine{on: "shanks", seat: broker.TheBuildSeat, subs: m.subs, gate: gate}
|
||||
if next, nq := nextAsk(t, other2); next == nil || nq.kind != queueBuilds {
|
||||
t.Fatalf("a second holder did not take a delivery build while the let-through key is held: %v", nq)
|
||||
}
|
||||
m.dropLetThrough(context.Background())
|
||||
}
|
||||
if want := []string{"2other", "1ready", "3ready"}; strings.Join(order, " ") != strings.Join(want, " ") {
|
||||
t.Fatalf("checks went %v, want %v (oldest first, one at a time)", order, want)
|
||||
}
|
||||
}
|
||||
|
||||
// 1688: a check younger than the bound is never let through, and no key is created for it.
|
||||
func TestACheckYoungerThanTheBoundIsHeld(t *testing.T) {
|
||||
m, _, ready, _, gate := aMachine()
|
||||
gate.setPlans("plan-1")
|
||||
ready.waiting = []*nats.Msg{askedAgo(29*time.Minute, 1, true)}
|
||||
if msg, _ := nextAsk(t, m); msg != nil {
|
||||
t.Fatal("a check of 29 minutes was let through")
|
||||
}
|
||||
if gate.taken != 0 {
|
||||
t.Fatal("a let-through key was created for a young check")
|
||||
}
|
||||
if ready.fetches() != 0 {
|
||||
t.Fatal("the age was read by a fetch; it must be read without taking")
|
||||
}
|
||||
}
|
||||
|
||||
// 1688: the holder that lost the race for the key, or the check, takes builds only; and a younger check that arrives in the
|
||||
// moment between the read and the fetch is handed back with the key released.
|
||||
func TestAYoungerCheckFetchedInTheRaceIsHandedBackAndTheKeyReleased(t *testing.T) {
|
||||
m, _, ready, _, gate := aMachine()
|
||||
gate.setPlans("plan-1")
|
||||
// The read sees the old check; before the fetch another holder takes it and a young one is first.
|
||||
gate.oldestFrom[m.subs[1]] = &fakeQueue{waiting: []*nats.Msg{askedAgo(40*time.Minute, 1, true)}}
|
||||
ready.waiting = []*nats.Msg{askedAgo(time.Minute, 2, true)}
|
||||
if msg, _ := nextAsk(t, m); msg != nil {
|
||||
t.Fatalf("a young check was run during a delivery: %s", msg.Data)
|
||||
}
|
||||
if gate.letKey || m.letKey {
|
||||
t.Fatal("the let-through key was left held")
|
||||
}
|
||||
}
|
||||
|
||||
// The key of a check let through is refreshed while it runs and deleted when it ends.
|
||||
func TestTheLetThroughKeyLivesAsLongAsTheCheckRuns(t *testing.T) {
|
||||
m, _, ready, _, gate := aMachine()
|
||||
gate.setPlans("plan-1")
|
||||
ready.waiting = []*nats.Msg{askedAgo(40*time.Minute, 1, true)}
|
||||
if msg, _ := nextAsk(t, m); msg == nil {
|
||||
t.Fatal("not let through")
|
||||
}
|
||||
end := m.keepLetThrough(context.Background(), 20*time.Millisecond)
|
||||
time.Sleep(120 * time.Millisecond)
|
||||
gate.mu.Lock()
|
||||
held, kept := gate.letKey, gate.kept
|
||||
gate.mu.Unlock()
|
||||
if !held || kept == 0 {
|
||||
t.Errorf("while the check ran the key was held %v and refreshed %d times", held, kept)
|
||||
}
|
||||
end()
|
||||
gate.mu.Lock()
|
||||
defer gate.mu.Unlock()
|
||||
if gate.letKey || m.letKey {
|
||||
t.Error("the key was not deleted when the check ended")
|
||||
}
|
||||
}
|
||||
@@ -88,6 +88,40 @@ type Queue struct {
|
||||
AckPending int `json:"ack-pending"`
|
||||
MaxDeliver int `json:"max-deliver"`
|
||||
Asks []QueuedAsk `json:"asks"`
|
||||
// Hold is why the checks wait: the delivery plans building and the check let through now (ADR 0483, issues 1687 and 1688);
|
||||
// HoldFault says the bucket could not be read, in which case Hold is empty.
|
||||
Hold CheckHold `json:"hold"`
|
||||
HoldFault string `json:"hold-fault,omitempty"`
|
||||
}
|
||||
|
||||
// LetThroughOf says which ask is let through under ADR 0483 clause 3: the check that waited LetThroughAfter or longer and is next
|
||||
// (the oldest waiting check across the two check queues, while a delivery builds), and the check in flight that was let through
|
||||
// (the let-through key names the machine that runs it). Empty when none.
|
||||
func (q Queue) LetThroughOf(now time.Time) (next, running string) {
|
||||
if len(q.Hold.Plans) == 0 && q.Hold.LetThrough == nil {
|
||||
return "", ""
|
||||
}
|
||||
var oldest *QueuedAsk
|
||||
for i := range q.Asks {
|
||||
a := &q.Asks[i]
|
||||
if a.State != AskWaiting || a.Request.Check == nil || a.AskedAt.IsZero() {
|
||||
continue
|
||||
}
|
||||
if oldest == nil || a.AskedAt.Before(oldest.AskedAt) {
|
||||
oldest = a
|
||||
}
|
||||
}
|
||||
if oldest != nil && len(q.Hold.Plans) > 0 && now.Sub(oldest.AskedAt) >= LetThroughAfter {
|
||||
next = oldest.ID
|
||||
}
|
||||
if q.Hold.LetThrough != nil {
|
||||
for _, a := range q.Asks {
|
||||
if a.State == AskInFlight && a.Request.Check != nil && a.On == q.Hold.LetThrough.Builder {
|
||||
running = a.ID
|
||||
}
|
||||
}
|
||||
}
|
||||
return next, running
|
||||
}
|
||||
|
||||
// Of is the asks in one state, in queue order.
|
||||
@@ -319,6 +353,16 @@ func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, e
|
||||
for _, r := range reads {
|
||||
q.Asks = append(q.Asks, ClassifyQueue(r.asks, r.delivered, r.ackPending, heard)...)
|
||||
}
|
||||
// And why the checks wait, for the build seat that has them (ADR 0483 clause 5). A bucket that cannot be read is said by the
|
||||
// queue, which then names no plan.
|
||||
if seat == broker.TheBuildSeat {
|
||||
hold, err := ReadCheckHold(ctx, js.Conn())
|
||||
if err != nil {
|
||||
q.HoldFault = err.Error()
|
||||
} else {
|
||||
q.Hold = hold
|
||||
}
|
||||
}
|
||||
return q, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// Issue 1747: a builder never holds a message it does not start. A message the server sends after a fetch gave up lands in that
|
||||
// queue's subscription; while an ask runs, the queues it does not come from are let go of, so such a message is not held
|
||||
// unheartbeated for the whole run, and the queues are bound again before the next fetch.
|
||||
|
||||
func TestARunningAskLetsGoOfTheQueuesItDidNotComeFrom(t *testing.T) {
|
||||
m, builds, _, other, _ := aMachine()
|
||||
// A delivery build reached the builds queue's subscription after its fetch gave up: it sits in the buffer.
|
||||
builds.waiting = []*nats.Msg{buildAsk()}
|
||||
took := m.subs[1] // the running ask came from the ready queue
|
||||
|
||||
m.quiesce(took)
|
||||
|
||||
if !took.bound || took.pull == nil {
|
||||
t.Error("the queue the running ask came from was let go of; its acknowledgements need it")
|
||||
}
|
||||
for _, q := range []*takeQueue{m.subs[0], m.subs[2]} {
|
||||
if q.bound || q.pull != nil || q.sub != nil {
|
||||
t.Errorf("the %s queue is still held while an ask runs", q.kind)
|
||||
}
|
||||
}
|
||||
// Nothing is fetched from a queue that was let go of: a message in it cannot be taken unheartbeated.
|
||||
if msg, _, _ := m.next(t.Context()); msg != nil {
|
||||
t.Fatalf("a message was taken from a queue that was let go of: %s", msg.Data)
|
||||
}
|
||||
if builds.fetches() != 0 || other.fetches() != 0 {
|
||||
t.Errorf("fetched from a queue that was let go of: builds %d, other %d", builds.fetches(), other.fetches())
|
||||
}
|
||||
}
|
||||
|
||||
// Once the ask is settled the queue it came from is let go of too: a second message the server delivered into its subscription
|
||||
// on a pull request still open is not run after the ask ends (review of 1747).
|
||||
func TestTheQueueTheAskCameFromIsLetGoOfWhenTheAskEnds(t *testing.T) {
|
||||
m, _, ready, _, _ := aMachine()
|
||||
ready.waiting = []*nats.Msg{askedAgo(time.Minute, 2, true)}
|
||||
m.quiesce(nil)
|
||||
for _, q := range m.subs {
|
||||
if q.bound || q.pull != nil {
|
||||
t.Errorf("the %s queue is still held after the ask ended", q.kind)
|
||||
}
|
||||
}
|
||||
m.subs[0].bound, m.subs[0].pull = true, &fakeQueue{}
|
||||
if msg, _, _ := m.next(t.Context()); msg != nil || ready.fetches() != 0 {
|
||||
t.Fatal("a message held in the kept queue's subscription was taken after the ask ended")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package ownmanifest
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// imagesOutOfPlace lists the image artifacts among manifests (module name to its manifest's text) whose Dockerfile
|
||||
// is not a file inside that module's own directory, root/<module> (novox/hq ADR 0487 clause 13, which replaced the
|
||||
// refusal of the image kind of ADR 0480 clause 11). The image builder builds a module from its directory alone.
|
||||
// It reads the manifest as plain JSON, since the catalogue parser refuses some of these itself. It mirrors mesh-catalog's checks/imageplace: an empty from, an absolute one, one leaving the directory with ..,
|
||||
// or a name that is no file there, is refused.
|
||||
func imagesOutOfPlace(t *testing.T, root string, manifests map[string][]byte) []string {
|
||||
t.Helper()
|
||||
var out []string
|
||||
for module, raw := range manifests {
|
||||
var m struct {
|
||||
Build struct {
|
||||
Artifacts []struct{ Name, Kind, From string } `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatalf("%s does not parse: %v", module, err)
|
||||
}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Kind != "image" {
|
||||
continue
|
||||
}
|
||||
clean := filepath.Clean(a.From)
|
||||
bad := a.From == "" || filepath.IsAbs(clean) || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator))
|
||||
if !bad {
|
||||
info, err := os.Stat(filepath.Join(root, module, clean))
|
||||
bad = err != nil || info.IsDir()
|
||||
}
|
||||
if bad {
|
||||
out = append(out, module+" ("+a.Name+" from "+a.From+")")
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Every image a module of this repository declares is built from a Dockerfile in the module's own directory
|
||||
// (novox/hq ADR 0487 clause 13). The catalogue's own modules are held to the same by checks/imageplace in mesh-catalog.
|
||||
func TestEveryImageArtifactBuildsFromADockerfileInItsOwnModuleDirectory(t *testing.T) {
|
||||
root := filepath.Join("..", "..", "modules")
|
||||
paths, err := filepath.Glob(filepath.Join(root, "*", "module.json"))
|
||||
if err != nil || len(paths) == 0 {
|
||||
t.Fatalf("no module manifests found in this repository's modules: %v", err)
|
||||
}
|
||||
manifests := map[string][]byte{}
|
||||
for _, p := range paths {
|
||||
raw, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
manifests[filepath.Base(filepath.Dir(p))] = raw
|
||||
}
|
||||
if got := imagesOutOfPlace(t, root, manifests); len(got) != 0 {
|
||||
t.Errorf("these image artifacts do not build from a Dockerfile in their own module's directory "+
|
||||
"(ADR 0487 clause 13): %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The check fails on each bad case and passes on a good one, so it holds something.
|
||||
func TestAnImageWhoseDockerfileIsOutOfPlaceIsFoundAndOneInPlaceIsNot(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(root, "x", "sub"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range []string{"x/Dockerfile", "x/sub/Dockerfile", "outside"} {
|
||||
if err := os.WriteFile(filepath.Join(root, f), nil, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
manifest := func(from string) map[string][]byte {
|
||||
return map[string][]byte{"x": []byte(`{"module":"x","version":"1","build":{"artifacts":[{"name":"server","kind":"image","from":"` + from + `"}]}}`)}
|
||||
}
|
||||
for _, from := range []string{"Dockerfile", "sub/Dockerfile", "./Dockerfile"} {
|
||||
if got := imagesOutOfPlace(t, root, manifest(from)); len(got) != 0 {
|
||||
t.Errorf("from %q: found %v, want none", from, got)
|
||||
}
|
||||
}
|
||||
for _, from := range []string{"", "/etc/Dockerfile", "../outside", "..", "sub/../../outside", "Missing", "sub"} {
|
||||
if got := imagesOutOfPlace(t, root, manifest(from)); len(got) != 1 {
|
||||
t.Errorf("from %q: found %v, want it refused", from, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
//
|
||||
// route-proxy keeps each ACME authority's account and certificates in a directory named after a
|
||||
// digest of the directory URL and of the root bundle its `trust` container copies in
|
||||
// (examples/route-proxy, forThisAuthority). Until ADR 0226 the URL was rendered from a binding to
|
||||
// (modules/route-proxy/cmd/route-proxy, forThisAuthority). Until ADR 0226 the URL was rendered from a binding to
|
||||
// `public-acme`'s `acme-ca`, spelled with the port. A URL spelled any other way — even the same
|
||||
// authority without `:443` — or a root bundle from another image is a new authority to the proxy:
|
||||
// a new account, and every routed name ordered again, on two machines at once, against Let's
|
||||
@@ -36,8 +36,11 @@ func TestRouteProxyKeepsItsPublicAccountDirectory(t *testing.T) {
|
||||
}
|
||||
|
||||
// As a build would leave it: each artifact a container names resolved to an image. Which image
|
||||
// does not matter to the file checked here.
|
||||
// does not matter to the file checked here. An archive resource carries no image: it unpacks a bundle.
|
||||
for _, res := range m.Resources {
|
||||
if res["type"] == "archive" {
|
||||
continue
|
||||
}
|
||||
if artifact, ok := res["artifact"].(string); ok {
|
||||
delete(res, "artifact")
|
||||
res["image"] = "registry.invalid/route-proxy/" + artifact +
|
||||
@@ -86,3 +89,57 @@ func TestRouteProxyKeepsItsPublicAccountDirectory(t *testing.T) {
|
||||
"plan for the account (ADR 0226)", trust, trustImage)
|
||||
}
|
||||
}
|
||||
|
||||
// The proxy's program is mounted into the pinned trust image, not baked into an image (novox/hq ADR 0480 clauses 1,
|
||||
// 2 and 4): a Go bundle of the module's own Go module stamped for alpine, unpacked by an archive resource, mounted
|
||||
// read-only at /opt/mesh, called through args, and named under restart-on so a new program never leaves the old
|
||||
// one running. The container's name, network and logging are what the jail and the routes rely on.
|
||||
func TestRouteProxyRunsItsMountedProgramOnTheTrustImage(t *testing.T) {
|
||||
m := catalogueManifest(t, "route-proxy")
|
||||
var bundle *Artifact
|
||||
for i, a := range m.Build.Artifacts {
|
||||
if a.Name == "server" {
|
||||
bundle = &m.Build.Artifacts[i]
|
||||
}
|
||||
if a.Kind == "image" {
|
||||
t.Errorf("artifact %s is an image built from a Dockerfile (ADR 0452 §1)", a.Name)
|
||||
}
|
||||
}
|
||||
if bundle == nil || bundle.Kind != "bundle" || bundle.Language != "go" || bundle.System != "alpine" ||
|
||||
bundle.From != "cmd/route-proxy" || bundle.Binary != "route-proxy" {
|
||||
t.Fatalf("the server artifact is not a Go bundle of cmd/route-proxy stamped for alpine: %+v", bundle)
|
||||
}
|
||||
var archive, server map[string]any
|
||||
for _, r := range m.Resources {
|
||||
switch {
|
||||
case r["type"] == "archive" && r["artifact"] == "server":
|
||||
archive = r
|
||||
case r["id"] == "server":
|
||||
server = r
|
||||
}
|
||||
}
|
||||
if archive == nil || server == nil {
|
||||
t.Fatalf("archive %v or server container %v is missing", archive, server)
|
||||
}
|
||||
if server["artifact"] != "trust" || server["name"] != "route-proxy" || server["network"] != "host" || server["logging"] != "journald" {
|
||||
t.Errorf("the server container moved: %v", server)
|
||||
}
|
||||
mount := archive["path"].(string) + ":/opt/mesh:ro"
|
||||
mounted := false
|
||||
for _, v := range server["volumes"].([]any) {
|
||||
mounted = mounted || v == mount
|
||||
}
|
||||
if !mounted {
|
||||
t.Errorf("the archive is not mounted read-only at /opt/mesh (%s): %v", mount, server["volumes"])
|
||||
}
|
||||
if args, _ := server["args"].([]any); len(args) != 1 || args[0] != "/opt/mesh/route-proxy" {
|
||||
t.Errorf("args are %v, want the mounted program", server["args"])
|
||||
}
|
||||
restarts := false
|
||||
for _, v := range server["restart-on"].([]any) {
|
||||
restarts = restarts || v == archive["id"]
|
||||
}
|
||||
if !restarts {
|
||||
t.Errorf("restart-on %v does not name the archive %v", server["restart-on"], archive["id"])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,10 +57,13 @@ var (
|
||||
// mesh-config publishes as its own (design 57 K3).
|
||||
Predefined = Mirror{Name: broker.MirrorPredefined, Bucket: broker.PredefinedBucket,
|
||||
Event: link.KeyPredefinedChanged}
|
||||
// DeclaredPackages is each node's packages its modules declare, keyed by the node (ADR 0507 §5). It raises no event:
|
||||
// the seat's emits, which the SDK states, do not name one, and a reader reads the bucket when it is asked.
|
||||
DeclaredPackages = Mirror{Name: broker.MirrorDeclaredPackages, Bucket: broker.DeclaredPackagesBucket}
|
||||
)
|
||||
|
||||
// Mirrors is every mirror this package writes, in the order they are written.
|
||||
func Mirrors() []Mirror { return []Mirror{DeclaredConfig, Predefined} }
|
||||
func Mirrors() []Mirror { return []Mirror{DeclaredConfig, Predefined, DeclaredPackages} }
|
||||
|
||||
// The changes an event says.
|
||||
const (
|
||||
@@ -251,7 +254,7 @@ func (w *Writer) put(ctx context.Context, key string, value []byte, before kept,
|
||||
|
||||
// say raises one write's event, tried once more when refused and then counted as lost.
|
||||
func (w *Writer) say(ctx context.Context, key, change string, fields []string, revision uint64) {
|
||||
if w.Announce == nil {
|
||||
if w.Announce == nil || w.Mirror.Event == "" {
|
||||
return
|
||||
}
|
||||
e := Event{Mirror: w.Mirror.Name, Key: key, Change: change, Fields: fields, Revision: revision, At: w.now().UTC()}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
@@ -246,3 +247,49 @@ func TestNatsARemovalWithoutTheLeaseRemovesNothing(t *testing.T) {
|
||||
t.Errorf("the key was removed without the lease: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1840/AC5 (ADR 0507 §5, ADR 0289 §1): the declared-packages mirror, keyed by the node, writes a node's value once; a second
|
||||
// pass over the same declaration writes nothing (the bucket takes no message); a package newly declared changes the one key;
|
||||
// a node gone removes its key. It raises no event: the seat's emits name none for it.
|
||||
func TestNatsTheDeclaredPackagesMirrorWritesOnlyOnAChange(t *testing.T) {
|
||||
js, api := aMirrorBus(t)
|
||||
events := listen(t, js.Conn(), "mesh.seat."+broker.ControllerSeat+".event.>")
|
||||
w := aWriter(t, js, api, DeclaredPackages)
|
||||
ctx := t.Context()
|
||||
value := func(node string, resources ...map[string]any) []byte {
|
||||
t.Helper()
|
||||
owner := map[string]string{}
|
||||
for _, r := range resources {
|
||||
owner[r["id"].(string)] = "web"
|
||||
}
|
||||
v, err := inventory.DeclaredPackagesValue(node, resources, owner)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
curl := map[string]any{"id": "web.curl", "type": "package", "package": "curl"}
|
||||
nano := map[string]any{"id": "web.nano", "type": "package", "package": "nano", "absent": true}
|
||||
|
||||
if r, err := w.Sync(ctx, map[string][]byte{"one": value("one", curl), "two": value("two")}); err != nil ||
|
||||
r != (Result{Written: 2}) {
|
||||
t.Fatalf("the first pass wrote %+v (%v)", r, err)
|
||||
}
|
||||
before := writes(t, api, DeclaredPackages.Bucket)
|
||||
if r, err := w.Sync(ctx, map[string][]byte{"one": value("one", curl), "two": value("two")}); err != nil || r != (Result{}) {
|
||||
t.Errorf("a pass over the same declarations wrote %+v (%v)", r, err)
|
||||
}
|
||||
if got := writes(t, api, DeclaredPackages.Bucket) - before; got != 0 {
|
||||
t.Errorf("a pass over the same declarations put %d messages in the bucket", got)
|
||||
}
|
||||
if r, err := w.Sync(ctx, map[string][]byte{"one": value("one", curl, nano), "two": value("two")}); err != nil ||
|
||||
r != (Result{Written: 1}) {
|
||||
t.Errorf("a package newly declared wrote %+v (%v)", r, err)
|
||||
}
|
||||
if r, err := w.Sync(ctx, map[string][]byte{"one": value("one", curl, nano)}); err != nil || r != (Result{Removed: 1}) {
|
||||
t.Errorf("a node gone wrote %+v (%v)", r, err)
|
||||
}
|
||||
if said := settle(events, 1); len(said) != 0 {
|
||||
t.Errorf("the declared-packages mirror raised %+v", said)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,18 +19,10 @@ import (
|
||||
// bus upgrade moves the catalogue's pin; this then fails in the merge check until the tests' pin moves
|
||||
// with it, so the controller is never tested against a bus the mesh no longer runs, or not yet.
|
||||
func TestTheBusTheTestsRunIsTheBusTheMeshRuns(t *testing.T) {
|
||||
dockerfile := filepath.Join(beside.Catalogue(t), "nats", "Dockerfile")
|
||||
raw, err := os.ReadFile(dockerfile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pinned := regexp.MustCompile(`upstream: nats ([0-9.]+)-alpine`).FindSubmatch(raw)
|
||||
if pinned == nil {
|
||||
t.Fatalf("%s says no release its digest is", dockerfile)
|
||||
}
|
||||
if string(pinned[1]) != Version {
|
||||
t.Errorf("the tests run bus %s and the catalogue's bus image is %s: move go.mod's nats-server pin "+
|
||||
"(and `go mod vendor`) with the image", Version, pinned[1])
|
||||
release, from := catalogueBusRelease(t, filepath.Join(beside.Catalogue(t), "nats"))
|
||||
if release != Version {
|
||||
t.Errorf("the tests run bus %s and the catalogue's bus image is %s (%s): move go.mod's nats-server pin "+
|
||||
"(and `go mod vendor`) with the image", Version, release, from)
|
||||
}
|
||||
path := os.Getenv("MESH_FACTS")
|
||||
if path == "" {
|
||||
@@ -57,3 +49,51 @@ func TestTheBusTheTestsRunIsTheBusTheMeshRuns(t *testing.T) {
|
||||
}
|
||||
|
||||
// A test's bus is its own: what one test makes, the next never sees.
|
||||
|
||||
// catalogueBusRelease is the server release the catalogue's nats module runs, and where it was read. Since
|
||||
// ADR 0497 the server is an upstream artifact in the module's manifest (`nats:<release>-alpine@sha256:…`);
|
||||
// before it, an image built from the module's Dockerfile, whose `upstream: nats <release>-alpine` line said
|
||||
// the release its digest is. The manifest is read first; the Dockerfile only where the manifest names no
|
||||
// upstream nats image (a catalogue from before that move, such as the captured copy). Neither is a failure.
|
||||
func catalogueBusRelease(t *testing.T, module string) (release, from string) {
|
||||
t.Helper()
|
||||
manifest := filepath.Join(module, "module.json")
|
||||
raw, err := os.ReadFile(manifest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Build struct {
|
||||
Artifacts []struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
From string `json:"from"`
|
||||
} `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatalf("%s: %v", manifest, err)
|
||||
}
|
||||
upstream := regexp.MustCompile(`^(?:[^/]+/)*nats:([0-9]+\.[0-9]+\.[0-9]+)-alpine(?:@sha256:[0-9a-f]{64})?$`)
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Kind != "upstream" || !strings.HasPrefix(a.From[strings.LastIndex(a.From, "/")+1:], "nats:") {
|
||||
continue
|
||||
}
|
||||
pinned := upstream.FindStringSubmatch(a.From)
|
||||
if pinned == nil {
|
||||
t.Fatalf("%s: the upstream artifact %q is %q, which says no release in the form nats:<x.y.z>-alpine",
|
||||
manifest, a.Name, a.From)
|
||||
}
|
||||
return pinned[1], manifest
|
||||
}
|
||||
dockerfile := filepath.Join(module, "Dockerfile")
|
||||
raw, err = os.ReadFile(dockerfile)
|
||||
if err != nil {
|
||||
t.Fatalf("%s names no upstream nats image, and the image built before it is not there either: %v", manifest, err)
|
||||
}
|
||||
pinned := regexp.MustCompile(`upstream: nats ([0-9.]+)-alpine`).FindSubmatch(raw)
|
||||
if pinned == nil {
|
||||
t.Fatalf("%s says no release its digest is", dockerfile)
|
||||
}
|
||||
return string(pinned[1]), dockerfile
|
||||
}
|
||||
|
||||
@@ -6,3 +6,4 @@ MESH_NPM_TOKEN_FILE=/run/mesh/package-registry.secret
|
||||
MESH_GENESIS_PACKAGE_REGISTRY=/run/mesh/genesis-package-registry.json
|
||||
MESH_WORKSPACE=${dir:workspace}
|
||||
MESH_JUDGE_CACHE=${dir:judge-cache}
|
||||
MESH_BUILD_CGROUP_PARENT=meshbuild.slice
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
# Managed by the mesh (module build-agent). Replaced on every push; edit the module's settings instead.
|
||||
#
|
||||
# Everything a build or check starts on this machine runs in this slice (novox/hq issue 1798): the build
|
||||
# agent starts every container with --cgroup-parent=meshbuild.slice. The weights only bind while the machine
|
||||
# is busy, so an idle machine still builds at full speed; the memory limits are the part that always binds.
|
||||
[Slice]
|
||||
CPUWeight=${setting:cpu-weight}
|
||||
IOWeight=${setting:io-weight}
|
||||
MemoryHigh=${setting:memory-high}
|
||||
MemoryMax=${setting:memory-max}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user