Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 jschoubben
|
858b06a94e |
@@ -90,6 +90,15 @@ redis-provisioner-image:
|
||||
@echo
|
||||
@docker image inspect $(REDIS_PROVISIONER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||
|
||||
# The proxy that turns a route grant into traffic reaching a workload.
|
||||
PROXY_IMAGE ?= mesh-route-proxy:$(VERSION)
|
||||
PROXY_DEV_TAG ?= mesh-route-proxy:development
|
||||
|
||||
proxy-image:
|
||||
docker build --build-arg GO_BASE=$(GO_BASE) -f modules/route-proxy/Dockerfile -t $(PROXY_IMAGE) -t $(PROXY_DEV_TAG) .
|
||||
@echo
|
||||
@docker image inspect $(PROXY_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||
|
||||
# The whole check. Raises a database, runs the repository's own check against it -- merge-check.sh,
|
||||
# the very script `mesh/repo-check` runs -- and takes the database down again, including when the
|
||||
# check fails, which is why the teardown is not conditional and the script's exit status is the
|
||||
@@ -108,7 +117,7 @@ redis-provisioner-image:
|
||||
# as the script runs them: a bus per test, of the release the mesh runs, makes the suite the same in any
|
||||
# order, and the timeout bounds a hang to a failure with a stack, never a stalled gate.
|
||||
check: postgres
|
||||
@sh merge-check.sh ; status=$$? ; docker rm -f -v $(PG_CONTAINER) >/dev/null 2>&1 || true ; exit $$status
|
||||
@sh merge-check.sh ; status=$$? ; docker rm -f $(PG_CONTAINER) >/dev/null 2>&1 || true ; exit $$status
|
||||
|
||||
# The unit level alone (untagged: no bus, no store, no other process), which a developer runs continuously. The tests
|
||||
# that need a bus or the store carry `//go:build integration` and run in `check`, not here; without a database the
|
||||
@@ -126,7 +135,7 @@ fmt:
|
||||
if [ -n "$$unformatted" ] ; then echo "not gofmt'd:" ; echo "$$unformatted" ; exit 1 ; fi
|
||||
|
||||
postgres:
|
||||
@docker rm -f -v $(PG_CONTAINER) >/dev/null 2>&1 || true
|
||||
@docker rm -f $(PG_CONTAINER) >/dev/null 2>&1 || true
|
||||
@docker run -d --name $(PG_CONTAINER) -e POSTGRES_PASSWORD=check \
|
||||
-p 127.0.0.1:$(PG_PORT):5432 $(PG_IMAGE) >/dev/null
|
||||
@printf 'waiting for postgres'
|
||||
@@ -138,7 +147,7 @@ postgres:
|
||||
echo ' — never came up' ; docker logs $(PG_CONTAINER) | tail -20 ; exit 1
|
||||
|
||||
postgres-stop:
|
||||
@docker rm -f -v $(PG_CONTAINER) >/dev/null 2>&1 || true
|
||||
@docker rm -f $(PG_CONTAINER) >/dev/null 2>&1 || true
|
||||
|
||||
clean:
|
||||
rm -rf build/
|
||||
|
||||
@@ -6,11 +6,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
)
|
||||
|
||||
// **The build agent says whether it is busy, and being asked drains it** (novox/hq ADR 0335 §1, §5). Its
|
||||
@@ -77,77 +73,14 @@ func drainedTooLong(now time.Time) error {
|
||||
"within %s (novox/hq ADR 0335)", began.UTC().Format(time.RFC3339), drainTooLong)
|
||||
}
|
||||
|
||||
// inHand is true while this process has claimed a build and not yet released it. The mark in the temporary directory
|
||||
// is only believed while it is: a mark with no build of this process behind it is stale (a crash, a fault in an
|
||||
// earlier path) and is removed, because left alone it hands back every ask for ever (issue 1768).
|
||||
var inHand atomic.Bool
|
||||
|
||||
// markRetry is how long an agent that cannot mark a build in hand takes no work, and markFault when it last failed to.
|
||||
// Taking nothing is the point: an ask fetched and handed back at once, many times a second, spends all its deliveries.
|
||||
const markRetry = 30 * time.Second
|
||||
|
||||
var (
|
||||
markMu sync.Mutex
|
||||
markFault time.Time
|
||||
markSaid string
|
||||
)
|
||||
|
||||
// markFaulty says the agent could not mark a build in hand and has not yet been able to since: it takes no work. Once
|
||||
// markRetry has passed it tries the mark itself, with no ask in hand, so that an ask is fetched only when marking works.
|
||||
func markFaulty(now time.Time) bool {
|
||||
markMu.Lock()
|
||||
failed := markFault
|
||||
markMu.Unlock()
|
||||
if failed.IsZero() {
|
||||
return false
|
||||
}
|
||||
if now.Sub(failed) < markRetry {
|
||||
return true
|
||||
}
|
||||
f, err := os.OpenFile(inHandMark(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
|
||||
if err != nil && errors.Is(err, os.ErrExist) && !inHand.Load() {
|
||||
_ = os.Remove(inHandMark())
|
||||
f, err = os.OpenFile(inHandMark(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
|
||||
}
|
||||
if err != nil {
|
||||
markFailed(err)
|
||||
return true
|
||||
}
|
||||
_ = f.Close()
|
||||
_ = os.Remove(inHandMark())
|
||||
markMu.Lock()
|
||||
markFault, markSaid = time.Time{}, ""
|
||||
markMu.Unlock()
|
||||
fmt.Fprintf(os.Stderr, "the mark of a build in hand can be made again (%s): taking builds\n", inHandMark())
|
||||
return false
|
||||
}
|
||||
|
||||
// markFailed notes that a build could not be marked in hand, and says it when it is a new reason.
|
||||
func markFailed(err error) {
|
||||
markMu.Lock()
|
||||
defer markMu.Unlock()
|
||||
markFault = time.Now()
|
||||
if said := err.Error(); said != markSaid {
|
||||
markSaid = said
|
||||
fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so this agent takes no build for %s and tries again: %v\n",
|
||||
inHandMark(), markRetry, err)
|
||||
}
|
||||
}
|
||||
|
||||
// claim marks a request in hand, unless the agent is drained: then nothing is in hand, and the request goes
|
||||
// back to the queue.
|
||||
func claim() bool {
|
||||
if !inHand.Load() {
|
||||
// Nothing of this process is in hand, so a mark already there is stale.
|
||||
if err := os.Remove(inHandMark()); err == nil {
|
||||
fmt.Fprintf(os.Stderr, "removed the mark of a build in hand (%s): no build of this agent is in hand\n", inHandMark())
|
||||
}
|
||||
}
|
||||
// Made, never written over: one build at a time, and a mark already there is a build in hand.
|
||||
f, err := os.OpenFile(inHandMark(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
|
||||
if err != nil {
|
||||
// Unmarked, an ask would say idle while this build runs: the build is not taken.
|
||||
markFailed(err)
|
||||
fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so it is handed back: %v\n", inHandMark(), err)
|
||||
return false
|
||||
}
|
||||
_, err = f.WriteString(time.Now().UTC().Format(time.RFC3339) + "\n")
|
||||
@@ -156,7 +89,7 @@ func claim() bool {
|
||||
}
|
||||
if err != nil {
|
||||
_ = os.Remove(inHandMark())
|
||||
markFailed(err)
|
||||
fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so it is handed back: %v\n", inHandMark(), err)
|
||||
return false
|
||||
}
|
||||
if drained() {
|
||||
@@ -164,7 +97,6 @@ func claim() bool {
|
||||
fmt.Fprintln(os.Stderr, "asked whether busy, so this agent takes no new build: handed back")
|
||||
return false
|
||||
}
|
||||
inHand.Store(true)
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -175,7 +107,6 @@ func holding(what string) {
|
||||
|
||||
// released ends the build in hand.
|
||||
func released() {
|
||||
inHand.Store(false)
|
||||
if err := os.Remove(inHandMark()); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
fmt.Fprintf(os.Stderr, "cannot remove the mark of the build in hand (%s): %v\n", inHandMark(), err)
|
||||
}
|
||||
@@ -213,23 +144,3 @@ func busy() (string, error) {
|
||||
}
|
||||
return fmt.Sprintf("busy: %s, in hand since %s; taking no new build", what, since), nil
|
||||
}
|
||||
|
||||
var (
|
||||
roomMu sync.Mutex
|
||||
roomSaid bool
|
||||
)
|
||||
|
||||
// noRoom says this node is under the floor of free space (novox/hq issue 1834): it takes no work, so another build node
|
||||
// takes the ask instead of this one taking it and ending it in an error. Said once when it starts and again when it ends.
|
||||
func noRoom(workspace string) bool {
|
||||
err := builder.RoomToWork("/", workspace)
|
||||
roomMu.Lock()
|
||||
defer roomMu.Unlock()
|
||||
if err != nil && !roomSaid {
|
||||
fmt.Fprintf(os.Stderr, "taking no builds: %v\n", err)
|
||||
} else if err == nil && roomSaid {
|
||||
fmt.Fprintf(os.Stderr, "there is room again: taking builds\n")
|
||||
}
|
||||
roomSaid = err != nil
|
||||
return err != nil
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -161,79 +160,3 @@ func TestAnUnknownCommandFails(t *testing.T) {
|
||||
t.Fatalf("help failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1768/AC2: a mark naming no build of this agent (left by a crash or a fault) is stale: the next ask removes it and is
|
||||
// run, never fetched and handed back in a loop.
|
||||
func TestAStaleInHandMarkIsRemovedAndTheAskRuns(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
inHand.Store(false)
|
||||
if err := undrained(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(inHandMark(), []byte("2026-10-21T18:45:06Z\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !claim() {
|
||||
t.Fatal("a stale in-hand mark made the agent hand back an ask")
|
||||
}
|
||||
// A live one — this agent's own build in hand — is never taken over.
|
||||
if claim() {
|
||||
t.Fatal("a second build was taken while one is in hand")
|
||||
}
|
||||
if _, err := busy(); err != nil {
|
||||
t.Fatalf("the build in hand was lost: %v", err)
|
||||
}
|
||||
released()
|
||||
}
|
||||
|
||||
// 1768/AC3: an agent that cannot mark a build in hand for another reason takes no work, instead of fetching asks and
|
||||
// handing them back; it takes work again once marking works.
|
||||
func TestAnAgentThatCannotMarkABuildTakesNoWorkUntilItCan(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Setenv("TMPDIR", dir)
|
||||
inHand.Store(false)
|
||||
markFault, markSaid = time.Time{}, ""
|
||||
t.Cleanup(func() { markFault, markSaid = time.Time{}, "" })
|
||||
if err := undrained(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The mark's place is a directory that cannot be removed or made over: a directory named as the mark.
|
||||
if err := os.MkdirAll(inHandMark()+"/x", 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if claim() {
|
||||
t.Fatal("an agent that cannot mark a build took it")
|
||||
}
|
||||
if !markFaulty(time.Now()) {
|
||||
t.Fatal("an agent that cannot mark a build still takes work")
|
||||
}
|
||||
// Still failing after the retry time: still takes nothing, and no ask was needed to find out.
|
||||
if !markFaulty(time.Now().Add(2 * markRetry)) {
|
||||
t.Fatal("an agent whose mark still cannot be made takes work")
|
||||
}
|
||||
if err := os.RemoveAll(inHandMark()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if markFaulty(time.Now().Add(4 * markRetry)) {
|
||||
t.Fatal("an agent whose mark can be made again takes no work")
|
||||
}
|
||||
if !claim() {
|
||||
t.Fatal("an agent that can mark takes no build")
|
||||
}
|
||||
released()
|
||||
}
|
||||
|
||||
// 1834/AC4: given a node under the floor of free space, when the taking loop asks, then it takes nothing, and takes again
|
||||
// once there is room.
|
||||
func TestANodeUnderTheFloorOfFreeSpaceTakesNoWork(t *testing.T) {
|
||||
was := builder.FreeBytes
|
||||
t.Cleanup(func() { builder.FreeBytes = was })
|
||||
builder.FreeBytes = func(string) (uint64, error) { return 2 << 30, nil }
|
||||
if !noRoom(t.TempDir()) {
|
||||
t.Error("a node with 2 GiB free took work")
|
||||
}
|
||||
builder.FreeBytes = func(string) (uint64, error) { return 200 << 30, nil }
|
||||
if noRoom(t.TempDir()) {
|
||||
t.Error("a node with 200 GiB free refused work")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,17 +36,3 @@ func TestTheSiblingsAnAskNamesReachTheSpecAndTheOutcome(t *testing.T) {
|
||||
t.Fatalf("a request that names none has siblings %+v", plain.Siblings)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC2: the held values a check request carries reach the spec; a request that carries none leaves it empty.
|
||||
func TestTheHeldValuesAnAskCarriesReachTheSpec(t *testing.T) {
|
||||
spec := checkSpecOf(link.BuildRequest{Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog",
|
||||
HeldConfig: []link.HeldConfig{{Pull: "novox/mesh-catalog#434", Head: "a", Node: "ace", Module: "n8n", Key: "media-gid",
|
||||
Value: []byte("2000"), Approved: "the operator"}}}})
|
||||
if len(spec.Held) != 1 || spec.Held[0].Node != "ace" || spec.Held[0].Key != "media-gid" || string(spec.Held[0].Value) != "2000" ||
|
||||
spec.Held[0].Approved != "the operator" {
|
||||
t.Fatalf("the spec reads %+v", spec.Held)
|
||||
}
|
||||
if plain := checkSpecOf(link.BuildRequest{Check: &link.CheckRequest{Owner: "o", Repo: "r"}}); plain.Held != nil {
|
||||
t.Fatalf("a request that carries none has %+v", plain.Held)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -91,7 +91,7 @@ func TestKillEndsTheBuildRunningHereAndNoOther(t *testing.T) {
|
||||
t.Errorf("kill said %q", said)
|
||||
}
|
||||
// Removed at the kill and again once the build had ended: a container made in between is caught.
|
||||
if len(removed) != 6 || !strings.Contains(removed[0], "label=mesh.build=build-1") || !strings.Contains(removed[3], "label=mesh.build=build-1") {
|
||||
if len(removed) != 4 || !strings.Contains(removed[0], "label=mesh.build=build-1") || !strings.Contains(removed[2], "label=mesh.build=build-1") {
|
||||
t.Errorf("removed %v", removed)
|
||||
}
|
||||
if c := h.current(); c.Running != nil {
|
||||
@@ -160,7 +160,7 @@ func TestAHolderStartingRemovesWhatEarlierBuildsLeftHere(t *testing.T) {
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
if n := h.removeLeftBehind(); n != 2 || len(removed) != 3 || removed[2] != "docker rm -f -v c1 c2" {
|
||||
if n := h.removeLeftBehind(); n != 2 || len(removed) != 2 || removed[1] != "docker rm -f c1 c2" {
|
||||
t.Fatalf("removed %d: %v", n, removed)
|
||||
}
|
||||
h.remove = func(context.Context, string, string, ...string) (string, error) {
|
||||
|
||||
@@ -64,8 +64,6 @@ is dialled except the broker.
|
||||
MESH_NPM_TOKEN the token for it, likewise
|
||||
MESH_NPM_SCOPE the scope it answers for (default: @novox)
|
||||
MESH_WORKSPACE where to clone and build (default: a temporary directory)
|
||||
MESH_BUILD_CGROUP_PARENT the slice every container a build or check starts joins, which carries the
|
||||
machine's limits on builds (novox/hq issue 1798; default: the runtime's own)
|
||||
|
||||
It also builds one module and stops, which is how a mesh is raised — before there is a
|
||||
broker to take work from or a registry to publish into:
|
||||
@@ -139,10 +137,6 @@ func run() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// A build and a check run below the machine's own work (novox/hq issue 1798): the agent itself at the
|
||||
// lowest priority, which its git and go children inherit, and every container in the build slice.
|
||||
_ = syscall.Setpriority(syscall.PRIO_PROCESS, 0, 19)
|
||||
builder.CgroupParent = strings.TrimSpace(os.Getenv("MESH_BUILD_CGROUP_PARENT"))
|
||||
workspace := os.Getenv("MESH_WORKSPACE")
|
||||
if workspace == "" {
|
||||
workspace = os.TempDir() + "/mesh-builder"
|
||||
@@ -190,7 +184,7 @@ func run() error {
|
||||
// Paused or drained, it takes nothing new (novox/hq ADR 0219, ADR 0335): a drained agent finishes the build
|
||||
// in hand and then says idle, and its container is replaced between builds.
|
||||
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{
|
||||
Paused: func() bool { return h.Paused() || drained() || markFaulty(time.Now()) || noRoom(workspace) },
|
||||
Paused: func() bool { return h.Paused() || drained() },
|
||||
Taking: func() { taking(seat) },
|
||||
Claim: claim,
|
||||
})
|
||||
@@ -505,10 +499,6 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
|
||||
Modules: c.Modules, New: c.New, Manifests: c.Manifests, Judge: c.Judge, Base: c.Base,
|
||||
Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held),
|
||||
Tests: c.Tests, StoreOnly: c.StoreOnly, SiblingsUnknown: c.SiblingsUnknown}
|
||||
for _, h := range c.HeldConfig {
|
||||
spec.Held = append(spec.Held, builder.HeldValue{Pull: h.Pull, Head: h.Head, Node: h.Node, Module: h.Module, Key: h.Key,
|
||||
Value: h.Value, Approved: h.Approved, By: h.By, At: h.At})
|
||||
}
|
||||
for _, s := range c.Siblings {
|
||||
spec.Siblings = append(spec.Siblings, builder.Sibling{Repository: s.Repository, Number: s.Number, Head: s.Head,
|
||||
Base: s.Base, Dir: s.Dir})
|
||||
|
||||
@@ -408,21 +408,12 @@ func issueOnAssign(ctx context.Context, open *stores, node, module string) strin
|
||||
return ""
|
||||
}
|
||||
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username()
|
||||
_, reads := m.OwnSecrets["broker"]
|
||||
if !reads && !runsTheProvisionerClient(m) {
|
||||
// A module whose bus accounts are its processes' alone has none of its own: only the processes that
|
||||
// are not issued yet are issued (issue 1226).
|
||||
return processAccountsOnAssign(ctx, inv, m, node)
|
||||
}
|
||||
if !reads {
|
||||
if _, reads := m.OwnSecrets["broker"]; !reads {
|
||||
// A module running the provisioner client alone is minted the client's account (issue 917).
|
||||
user = broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: module}.Username()
|
||||
}
|
||||
if _, minted, err := inv.BusUserHash(ctx, user); err != nil {
|
||||
if _, minted, err := inv.BusUserHash(ctx, user); err != nil || minted {
|
||||
return ""
|
||||
} else if minted {
|
||||
// Its own account stands and is kept; a process it gained is not issued yet (issue 1226).
|
||||
return processAccountsOnAssign(ctx, inv, m, node)
|
||||
}
|
||||
busAddress, err := broker.BusAddress()
|
||||
if err == nil {
|
||||
@@ -435,16 +426,6 @@ func issueOnAssign(ctx context.Context, open *stores, node, module string) strin
|
||||
return fmt.Sprintf("its bus credential is issued and sealed to %s, and arrives with the push", node)
|
||||
}
|
||||
|
||||
// processAccountsOnAssign issues the process accounts of an assigned module not issued yet, and says what it did.
|
||||
func processAccountsOnAssign(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node string) string {
|
||||
line, err := issueMissingProcessAccounts(ctx, inv, m, node)
|
||||
if err != nil {
|
||||
return fmt.Sprintf("its process accounts are not issued (%v): `module issue %s --node %s` first — "+
|
||||
"`push %s` refuses to send %s until it is", err, m.Module, node, node, m.Module)
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
// refusedMovingTheController refuses assigning or unassigning the controller's module through a verb (review of
|
||||
// novox/hq ADR 0272): the node it is assigned to is the control-node, and the control-node's operator account is
|
||||
// the controller's terminal, so whoever moves the module chooses the terminal. At the terminal alone, as every
|
||||
|
||||
@@ -645,10 +645,6 @@ func doesWords(act conditions.Action) string {
|
||||
return "the delivery is stopped"
|
||||
case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["retry"] != "":
|
||||
return "the failed builds are asked again"
|
||||
case act.Verb == broker.ControllerSeat+".push" && act.Arguments["node"] != "":
|
||||
return "the held builds are sent to " + act.Arguments["node"]
|
||||
case act.Verb == broker.ControllerSeat+".upgrade" && act.Arguments["release-backlog"] != "":
|
||||
return "the builds held after the failed delivery are released"
|
||||
case strings.HasSuffix(act.Verb, ".restart"):
|
||||
return "its service is restarted on " + act.Machine
|
||||
}
|
||||
|
||||
@@ -232,8 +232,6 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
|
||||
fmt.Print(nothingReadsIt(repository, m))
|
||||
return nil
|
||||
}
|
||||
// **A module the merge adds is asked for now** (novox/hq issue 1617), not when the batch is cut.
|
||||
askNewModulesAtHearing(ctx, inv, m, entries, read, facts.from)
|
||||
// **Its delivery group's word, before the plans are held** (ADR 0306 decision 2): a slow answer holds up no walk.
|
||||
own := ownPath(touches)
|
||||
waits, holding, err := groupWaitHolding(ctx, inv, m, deliverySeatHeld(entries))
|
||||
@@ -800,9 +798,6 @@ type keptMerge struct {
|
||||
// FirstBuild says the merge is a first merge into a repository the mesh builds nothing from, kept to ask the first
|
||||
// build of its modules once mesh-delivery's word on it is kept (novox/hq ADR 0432, issue 1146).
|
||||
FirstBuild bool `json:"first_build,omitempty"`
|
||||
// Held says which first builds of the merge were refused for a word the running build-agent does not read: the merge
|
||||
// waits for a build-agent that reads them, and is never failed (novox/hq ADR 0482 §5, issue 1173).
|
||||
Held []heldFirstBuild `json:"held,omitempty"`
|
||||
}
|
||||
|
||||
// announcedOf is a kept merge as the forge announced it, and what it moved; empty where the record says none.
|
||||
|
||||
@@ -768,9 +768,6 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
||||
}
|
||||
return manifest, kept, err
|
||||
}
|
||||
// A bus process the new definition gains gets its account now, on each machine assigned the module, so the
|
||||
// next push composes a real account (issue 1226).
|
||||
issueGainedProcessAccounts(ctx, inv, manifest)
|
||||
// Which repository it is registered from, by the forge's own id (novox/hq ADR 0266).
|
||||
if err := inv.SetSourceIdentity(ctx, manifest.Module, repoID); err != nil {
|
||||
return manifest, kept, err
|
||||
|
||||
@@ -32,13 +32,6 @@ func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Ra
|
||||
if err := broker.Raise(r, names); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// And each node's bucket for the files of its declaration that travel by digest (novox/hq ADR 0486 §4), so a node finds its own
|
||||
// whenever the bus is up.
|
||||
if files, ok := r.(interface{ EnsureDeclarationFiles(nodes []string) error }); ok {
|
||||
if err := files.EnsureDeclarationFiles(names); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
// The work queues of the mesh's own roles (novox/hq ADR 0121). The queue before the holder,
|
||||
// deliberately: work queues until somebody arrives to do it, so assigning a build machine a week
|
||||
// after something started asking for builds flushes the backlog instead of having lost it.
|
||||
|
||||
@@ -191,16 +191,6 @@ func (f following) pullUpdated(ctx context.Context, p link.PullUpdated, w io.Wri
|
||||
scope.Modules, scope.Dependents, scope.New, scope.Manifests = nil, nil, nil, nil
|
||||
composedGate(&direct)
|
||||
}
|
||||
// **Mesh-config is told what the head declares at every announced head of a pull request in a repository that holds
|
||||
// modules** (novox/hq ADR 0502 clauses 2 and 6, issue 1826), also when the head touches no module: a head that reverts
|
||||
// the change must drop the value held for it. Asked of nothing else here.
|
||||
this := pullAtOf(p)
|
||||
if this.Truncated {
|
||||
this.Paths, this.Truncated = truncatedPaths(entries, p.Owner, p.Repo, scope.Modules), false
|
||||
}
|
||||
if scope.gated() || holdsModules(entries, p.Owner, p.Repo) {
|
||||
tellPullDeclares(ctx, w, []pullAt{this})
|
||||
}
|
||||
switch {
|
||||
case !scope.gated() && !scope.Mesh:
|
||||
fmt.Fprintf(w, "%s/%s#%d (%.8s): %s, and the repository is not the mesh's: said, nothing run\n",
|
||||
@@ -219,9 +209,6 @@ func (f following) pullUpdated(ctx context.Context, p link.PullUpdated, w io.Wri
|
||||
}
|
||||
request.Check.Plan = &plan
|
||||
request.Check.Composed = composed
|
||||
if !composed {
|
||||
carryHeld(ctx, w, &request, []pullAt{this})
|
||||
}
|
||||
// **The group's other open pull requests are read beside it** (novox/hq ADR 0382 §1).
|
||||
siblingsFrom(&request, p, entries, lines, groupsErr)
|
||||
seat := buildSeatHeld(ctx)
|
||||
@@ -607,13 +594,11 @@ func checkedOf(result link.BuildResult) link.Checked {
|
||||
// A delivery group's composed check (novox/hq ADR 0239): every head it judged, this one first.
|
||||
if g := result.Checked; g.Group != "" {
|
||||
c.Group = g.Group
|
||||
// Every member is named with its trunk commit (novox/hq issue 1106): the seat's word, else the one the forge
|
||||
// announced with the head, so the forge's merge tool never meets a member whose verdict names none.
|
||||
c.Members = append(c.Members, link.CheckedMember{Owner: g.Owner, Repo: g.Repo, Number: g.Number,
|
||||
Commit: result.Ref, Trunk: firstTrunk(c.Trunk, g.Trunk)})
|
||||
Commit: result.Ref, Trunk: c.Trunk})
|
||||
for _, m := range g.Members {
|
||||
c.Members = append(c.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo, Number: m.Number,
|
||||
Commit: m.Ref, Trunk: firstTrunk(memberTrunk(result.Check, m), m.Trunk)})
|
||||
Commit: m.Ref, Trunk: memberTrunk(result.Check, m)})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -642,16 +627,6 @@ func checkedOf(result link.BuildResult) link.Checked {
|
||||
return c
|
||||
}
|
||||
|
||||
// firstTrunk is the first trunk commit named, as twelve characters; empty when none is.
|
||||
func firstTrunk(named ...string) string {
|
||||
for _, n := range named {
|
||||
if n != "" {
|
||||
return twelve(n)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// memberTrunk is the trunk commit the build seat merged one of a group's heads into (novox/hq ADR 0323 §6), empty
|
||||
// when it said none.
|
||||
func memberTrunk(outcome *link.CheckOutcome, m link.GroupMember) string {
|
||||
|
||||
@@ -250,31 +250,6 @@ func TestAVerdictNamesTheTrunkCommitItJudged(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// **A group's composed verdict names every member's trunk commit** (novox/hq issue 1106): a member the build seat named no
|
||||
// trunk for (an SDK member whose base the seat did not merge, or a seat from before the rule) is named by the trunk the forge
|
||||
// announced with its head, and the primary the same, so the merge tool never reads "names no trunk commit" on a member
|
||||
// whose trunk did not move.
|
||||
func TestAGroupVerdictNamesEveryMembersTrunkTheSDKMemberToo(t *testing.T) {
|
||||
group := &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog", Number: 7, Base: "main", Group: "feat/x",
|
||||
Trunk: "dddddddddddd0000",
|
||||
Members: []link.GroupMember{
|
||||
{Owner: "novox", Repo: "mesh-sdk", Number: 8, Ref: "cccccccccccc0000", Base: "main", Trunk: "eeeeeeeeeeee0000"},
|
||||
{Owner: "novox", Repo: "mesh-host", Number: 9, Ref: "bbbbbbbbbbbb0000", Base: "main", Trunk: "ffffffffffff0000"}}}
|
||||
c := checkedOf(link.BuildResult{ID: "g", Ref: "aaaaaaaaaaaa0000", Checked: group, Check: &link.CheckOutcome{
|
||||
Verdict: "pass", Head: "aaaaaaaaaaaa", Gate: &link.CheckLayer{Verdict: "pass"},
|
||||
Members: []link.CheckedMember{{Owner: "novox", Repo: "mesh-host", Commit: "bbbbbbbbbbbb", Trunk: "222222222222"}}}})
|
||||
want := []string{"dddddddddddd", "eeeeeeeeeeee", "222222222222"}
|
||||
if len(c.Members) != 3 {
|
||||
t.Fatalf("the composed check names its members %+v", c.Members)
|
||||
}
|
||||
for i, m := range c.Members {
|
||||
if m.Trunk != want[i] {
|
||||
t.Fatalf("member %d (%s/%s) is named with trunk %q, want %q (the seat's word first, else the announced one): %+v",
|
||||
i, m.Owner, m.Repo, m.Trunk, want[i], c.Members)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **The merge made for a check is never registered** (novox/hq ADR 0323 §2): a build of it — a commit on no branch
|
||||
// of the forge — is refused, naming the trunk. The rule of ADR 0238 §1 stands unchanged; this holds it for the
|
||||
// merge commit a check makes.
|
||||
|
||||
@@ -233,12 +233,6 @@ func prepareSend(ctx context.Context, open *stores, node string) ([]string, erro
|
||||
if !slices.Contains(in.BusMinted, "provisioner."+node) {
|
||||
continue
|
||||
}
|
||||
default:
|
||||
// A bus process's account nobody issued still gets the stand-in, until the composition leaves
|
||||
// the module off (issue 1753): the warning names the verb (issue 1226).
|
||||
if line := processAccountWarning(m, node, name, in.BusMinted); line != "" {
|
||||
sayWarning(ctx, line)
|
||||
}
|
||||
}
|
||||
if _, err := inv.SecretForModule(ctx, node, m.Module, name); err == nil {
|
||||
did = true
|
||||
|
||||
@@ -200,7 +200,6 @@ func composeQuestion(ctx context.Context, open *stores, node string, mode compos
|
||||
}
|
||||
sayFound(ctx, d)
|
||||
out := sendableOf(d)
|
||||
out.fileReferences = readsFileReferences(in.Input, node)
|
||||
if !withOrder {
|
||||
out.Sequence, out.Epoch, out.Generation = 0, 0, 0
|
||||
}
|
||||
@@ -239,11 +238,7 @@ func markUnresolvable(err error) error {
|
||||
// still names anything foreseen is refused, said, and never sent.
|
||||
func composeSend(ctx context.Context, open *stores, node string) (sendable, error) {
|
||||
declared, _, err := composeSendValidated(ctx, open, node)
|
||||
if err != nil {
|
||||
return declared, err
|
||||
}
|
||||
// Never a declaration naming an image the store does not hold (novox/hq issue 1742): that module is left out.
|
||||
return withoutMissingImages(ctx, open, node, declared), nil
|
||||
return declared, err
|
||||
}
|
||||
|
||||
// composeSendValidated is composeSend and what the node-engine's validator refuses of the declaration it composed, for the
|
||||
@@ -286,7 +281,6 @@ func composeSendValidated(ctx context.Context, open *stores, node string) (senda
|
||||
if len(left) == 0 && len(in.wouldMake) == 0 {
|
||||
sayFound(ctx, d)
|
||||
out := sendableOf(d)
|
||||
out.fileReferences = readsFileReferences(in.Input, node)
|
||||
// The sender stamps the order it allots (order.stamp); a composition for a send carries none of its own.
|
||||
out.Sequence, out.Epoch, out.Generation = 0, 0, 0
|
||||
here, _ := nodeOf(in.Input, node)
|
||||
@@ -372,7 +366,7 @@ func sayFound(ctx context.Context, d compose.Declared) {
|
||||
|
||||
// sendableOf is a composed declaration as the controller's sends know it.
|
||||
func sendableOf(d compose.Declared) sendable {
|
||||
out := sendable{Resources: compactKept(d.Resources), Sequence: d.Sequence, Epoch: d.Epoch, Generation: d.Generation,
|
||||
out := sendable{Resources: d.Resources, Sequence: d.Sequence, Epoch: d.Epoch, Generation: d.Generation,
|
||||
ReadsModules: d.ReadsModules, moduleOf: d.ModuleOf, buildIDs: d.BuildIDs, modules: d.Modules,
|
||||
localNetwork: d.LocalNetwork, Received: d.Received, Mesh: d.Mesh, BusUsers: d.BusUsers,
|
||||
LeftOut: d.LeftOut, leftOutWhy: d.LeftOutWhy, withheld: d.Withheld, unbound: d.Unbound, foreseen: d.Foreseen,
|
||||
|
||||
@@ -1,65 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// Issue 1638 (novox/hq ADR 0480 clause 7), taken in with mesh-sdk's container `groups`: the controller sends a container's
|
||||
// groups only to a node-engine that reports the capability container-groups, and holds the module on any other, naming it.
|
||||
|
||||
const n8nWithAGroup = `{"module":"n8n","version":"1",
|
||||
"config":{"media-gid":{"type":"number","minimum":1,"required":true,"description":"the group that owns the media library"}},
|
||||
"resources":[{"id":"server","type":"container","name":"n8n",
|
||||
"image":"n8nio/n8n@sha256:4846eb2f4b874ab04cde7fc1e249d2ddaec66e9aea64439beb2972cfea88e3c0",
|
||||
"groups":["${config:media-gid}"]}]}`
|
||||
|
||||
// printedBy runs reportUnhostable and returns what it printed.
|
||||
func printedBy(t *testing.T, node string, plan catalogue.Resolution) string {
|
||||
t.Helper()
|
||||
stdout := os.Stdout
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
os.Stdout = w
|
||||
reportUnhostable(node, plan)
|
||||
w.Close()
|
||||
os.Stdout = stdout
|
||||
out, _ := io.ReadAll(r)
|
||||
return string(out)
|
||||
}
|
||||
|
||||
// 1638/AC1: a node that reports the capability gets the module; one that does not has it held, the capability named.
|
||||
func TestAModuleWithGroupsIsHeldOnANodeThatDoesNotReportTheCapability(t *testing.T) {
|
||||
m, err := catalogue.ParseManifest([]byte(n8nWithAGroup))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
shelf := catalogue.Shelf{"n8n": m}
|
||||
reads := catalogue.Node{Name: "ace", Capabilities: map[string]bool{"container-runtime": true, "container-groups": true}}
|
||||
old := catalogue.Node{Name: "g14", Capabilities: map[string]bool{"container-runtime": true}}
|
||||
|
||||
got, err := catalogue.Resolve(shelf, []string{"n8n"}, reads, catalogue.World{})
|
||||
if err != nil || len(got.Unhostable) != 0 || len(got.Modules) != 1 {
|
||||
t.Fatalf("a node that reads groups: %v, held %+v", err, got.Unhostable)
|
||||
}
|
||||
if out := printedBy(t, "ace", got); out != "" {
|
||||
t.Errorf("a node that reads groups was told: %q", out)
|
||||
}
|
||||
|
||||
got, err = catalogue.Resolve(shelf, []string{"n8n"}, old, catalogue.World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got.Modules) != 0 || len(got.Unhostable) != 1 {
|
||||
t.Fatalf("n8n was sent to a node that does not report container-groups: hosted %d, held %+v", len(got.Modules), got.Unhostable)
|
||||
}
|
||||
if out := printedBy(t, "g14", got); !strings.Contains(out, "n8n") || !strings.Contains(out, "container-groups") {
|
||||
t.Errorf("the hold does not name the module and the capability: %q", out)
|
||||
}
|
||||
}
|
||||
@@ -1,110 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/overlay"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
)
|
||||
|
||||
// 1696/AC2, ADR 0486 §5: a node is sent a file by digest only when its profile reports the capability. A node whose profile lacks
|
||||
// it, and a node the mesh has no profile of, are sent every content inline; a node that reports it is sent the reference. The test
|
||||
// goes through composeQuestion and composeSend, the two places readsFileReferences is read (compose_sdk.go).
|
||||
func TestAFileTravelsByDigestOnlyToANodeThatReportsTheCapability(t *testing.T) {
|
||||
open := aMesh(t) // anchor and laptop report no file-references
|
||||
ctx := t.Context()
|
||||
big := strings.Repeat("0123456789abcdef", 8<<10) // 128 KiB
|
||||
register(t, open, catalogue.Manifest{Module: "bigfile", Version: "1",
|
||||
Resources: []map[string]any{{"id": "blob", "type": "file", "path": "/etc/bigfile/blob", "mode": "0644", "content": big}}})
|
||||
|
||||
// reader reports the capability; bare has never reported a profile at all.
|
||||
for i, n := range []struct {
|
||||
name string
|
||||
profile map[string]any
|
||||
}{
|
||||
{"reader", map[string]any{"capabilities": []any{
|
||||
map[string]any{"name": "container-runtime", "present": true},
|
||||
map[string]any{"name": "systemd", "present": true},
|
||||
map[string]any{"name": sdkdecl.FileReferencesCapability, "present": true}}}},
|
||||
{"bare", nil},
|
||||
} {
|
||||
record, err := open.inventory.AddNode(ctx, n.name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetPlace(ctx, n.name, n.name+".example:51820", "here", false, fmt.Sprintf("10.77.0.%d", i+10)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n.profile != nil {
|
||||
if err := open.inventory.RecordProfile(ctx, record.ID, n.profile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.Assign(ctx, n.name, overlay.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, node := range []string{"laptop", "reader", "bare"} {
|
||||
if _, err := assign(ctx, open, node, "bigfile"); err != nil {
|
||||
t.Fatalf("assign %s: %v", node, err)
|
||||
}
|
||||
}
|
||||
|
||||
byDigest := map[string]bool{"laptop": false, "bare": false, "reader": true}
|
||||
for node, want := range byDigest {
|
||||
asked, _, err := composeQuestion(ctx, open, node, compose.Reading, false)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", node, err)
|
||||
}
|
||||
sent, err := composeSend(ctx, open, node)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", node, err)
|
||||
}
|
||||
for what, s := range map[string]sendable{"question": asked, "send": sent} {
|
||||
if s.fileReferences != want {
|
||||
t.Fatalf("%s, %s: fileReferences %v, want %v", node, what, s.fileReferences, want)
|
||||
}
|
||||
body, err := s.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got struct {
|
||||
Resources []map[string]any `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var found bool
|
||||
for _, r := range got.Resources {
|
||||
if r["path"] != "/etc/bigfile/blob" {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
inline := r["content"] == big
|
||||
if want && (inline || r["digest"] == nil || r["size"] == nil) {
|
||||
t.Fatalf("%s, %s: a node that reads references is sent the content inline", node, what)
|
||||
}
|
||||
if !want && (!inline || r["digest"] != nil) {
|
||||
t.Fatalf("%s, %s: a node that does not report the capability is sent a reference", node, what)
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("%s, %s: the file is not in the declaration", node, what)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,253 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// **A declaration's large files travel by their digest to a node that can read one** (novox/hq ADR 0486 §1, §5, issue 1696).
|
||||
//
|
||||
// The declaration to a node that reports sdkdecl.FileReferencesCapability carries, in place of the content of every
|
||||
// file resource of more than sdkdecl.FileReferenceThreshold bytes that may travel so (File.TravelsByDigest: plain content or
|
||||
// bytes, not sealed, not filled with secrets, not written into another file), its sha256 digest and size. The content is put
|
||||
// into the bucket of the bus under the hex of the digest before the declaration is sent. A node that does not report the
|
||||
// capability is sent every content inline, as before: an older node-engine refuses a field it does not know, whole.
|
||||
|
||||
// readsFileReferences says the node reports the capability of reading a file given by digest (ADR 0486 §5). A node the mesh has
|
||||
// no profile of reads none.
|
||||
func readsFileReferences(in compose.Input, node string) bool {
|
||||
n, ok := nodeOf(in, node)
|
||||
return ok && n.Capabilities[sdkdecl.FileReferencesCapability]
|
||||
}
|
||||
|
||||
// referencing is the resources as a node that reads references is sent them, and the content of each file that went by
|
||||
// digest, by the name of its object (the hex of its digest). The resources it is given are not changed. A file that does not
|
||||
// travel by digest is passed as it is.
|
||||
func referencing(resources []map[string]any) ([]map[string]any, map[string][]byte, error) {
|
||||
out := make([]map[string]any, len(resources))
|
||||
var objects map[string][]byte
|
||||
for i, r := range resources {
|
||||
out[i] = r
|
||||
if kind, _ := r["type"].(string); kind != string(sdkdecl.TypeFile) {
|
||||
continue
|
||||
}
|
||||
// Cheap first: only a file with a content or bytes of the size is worth reading whole.
|
||||
content, _ := r["content"].(string)
|
||||
bytesField, _ := r["bytes"].(string)
|
||||
if len(content) <= sdkdecl.FileReferenceThreshold && len(bytesField) <= sdkdecl.FileReferenceThreshold {
|
||||
continue
|
||||
}
|
||||
raw, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("resource %v: %w", r["id"], err)
|
||||
}
|
||||
var f sdkdecl.File
|
||||
if err := json.Unmarshal(raw, &f); err != nil {
|
||||
return nil, nil, fmt.Errorf("resource %v is not a file the controller can read: %w", r["id"], err)
|
||||
}
|
||||
if !f.TravelsByDigest() {
|
||||
continue
|
||||
}
|
||||
ref, body, err := f.AsReference()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
name, err := sdkdecl.ObjectName(ref.Digest)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
with := make(map[string]any, len(r)+2)
|
||||
for k, v := range r {
|
||||
with[k] = v
|
||||
}
|
||||
delete(with, "content")
|
||||
delete(with, "bytes")
|
||||
with["digest"], with["size"] = ref.Digest, ref.Size
|
||||
out[i] = with
|
||||
if objects == nil {
|
||||
objects = map[string][]byte{}
|
||||
}
|
||||
objects[name] = body
|
||||
}
|
||||
return out, objects, nil
|
||||
}
|
||||
|
||||
// namedObjects are the objects of the bucket a body names: the object name of every file resource given by reference,
|
||||
// with its size. It reads a declaration as the node does, so it also reads the latest one the bus holds.
|
||||
func namedObjects(body []byte) (map[string]int64, error) {
|
||||
var envelope struct {
|
||||
Resources []struct {
|
||||
Type string `json:"type"`
|
||||
Digest string `json:"digest"`
|
||||
Size int64 `json:"size"`
|
||||
} `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &envelope); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out map[string]int64
|
||||
for _, r := range envelope.Resources {
|
||||
if r.Type != string(sdkdecl.TypeFile) || r.Digest == "" {
|
||||
continue
|
||||
}
|
||||
name, err := sdkdecl.ObjectName(r.Digest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if out == nil {
|
||||
out = map[string]int64{}
|
||||
}
|
||||
out[name] = r.Size
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// objectStore is what putting the objects needs of a node's bucket.
|
||||
type objectStore interface {
|
||||
// Has says whether an object of that name is there, complete, and whether it carries the sweep's mark of being named by no
|
||||
// node's latest declaration.
|
||||
Has(ctx context.Context, name string) (have, marked bool, err error)
|
||||
// Put writes the object under that name, replacing one of the same name (and so erasing its mark).
|
||||
Put(ctx context.Context, name string, content []byte) error
|
||||
}
|
||||
|
||||
// putObjects makes sure every object of the declaration is in the bucket before the declaration that names it is sent: a node
|
||||
// that reads the declaration finds each object. An object already there (the same content, the same name) is not written again,
|
||||
// **unless the sweep marked it unnamed**: then it is written again, so the mark that would have it removed is gone with the
|
||||
// old object (issue 1696, review: a mark read just before a send must not take an object the send relies on). Refuses on the
|
||||
// first it cannot put, naming it, so the declaration is not sent.
|
||||
func putObjects(ctx context.Context, store objectStore, objects map[string][]byte) error {
|
||||
names := make([]string, 0, len(objects))
|
||||
for name := range objects {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
have, marked, err := store.Has(ctx, name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("whether the object %s is in the bucket: %w", name[:12], err)
|
||||
}
|
||||
if have && !marked {
|
||||
continue
|
||||
}
|
||||
if err := store.Put(ctx, name, objects[name]); err != nil {
|
||||
return fmt.Errorf("putting the object %s (%d bytes) in the bucket: %w", name[:12], len(objects[name]), err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// busObjects is a node's bucket on the controller's own connection.
|
||||
type busObjects struct{ store jetstream.ObjectStore }
|
||||
|
||||
// openBucket opens the bucket of a node's files, making it first if it is not there (it is asserted for every node at the
|
||||
// controller's start; a node adopted since is covered here).
|
||||
func openBucket(ctx context.Context, conn *nats.Conn, node string) (busObjects, error) {
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return busObjects{}, err
|
||||
}
|
||||
bucket, err := broker.DeclarationFilesBucketOf(node)
|
||||
if err != nil {
|
||||
return busObjects{}, fmt.Errorf("the object bucket of node %q: %w", node, err)
|
||||
}
|
||||
store, err := js.ObjectStore(ctx, bucket)
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
if err = broker.EnsureDeclarationFiles(ctx, js, node); err == nil {
|
||||
store, err = js.ObjectStore(ctx, bucket)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return busObjects{}, fmt.Errorf("the object bucket %s: %w", bucket, err)
|
||||
}
|
||||
return busObjects{store}, nil
|
||||
}
|
||||
|
||||
func (b busObjects) Has(ctx context.Context, name string) (bool, bool, error) {
|
||||
info, err := b.store.GetInfo(ctx, name)
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrObjectNotFound):
|
||||
return false, false, nil
|
||||
case err != nil:
|
||||
return false, false, err
|
||||
}
|
||||
_, marked := info.Metadata[declarationUnnamed]
|
||||
return !info.Deleted, marked, nil
|
||||
}
|
||||
|
||||
func (b busObjects) Put(ctx context.Context, name string, content []byte) error {
|
||||
_, err := b.store.PutBytes(ctx, name, content)
|
||||
return err
|
||||
}
|
||||
|
||||
// filesLocks hold a node's bucket for one act at a time in this process: a send puts its objects and publishes the declaration that
|
||||
// names them under the node's lock, and the sweep judges the node's bucket under the same one, so the sweep never removes an
|
||||
// object between a send's put and its declaration. Another process (a person's push) is covered by the put erasing a mark and by
|
||||
// the sweep looking at an object again just before it removes it.
|
||||
var filesLocks sync.Map
|
||||
|
||||
func lockFilesOf(node string) func() {
|
||||
l, _ := filesLocks.LoadOrStore(node, &sync.Mutex{})
|
||||
mu := l.(*sync.Mutex)
|
||||
mu.Lock()
|
||||
return mu.Unlock
|
||||
}
|
||||
|
||||
// kindFilesNotPut is a node whose files that travel by digest could not be put in its bucket, send after send.
|
||||
const kindFilesNotPut = "declaration-files-not-put"
|
||||
|
||||
// filesNotPutAfter is how many sends in a row may fail to put before the condition is raised: the first send to the machine holding
|
||||
// the bus fails for want of the grant that very send delivers, and says so, and is not yet a fault.
|
||||
const filesNotPutAfter = 2
|
||||
|
||||
var (
|
||||
filesFailMu sync.Mutex
|
||||
filesFailed = map[string]int{}
|
||||
)
|
||||
|
||||
func filesNotPutObservation(node, summary string) conditions.Observation {
|
||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: kindFilesNotPut, Token: kindFilesNotPut,
|
||||
Machine: node, Severity: conditions.Warning, Source: "sends", Summary: summary}
|
||||
}
|
||||
|
||||
// notePut records how putting a send's objects went: a failure counts, the second in a row raises the condition, a success
|
||||
// clears the count and the condition.
|
||||
func notePut(ctx context.Context, node string, put error) {
|
||||
filesFailMu.Lock()
|
||||
if put == nil {
|
||||
delete(filesFailed, node)
|
||||
} else {
|
||||
filesFailed[node]++
|
||||
}
|
||||
n := filesFailed[node]
|
||||
filesFailMu.Unlock()
|
||||
keeper := conditionsFrom
|
||||
if keeper == nil {
|
||||
return
|
||||
}
|
||||
key := filesNotPutObservation(node, "").Key()
|
||||
switch {
|
||||
case put == nil:
|
||||
if _, err := keeper.Clear(ctx, key, node+" was sent its files by digest"); err != nil {
|
||||
fmt.Printf("the condition %s could not be cleared: %v\n", key, err)
|
||||
}
|
||||
case n >= filesNotPutAfter:
|
||||
summary := fmt.Sprintf("%s's files that travel by digest could not be put in its bucket %d sends in a row, so it is sent them "+
|
||||
"inline: %v", node, n, put)
|
||||
if _, err := keeper.Observe(ctx, filesNotPutObservation(node, summary)); err != nil {
|
||||
fmt.Printf("the condition %s could not be raised: %v\n", key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,246 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
type sigOf struct{}
|
||||
|
||||
func (sigOf) Sign(context.Context, []byte) ([]byte, error) { return make([]byte, 64), nil }
|
||||
|
||||
// 1696/AC7 (ADR 0486 §1): on a real bus with its default largest message of 1 MiB, a declaration whose files add up to more than
|
||||
// 3 MiB is sent as references, its objects are in the bucket, each fetched back whole and verified against the digest and size the
|
||||
// declaration names; the same declaration inline is refused by the controller, naming size and limit. A changed byte in an object
|
||||
// fails the verification.
|
||||
func TestADeclarationOfMoreThanThreeMiBIsSentByReferenceOnARealBus(t *testing.T) { // 1696/AC7
|
||||
s := testbus.Start(t)
|
||||
conn, err := nats.Connect(s.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
nc, err := conn.JetStream()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := nc.AddStream(&nats.StreamConfig{Name: "NODES", Subjects: []string{"mesh.node.*.declare"},
|
||||
MaxMsgsPerSubject: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
api, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := broker.EnsureDeclarationFiles(ctx, api, "novox"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Run twice: an assertion of an existing bucket is no error and no change.
|
||||
if err := broker.EnsureDeclarationFiles(ctx, api, "novox"); err != nil {
|
||||
t.Fatalf("asserting the bucket again: %v", err)
|
||||
}
|
||||
|
||||
var resources []map[string]any
|
||||
for i := 0; i < 3; i++ {
|
||||
resources = append(resources, aFileResource("big-"+string(rune('a'+i)), contentOf("big"+string(rune('a'+i)), 1200<<10)))
|
||||
}
|
||||
resources = append(resources, aFileResource("small", "tiny\n"))
|
||||
inline := sendable{Resources: resources}
|
||||
byDigest := sendable{fileReferences: true, Resources: resources}
|
||||
|
||||
bus := link.OverNATS{Conn: conn, JS: nc}
|
||||
body, err := inline.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := link.Declare(ctx, bus, sigOf{}, "novox", body, 5*time.Second); err == nil ||
|
||||
!strings.Contains(err.Error(), "1048576") || strings.Contains(err.Error(), "maximum payload exceeded") {
|
||||
t.Fatalf("the inline declaration of more than 3 MiB: %v", err)
|
||||
}
|
||||
|
||||
bucket, err := openBucket(ctx, conn, "novox")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
objects, err := byDigest.objects()
|
||||
if err != nil || len(objects) != 3 {
|
||||
t.Fatalf("objects: %d %v", len(objects), err)
|
||||
}
|
||||
if err := putObjects(ctx, bucket, objects); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := putObjects(ctx, bucket, objects); err != nil {
|
||||
t.Fatalf("putting them again: %v", err)
|
||||
}
|
||||
refBody, err := byDigest.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := link.Declare(ctx, bus, sigOf{}, "novox", refBody, 5*time.Second); err != nil {
|
||||
t.Fatalf("the declaration by reference: %v", err)
|
||||
}
|
||||
|
||||
// What the stream holds for the node is the reference form, and it names exactly the three objects.
|
||||
latest, err := latestDeclarations(ctx, conn, []string{"novox", "ace"})
|
||||
if err != nil || len(latest) != 1 {
|
||||
t.Fatalf("the latest declarations: %v %v", len(latest), err)
|
||||
}
|
||||
named, err := namedByLatest("novox", latest["novox"])
|
||||
if err != nil || len(named) != 3 {
|
||||
t.Fatalf("what the latest declaration names: %v %v", named, err)
|
||||
}
|
||||
novoxBucket, err := broker.DeclarationFilesBucketOf("novox")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store, err := api.ObjectStore(ctx, novoxBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var envelope struct {
|
||||
Resources []sdkdecl.File `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(refBody, &envelope); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range envelope.Resources {
|
||||
if !f.IsReference() {
|
||||
continue
|
||||
}
|
||||
name, _ := sdkdecl.ObjectName(f.Digest)
|
||||
got, err := store.GetBytes(ctx, name)
|
||||
if err != nil {
|
||||
t.Fatalf("the object of %s: %v", f.ID, err)
|
||||
}
|
||||
if err := f.Verify(got); err != nil {
|
||||
t.Errorf("the object does not verify: %v", err)
|
||||
}
|
||||
got[len(got)/2] ^= 1
|
||||
if err := f.Verify(got); err == nil {
|
||||
t.Errorf("a changed byte in the object of %s passed the verification", f.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// The sweep against the real bucket: all named, nothing marked; a fourth object named by none is marked, kept a day less a
|
||||
// minute, and removed at the day.
|
||||
extra := bytes.Repeat([]byte("x"), 100)
|
||||
extraName := strings.Repeat("e", 64)
|
||||
if err := putObjects(ctx, bucket, map[string][]byte{extraName: extra}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := busSweptBucket{store}
|
||||
t0 := time.Now()
|
||||
if r, err := sweepDeclarationFiles(ctx, b, named, t0); err != nil || r.Marked != 1 || r.Removed != 0 || r.Kept != 3 {
|
||||
t.Fatalf("first sweep: %+v %v", r, err)
|
||||
}
|
||||
if r, err := sweepDeclarationFiles(ctx, b, named, t0.Add(24*time.Hour-time.Minute)); err != nil || r.Removed != 0 {
|
||||
t.Fatalf("a minute short of a day: %+v %v", r, err)
|
||||
}
|
||||
if r, err := sweepDeclarationFiles(ctx, b, named, t0.Add(24*time.Hour+time.Second)); err != nil || r.Removed != 1 {
|
||||
t.Fatalf("a day after: %+v %v", r, err)
|
||||
}
|
||||
if infos, err := store.List(ctx); err != nil || len(infos) != 3 {
|
||||
t.Fatalf("the bucket holds %d objects (%v), want the 3 the declaration names", len(infos), err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1696/AC8 on a real bucket: the sweep removes an object only if it still carries the mark it judged it by. An object marked
|
||||
// unnamed, then put again by a send (which erases the mark), is kept when the sweep deletes it by the old mark.
|
||||
func TestTheSweepsDeleteLeavesAnObjectASendPutAgainAfterItWasMarked(t *testing.T) { // 1696/AC8
|
||||
s := testbus.Start(t)
|
||||
conn, err := nats.Connect(s.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
bucket, err := openBucket(ctx, conn, "novox")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
name := strings.Repeat("a", 64)
|
||||
content := map[string][]byte{name: bytes.Repeat([]byte("y"), 100)}
|
||||
if err := putObjects(ctx, bucket, content); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := busSweptBucket{bucket.store}
|
||||
judged := time.Now().Add(-25 * time.Hour).Truncate(time.Second)
|
||||
if err := b.MarkUnnamed(ctx, name, judged); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, marked, err := bucket.Has(ctx, name); err != nil || !marked {
|
||||
t.Fatalf("the object was not marked: %v %v", marked, err)
|
||||
}
|
||||
// The sweep has read the mark; a send now puts the object again, which erases it.
|
||||
if err := putObjects(ctx, bucket, content); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if have, marked, err := bucket.Has(ctx, name); err != nil || !have || marked {
|
||||
t.Fatalf("a send did not put the marked object again: have %v marked %v %v", have, marked, err)
|
||||
}
|
||||
gone, err := b.Delete(ctx, name, judged)
|
||||
if err != nil || gone {
|
||||
t.Fatalf("the sweep deleted by an old mark: gone %v, %v", gone, err)
|
||||
}
|
||||
if have, _, err := bucket.Has(ctx, name); err != nil || !have {
|
||||
t.Fatalf("the object a send relies on is gone: %v %v", have, err)
|
||||
}
|
||||
// And the mark still on it is honoured: marked again and deleted by that mark, it goes.
|
||||
if err := b.MarkUnnamed(ctx, name, judged); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if gone, err := b.Delete(ctx, name, judged); err != nil || !gone {
|
||||
t.Fatalf("an object still carrying the mark it was judged by is kept: gone %v, %v", gone, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1696/AC9 on a real bus: the sweep of a node's bucket takes the node's lock first, so it waits for a send that has put its objects
|
||||
// and not yet published the declaration that names them.
|
||||
func TestTheSweepOfANodesBucketHoldsTheNodesLock(t *testing.T) { // 1696/AC9
|
||||
s := testbus.Start(t)
|
||||
conn, err := nats.Connect(s.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
const node = "sweep-lock-node"
|
||||
if _, err := openBucket(ctx, conn, node); err != nil { // the bucket exists, or the sweep would return before the lock
|
||||
t.Fatal(err)
|
||||
}
|
||||
release := lockFilesOf(node) // a send is between its put and its publication
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
_, _ = sweepNode(ctx, conn, node)
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
t.Fatal("the sweep judged the node's bucket while a send held it")
|
||||
case <-time.After(300 * time.Millisecond):
|
||||
}
|
||||
release()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(20 * time.Second):
|
||||
t.Fatal("the sweep did not go on once the send let go")
|
||||
}
|
||||
}
|
||||
@@ -1,534 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// contentOf is a deterministic content of n bytes that does not compress to nothing and holds no character JSON escapes.
|
||||
func contentOf(seed string, n int) string {
|
||||
var b strings.Builder
|
||||
for i := 0; b.Len() < n; i++ {
|
||||
fmt.Fprintf(&b, "%s-%07d-%x;", seed, i, i*2654435761%1000003)
|
||||
}
|
||||
return b.String()[:n]
|
||||
}
|
||||
|
||||
func aFileResource(id, content string) map[string]any {
|
||||
return map[string]any{"id": id, "type": "file", "path": "/etc/" + id, "mode": "0644", "content": content}
|
||||
}
|
||||
|
||||
// controlNodeShape is a declaration shaped like the control node's on 2026-10-21 (issue 1689), the sizes measured then: a bus users
|
||||
// file of 252,678 bytes, the vault's export of 138,475 bytes (compact), the tool runtime 21 KB, the packet filter 19 KB, the launcher
|
||||
// 14 KB, and about 600 small resources for the rest (about 286 KB). `more` scales the count of small ones: 1 is the shape, 1.4 forty
|
||||
// percent more of them.
|
||||
func controlNodeShape(more float64) []map[string]any {
|
||||
res := []map[string]any{
|
||||
aFileResource("nats.bus-users", contentOf("users", 252678)),
|
||||
aFileResource("mesh-vault.kept", contentOf("kept", 138475)),
|
||||
aFileResource("node-tools.runtime", contentOf("runtime", 21<<10)),
|
||||
aFileResource("nftables.filtering", contentOf("filter", 19<<10)),
|
||||
aFileResource("mesh-host.launcher", contentOf("launcher", 14<<10)),
|
||||
}
|
||||
for i := 0; i < int(600*more); i++ {
|
||||
res = append(res, aFileResource(fmt.Sprintf("module-%03d.config", i), contentOf(fmt.Sprintf("small%d", i), 380)))
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func wireSizeOf(t *testing.T, s sendable) int {
|
||||
t.Helper()
|
||||
body, err := s.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The signature the controller's key makes is 64 bytes.
|
||||
msg, err := link.Envelope(body, make([]byte, 64))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return len(msg)
|
||||
}
|
||||
|
||||
// 1696/AC1: for a node that reports the capability, the two big files of a control-node-shaped declaration carry their digest and size
|
||||
// and no content, and the declaration is far under the bus's 1 MiB.
|
||||
func TestAFileOver64KiBTravelsByDigestToANodeThatReadsOne(t *testing.T) { // 1696/AC1
|
||||
users, kept := contentOf("users", 252678), contentOf("kept", 138475)
|
||||
s := sendable{fileReferences: true, Resources: []map[string]any{
|
||||
aFileResource("nats.bus-users", users), aFileResource("mesh-vault.kept", kept),
|
||||
aFileResource("small", "tiny\n"),
|
||||
}}
|
||||
body, err := s.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got struct {
|
||||
Resources []map[string]any `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, want := range []string{users, kept} {
|
||||
r := got.Resources[i]
|
||||
digest := sdkdecl.DigestOf([]byte(want))
|
||||
if _, has := r["content"]; has || r["digest"] != digest || r["size"] != float64(len(want)) {
|
||||
t.Errorf("%v goes with content %v, digest %v, size %v; want no content, %s and %d", r["id"], r["content"] != nil,
|
||||
r["digest"], r["size"], digest, len(want))
|
||||
}
|
||||
if r["path"] == nil || r["mode"] != "0644" {
|
||||
t.Errorf("%v lost its path or mode: %v", r["id"], r)
|
||||
}
|
||||
}
|
||||
if got.Resources[2]["content"] != "tiny\n" || got.Resources[2]["digest"] != nil {
|
||||
t.Errorf("a small file is not inline: %v", got.Resources[2])
|
||||
}
|
||||
if strings.Contains(string(body), users[:200]) || strings.Contains(string(body), kept[:200]) {
|
||||
t.Error("the content of a file that went by digest is in the body")
|
||||
}
|
||||
if n := wireSizeOf(t, s); n > 100<<10 {
|
||||
t.Errorf("the declaration is %d bytes on the wire, want under 100 KB", n)
|
||||
}
|
||||
// The objects to put are the contents, each under the hex of its digest, and a reference verifies against its object.
|
||||
objects, err := s.objects()
|
||||
if err != nil || len(objects) != 2 {
|
||||
t.Fatalf("objects: %v %v", objects, err)
|
||||
}
|
||||
for _, want := range []string{users, kept} {
|
||||
name, _ := sdkdecl.ObjectName(sdkdecl.DigestOf([]byte(want)))
|
||||
if string(objects[name]) != want {
|
||||
t.Errorf("the object %s does not hold the content", name[:12])
|
||||
}
|
||||
}
|
||||
named, err := namedObjects(body)
|
||||
if err != nil || len(named) != 2 {
|
||||
t.Fatalf("the objects the body names: %v %v", named, err)
|
||||
}
|
||||
for name := range objects {
|
||||
if _, ok := named[name]; !ok {
|
||||
t.Errorf("the body does not name the object %s it was sent with", name[:12])
|
||||
}
|
||||
if _, err := hex.DecodeString(name); err != nil || len(name) != 64 {
|
||||
t.Errorf("the object name %q is not 64 hex characters", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1696/AC2: a node that does not report the capability is sent every content inline, as before, and has no object put.
|
||||
func TestANodeWithoutTheCapabilityIsSentEveryContentInline(t *testing.T) { // 1696/AC2
|
||||
users := contentOf("users", 252678)
|
||||
s := sendable{Resources: []map[string]any{aFileResource("nats.bus-users", users)}}
|
||||
body, err := s.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got struct {
|
||||
Resources []map[string]any `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Resources[0]["content"] != users || got.Resources[0]["digest"] != nil || got.Resources[0]["size"] != nil {
|
||||
t.Fatalf("a node that reads no references is not sent the content inline: %v", got.Resources[0]["digest"])
|
||||
}
|
||||
if objects, err := s.objects(); err != nil || len(objects) != 0 {
|
||||
t.Fatalf("objects for a node that reads none: %v %v", len(objects), err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1696/AC3: only a file that may travel by digest does: one of exactly 64 KiB, a sealed one, one filled with secrets, one written
|
||||
// into another file, and a directory stay inline, whatever the node reads.
|
||||
func TestOnlyAFileThatMayTravelByDigestDoes(t *testing.T) { // 1696/AC3
|
||||
exact := contentOf("exact", 64<<10)
|
||||
big := contentOf("big", 64<<10+1)
|
||||
sealed := aFileResource("sealed", big)
|
||||
sealed["sealed"], sealed["content"] = "box", ""
|
||||
sealed["content"] = big // a sealed file's content is none; the body must pass it as it is, whatever it holds
|
||||
secret := aFileResource("secret", big)
|
||||
secret["secrets"] = map[string]any{"token": "x"}
|
||||
into := aFileResource("into", big)
|
||||
into["into"] = "/etc/other"
|
||||
s := sendable{fileReferences: true, Resources: []map[string]any{
|
||||
aFileResource("exact", exact), sealed, secret, into,
|
||||
{"id": "dir", "type": "directory", "path": "/etc/d", "content": big},
|
||||
aFileResource("big", big),
|
||||
}}
|
||||
body, err := s.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got struct {
|
||||
Resources []map[string]any `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, id := range []string{"exact", "sealed", "secret", "into", "dir"} {
|
||||
if got.Resources[i]["id"] != id || got.Resources[i]["digest"] != nil || got.Resources[i]["content"] == nil {
|
||||
t.Errorf("%s went by digest or lost its content: %v", id, got.Resources[i]["digest"])
|
||||
}
|
||||
}
|
||||
if big := got.Resources[5]; big["id"] != "big" || big["digest"] == nil || big["content"] != nil {
|
||||
t.Errorf("a plain file of 64 KiB and a byte did not go by digest: %v", big["digest"])
|
||||
}
|
||||
}
|
||||
|
||||
// A file given as bytes goes by digest on its decoded length, with the decoded bytes in the object.
|
||||
func TestAFileOfBytesTravelsByDigestOnItsDecodedLength(t *testing.T) {
|
||||
raw := []byte(contentOf("png", 70<<10))
|
||||
r := map[string]any{"id": "icon", "type": "file", "path": "/usr/share/icon.png", "bytes": stdBase64(raw)}
|
||||
s := sendable{fileReferences: true, Resources: []map[string]any{r}}
|
||||
objects, err := s.objects()
|
||||
if err != nil || len(objects) != 1 {
|
||||
t.Fatalf("%v %v", len(objects), err)
|
||||
}
|
||||
name, _ := sdkdecl.ObjectName(sdkdecl.DigestOf(raw))
|
||||
if string(objects[name]) != string(raw) {
|
||||
t.Fatal("the object is not the decoded bytes")
|
||||
}
|
||||
if r["bytes"] == nil {
|
||||
t.Fatal("the composed resource was changed in place")
|
||||
}
|
||||
}
|
||||
|
||||
type memObjects struct {
|
||||
have map[string][]byte
|
||||
marked map[string]bool
|
||||
puts []string
|
||||
fail map[string]error
|
||||
}
|
||||
|
||||
func (m *memObjects) Has(_ context.Context, name string) (bool, bool, error) {
|
||||
_, ok := m.have[name]
|
||||
return ok, m.marked[name], nil
|
||||
}
|
||||
func (m *memObjects) Put(_ context.Context, name string, c []byte) error {
|
||||
if err := m.fail[name]; err != nil {
|
||||
return err
|
||||
}
|
||||
m.puts = append(m.puts, name)
|
||||
m.have[name] = c
|
||||
delete(m.marked, name)
|
||||
return nil
|
||||
}
|
||||
|
||||
// 1696/AC4: an object already in the bucket is not put again, one absent is, and a put that fails stops the send, naming the object.
|
||||
func TestObjectsAreInTheBucketBeforeTheDeclarationIsSent(t *testing.T) { // 1696/AC4
|
||||
objects := map[string][]byte{strings.Repeat("a", 64): []byte("one"), strings.Repeat("b", 64): []byte("two")}
|
||||
b := &memObjects{have: map[string][]byte{strings.Repeat("a", 64): []byte("one")}}
|
||||
if err := putObjects(t.Context(), b, objects); err != nil || len(b.puts) != 1 || b.puts[0] != strings.Repeat("b", 64) {
|
||||
t.Fatalf("put %v (%v), want the one absent", b.puts, err)
|
||||
}
|
||||
// An object the sweep marked unnamed is put again, which erases the mark: a send relies on it.
|
||||
b = &memObjects{have: map[string][]byte{strings.Repeat("a", 64): []byte("one"), strings.Repeat("b", 64): []byte("two")},
|
||||
marked: map[string]bool{strings.Repeat("a", 64): true}}
|
||||
if err := putObjects(t.Context(), b, objects); err != nil || len(b.puts) != 1 || b.puts[0] != strings.Repeat("a", 64) || b.marked[strings.Repeat("a", 64)] {
|
||||
t.Fatalf("a marked object: put %v (%v) marked %v, want it put again and the mark gone", b.puts, err, b.marked)
|
||||
}
|
||||
b = &memObjects{have: map[string][]byte{}, fail: map[string]error{strings.Repeat("a", 64): errors.New("permissions violation")}}
|
||||
err := putObjects(t.Context(), b, objects)
|
||||
if err == nil || !strings.Contains(err.Error(), "aaaaaaaaaaaa") || !strings.Contains(err.Error(), "permissions violation") {
|
||||
t.Fatalf("a failed put: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// fallbackDelivery refuses the first declaration it is sent with the bucket's failure, as a bus that has not yet been given the
|
||||
// controller's grant does, and takes the second.
|
||||
type fallbackDelivery struct {
|
||||
bodies [][]byte
|
||||
first error
|
||||
}
|
||||
|
||||
func (f *fallbackDelivery) grant(context.Context, []readyNode) error { return nil }
|
||||
func (f *fallbackDelivery) declare(_ context.Context, _ readyNode, body []byte) (string, error) {
|
||||
f.bodies = append(f.bodies, body)
|
||||
if len(f.bodies) == 1 && f.first != nil {
|
||||
return "", fmt.Errorf("x: %w: %w", errFilesNotPut, f.first)
|
||||
}
|
||||
return "digest", nil
|
||||
}
|
||||
|
||||
// 1696/AC5: when the bucket cannot be written yet (the grant travels in the declaration of the machine holding the bus), that send goes
|
||||
// with every content inline and is made; a send that fails for another reason is not retried inline.
|
||||
func TestASendWhoseFilesCannotBePutGoesInlineOnce(t *testing.T) { // 1696/AC5
|
||||
users := contentOf("users", 100<<10)
|
||||
s := readyNode{node: "novox", declared: sendable{fileReferences: true, Resources: []map[string]any{aFileResource("nats.bus-users", users)}}}
|
||||
f := &fallbackDelivery{first: errors.New("timeout")}
|
||||
digests, err := deliver(t.Context(), f, "novox", []readyNode{s})
|
||||
if err != nil || digests["novox"] != "digest" || len(f.bodies) != 2 {
|
||||
t.Fatalf("%v %v %d", digests, err, len(f.bodies))
|
||||
}
|
||||
if strings.Contains(string(f.bodies[0]), users[:100]) || !strings.Contains(string(f.bodies[1]), users[:100]) {
|
||||
t.Error("the first body should name the file by digest and the second carry it inline")
|
||||
}
|
||||
other := &fallbackDelivery{}
|
||||
other.first = nil
|
||||
failing := &failingDelivery{err: errors.New("the bus is down")}
|
||||
if _, err := deliver(t.Context(), failing, "novox", []readyNode{s}); err == nil || failing.calls != 1 {
|
||||
t.Fatalf("a failure of another kind: %v after %d send(s)", err, failing.calls)
|
||||
}
|
||||
}
|
||||
|
||||
type failingDelivery struct {
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *failingDelivery) grant(context.Context, []readyNode) error { return nil }
|
||||
func (f *failingDelivery) declare(context.Context, readyNode, []byte) (string, error) {
|
||||
f.calls++
|
||||
return "", f.err
|
||||
}
|
||||
|
||||
// A bucket for the sweep.
|
||||
type sweepBucketFake struct {
|
||||
objects map[string]time.Time // zero: named at the last look
|
||||
deleted []string
|
||||
}
|
||||
|
||||
func (b *sweepBucketFake) Objects(context.Context) ([]bucketObject, error) {
|
||||
var out []bucketObject
|
||||
for n, at := range b.objects {
|
||||
out = append(out, bucketObject{Name: n, UnnamedSince: at})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (b *sweepBucketFake) MarkUnnamed(_ context.Context, n string, at time.Time) error {
|
||||
b.objects[n] = at
|
||||
return nil
|
||||
}
|
||||
func (b *sweepBucketFake) ClearUnnamed(_ context.Context, n string) error {
|
||||
b.objects[n] = time.Time{}
|
||||
return nil
|
||||
}
|
||||
func (b *sweepBucketFake) Delete(_ context.Context, n string, markedAt time.Time) (bool, error) {
|
||||
if at, ok := b.objects[n]; !ok || !at.Equal(markedAt) {
|
||||
return false, nil // put again since it was judged: its mark is gone
|
||||
}
|
||||
delete(b.objects, n)
|
||||
b.deleted = append(b.deleted, n)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// 1696/AC6: an object a latest declaration still names is not removed, however long; one unnamed is marked at the first look and removed
|
||||
// only when the mark is 24 hours old; one named again loses its mark.
|
||||
func TestAnObjectIsRemovedADayAfterNoLatestDeclarationNamesIt(t *testing.T) { // 1696/AC6
|
||||
named, unnamed, back := strings.Repeat("1", 64), strings.Repeat("2", 64), strings.Repeat("3", 64)
|
||||
t0 := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)
|
||||
b := &sweepBucketFake{objects: map[string]time.Time{named: t0.Add(-100 * time.Hour), unnamed: {}, back: t0.Add(-5 * time.Hour)}}
|
||||
named3 := map[string]bool{named: true, back: true}
|
||||
r, err := sweepDeclarationFiles(t.Context(), b, named3, t0)
|
||||
if err != nil || r.Marked != 1 || r.Cleared != 2 || r.Removed != 0 || len(b.deleted) != 0 {
|
||||
t.Fatalf("first look: %+v %v %v", r, err, b.deleted)
|
||||
}
|
||||
if !b.objects[named].IsZero() || !b.objects[back].IsZero() || !b.objects[unnamed].Equal(t0) {
|
||||
t.Fatalf("marks after the first look: %v", b.objects)
|
||||
}
|
||||
// Still unnamed 23 hours 59 minutes later: kept. A day after the mark: removed. The named ones stay for ever.
|
||||
if r, err = sweepDeclarationFiles(t.Context(), b, map[string]bool{named: true, back: true}, t0.Add(24*time.Hour-time.Minute)); err != nil ||
|
||||
r.Removed != 0 || len(b.deleted) != 0 {
|
||||
t.Fatalf("a minute short of a day: %+v %v %v", r, err, b.deleted)
|
||||
}
|
||||
if r, err = sweepDeclarationFiles(t.Context(), b, map[string]bool{named: true, back: true}, t0.Add(24*time.Hour)); err != nil ||
|
||||
r.Removed != 1 || len(b.deleted) != 1 || b.deleted[0] != unnamed {
|
||||
t.Fatalf("a day after: %+v %v %v", r, err, b.deleted)
|
||||
}
|
||||
if _, ok := b.objects[named]; !ok {
|
||||
t.Fatal("an object a latest declaration names was removed")
|
||||
}
|
||||
// An object named again before the day is up loses its mark and is not removed.
|
||||
b = &sweepBucketFake{objects: map[string]time.Time{unnamed: t0}}
|
||||
if _, err := sweepDeclarationFiles(t.Context(), b, map[string]bool{unnamed: true}, t0.Add(23*time.Hour)); err != nil || !b.objects[unnamed].IsZero() {
|
||||
t.Fatalf("named again: %v %v", b.objects, err)
|
||||
}
|
||||
}
|
||||
|
||||
// What a node's latest declaration names is read from the signed envelope as stored on the bus; an unreadable one is an error and
|
||||
// never "names none", which would mark an object a node needs.
|
||||
func TestTheObjectsALatestDeclarationNamesAreReadFromTheBus(t *testing.T) {
|
||||
s := sendable{fileReferences: true, Resources: []map[string]any{aFileResource("a", contentOf("a", 70<<10))}}
|
||||
body, _ := s.Body()
|
||||
msg, _ := link.Envelope(body, []byte("sig"))
|
||||
named, err := namedByLatest("novox", msg)
|
||||
name, _ := sdkdecl.ObjectName(sdkdecl.DigestOf([]byte(contentOf("a", 70<<10))))
|
||||
if err != nil || len(named) != 1 || !named[name] {
|
||||
t.Fatalf("%v %v", named, err)
|
||||
}
|
||||
if none, err := namedByLatest("ace", []byte(`{"declaration":"e30=","signature":""}`)); err != nil || len(none) != 0 {
|
||||
t.Fatalf("a declaration naming none: %v %v", none, err)
|
||||
}
|
||||
if _, err := namedByLatest("novox", []byte("not json")); err == nil {
|
||||
t.Fatal("an unreadable latest declaration was read as naming none")
|
||||
}
|
||||
}
|
||||
|
||||
// 1698/AC2 (ADR 0486 §9): a declaration shaped like the control node's, for a node that reads references, stays under half of the
|
||||
// bus's largest message on the wire; the inline form of the same does not fit at all; and the same shape with 40 percent more resources
|
||||
// fails the test's own limit, so a change that eats the room fails in a pull request and not on the live mesh.
|
||||
func TestTheControlNodesDeclarationInTheReferenceFormStaysUnderHalfTheBusLimit(t *testing.T) { // 1698/AC2
|
||||
const limit = 1048576
|
||||
shape := sendable{fileReferences: true, Resources: controlNodeShape(1)}
|
||||
n := wireSizeOf(t, shape)
|
||||
t.Logf("the control node's shape in the reference form: %d bytes on the wire, %.1f%% of %d", n, 100*float64(n)/limit, limit)
|
||||
if n > limit/2 {
|
||||
t.Errorf("the reference form is %d bytes on the wire, over half (%d) of the bus's largest message", n, limit/2)
|
||||
}
|
||||
if inline := wireSizeOf(t, sendable{Resources: controlNodeShape(1)}); inline < limit*3/4 {
|
||||
t.Errorf("the inline form is %d bytes, which is not the shape that was over the limit", inline)
|
||||
}
|
||||
grown := wireSizeOf(t, sendable{fileReferences: true, Resources: controlNodeShape(1.4)})
|
||||
if grown <= limit/2 {
|
||||
t.Errorf("the shape with 40%% more resources is %d bytes, which the test would let pass", grown)
|
||||
}
|
||||
}
|
||||
|
||||
func stdBase64(b []byte) string { return base64.StdEncoding.EncodeToString(b) }
|
||||
|
||||
// A sweep that read an object's mark before a send put it again must not remove it: the object, now without the mark, is left.
|
||||
func TestASweepDoesNotRemoveAnObjectASendPutAgainAfterItWasJudged(t *testing.T) { // 1696/AC8
|
||||
n := strings.Repeat("4", 64)
|
||||
t0 := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)
|
||||
b := &sweepBucketFake{objects: map[string]time.Time{n: t0}}
|
||||
judged := bucketObject{Name: n, UnnamedSince: t0}
|
||||
b.objects[n] = time.Time{} // the send put it again: no mark
|
||||
if gone, err := b.Delete(t.Context(), judged.Name, judged.UnnamedSince); err != nil || gone || len(b.deleted) != 0 {
|
||||
t.Fatalf("an object put again was removed: %v %v %v", gone, err, b.deleted)
|
||||
}
|
||||
}
|
||||
|
||||
// One node's bucket is held by one act at a time in this process: the sweep waits for a send that has put its objects and not yet
|
||||
// published the declaration that names them (1696/AC9).
|
||||
func TestTheSweepOfANodesBucketWaitsForASendThatHasPutItsObjects(t *testing.T) { // 1696/AC9
|
||||
unlock := lockFilesOf("lock-test-node")
|
||||
got := make(chan struct{})
|
||||
go func() {
|
||||
defer lockFilesOf("lock-test-node")()
|
||||
close(got)
|
||||
}()
|
||||
select {
|
||||
case <-got:
|
||||
t.Fatal("the sweep took the node's bucket while a send held it")
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
other := lockFilesOf("lock-test-other-node") // another node's bucket is not held
|
||||
other()
|
||||
unlock()
|
||||
select {
|
||||
case <-got:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("the sweep did not take the node's bucket once the send was done")
|
||||
}
|
||||
}
|
||||
|
||||
// A put that fails send after send raises a condition; the first failure does not (the grant it waits for travels in that very
|
||||
// declaration), and a put that works clears it (1696/AC10).
|
||||
func TestAPutThatFailsSendAfterSendRaisesAConditionAndClearsWhenItWorks(t *testing.T) { // 1696/AC10
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||
Teller: &conditions.Told{}, Now: func() time.Time { return time.Date(2026, 10, 21, 14, 0, 0, 0, time.UTC) }})
|
||||
t.Cleanup(func() { k.Close(t.Context()) })
|
||||
was := conditionsFrom
|
||||
conditionsFrom = k
|
||||
t.Cleanup(func() { conditionsFrom = was })
|
||||
key := filesNotPutObservation("cond-node", "").Key()
|
||||
open := func() bool {
|
||||
_, found, err := conditions.ReadOne(t.Context(), store, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return found
|
||||
}
|
||||
boom := errors.New("permissions violation")
|
||||
notePut(t.Context(), "cond-node", boom)
|
||||
if open() {
|
||||
t.Fatal("the first failure raised the condition")
|
||||
}
|
||||
notePut(t.Context(), "cond-node", boom)
|
||||
if !open() {
|
||||
t.Fatal("the second failure in a row did not raise the condition")
|
||||
}
|
||||
notePut(t.Context(), "cond-node", nil)
|
||||
if open() {
|
||||
t.Fatal("a put that worked did not clear the condition")
|
||||
}
|
||||
notePut(t.Context(), "cond-node", boom)
|
||||
if open() {
|
||||
t.Fatal("the count did not start again after a put that worked")
|
||||
}
|
||||
}
|
||||
|
||||
// aBusThatSays is a bus that keeps the declarations published to it and, at each, asks whether the node's bucket is held.
|
||||
type aBusThatSays struct {
|
||||
link.Bus
|
||||
published chan string
|
||||
heldAtPublish func() bool
|
||||
}
|
||||
|
||||
func (b *aBusThatSays) PublishDeclaration(_ context.Context, node string, _ []byte) error {
|
||||
if b.heldAtPublish != nil && !b.heldAtPublish() {
|
||||
return errors.New("published while the node's bucket was not held")
|
||||
}
|
||||
b.published <- node
|
||||
return errors.New("stop here: nothing is recorded")
|
||||
}
|
||||
|
||||
type aSignerOfZeros struct{}
|
||||
|
||||
func (aSignerOfZeros) Sign(context.Context, []byte) ([]byte, error) { return make([]byte, 64), nil }
|
||||
|
||||
// whileHeld says whether the node's lock is held by someone right now, without keeping it.
|
||||
func whileHeld(node string) bool {
|
||||
got := make(chan struct{})
|
||||
go func() {
|
||||
defer lockFilesOf(node)()
|
||||
close(got)
|
||||
}()
|
||||
select {
|
||||
case <-got:
|
||||
return false
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
// held: the goroutine stays waiting and takes and releases the lock once it is let go.
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// 1696/AC9: a send holds the node's lock from the put to the publication, so the sweep never removes an object between them. The
|
||||
// test goes through overTheBus.declare: while another holds the lock the declaration is not published, and at the publication the
|
||||
// lock is held.
|
||||
func TestASendHoldsTheNodesLockFromThePutToThePublication(t *testing.T) { // 1696/AC9
|
||||
const node = "lock-send-node"
|
||||
bus := &aBusThatSays{published: make(chan string, 1)}
|
||||
bus.heldAtPublish = func() bool { return whileHeld(node) }
|
||||
send := overTheBus{server: link.SendingOn(bus), signer: aSignerOfZeros{}}
|
||||
release := lockFilesOf(node) // a sweep is judging this node's bucket
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := send.declare(t.Context(), readyNode{node: node}, []byte(`{}`))
|
||||
done <- err
|
||||
}()
|
||||
select {
|
||||
case n := <-bus.published:
|
||||
t.Fatalf("%s was sent its declaration while the sweep held the node's bucket", n)
|
||||
case <-time.After(150 * time.Millisecond):
|
||||
}
|
||||
release()
|
||||
select {
|
||||
case <-bus.published:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the send did not go on once the sweep let go")
|
||||
}
|
||||
if err := <-done; err == nil || !strings.Contains(err.Error(), "nothing is recorded") {
|
||||
t.Fatalf("the send did not reach the publication (a lock held at it): %v", err)
|
||||
}
|
||||
}
|
||||
@@ -1,96 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A declaration is one bus message, and the bus's largest message is a cliff** (novox/hq issue 1689). On 2026-10-21
|
||||
// the control node's declaration passed it by 2 KB and nothing reached that node until two assignments were undone.
|
||||
// So the size is measured at every send: a warning condition from 75% of the limit (ADR 0486 §8, issue 1698: it was 90% while every file travelled inline), urgent over it, cleared when
|
||||
// the declaration is back under 75%. The limit is the connection's own, not a constant.
|
||||
|
||||
// kindDeclarationNearLimit is a declaration at or near the bus's largest message.
|
||||
const kindDeclarationNearLimit = "declaration-near-limit"
|
||||
|
||||
// declarationRoomSource is the condition's source: the sends.
|
||||
const declarationRoomSource = "sends"
|
||||
|
||||
func declarationRoomObservation(node string, size, limit int64) conditions.Observation {
|
||||
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: kindDeclarationNearLimit,
|
||||
Token: kindDeclarationNearLimit, Machine: node, Severity: conditions.Warning, Source: declarationRoomSource}
|
||||
if size > limit {
|
||||
o.Severity = conditions.Urgent
|
||||
}
|
||||
o.Summary = fmt.Sprintf("%s's declaration is %d bytes on the wire, %d%% of the bus's largest message of %d bytes",
|
||||
node, size, size*100/limit, limit)
|
||||
if size > limit {
|
||||
o.Summary += ": over it, so it cannot be sent"
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
func init() {
|
||||
link.DeclarationMeasured = func(ctx context.Context, node string, size, limit int64) {
|
||||
settleDeclarationRoom(ctx, conditionsFrom, node, size, limit)
|
||||
}
|
||||
}
|
||||
|
||||
// settleDeclarationRoom raises or clears the condition of a declaration's size. A nil keeper (a tool, not the
|
||||
// serving controller) says nothing.
|
||||
func settleDeclarationRoom(ctx context.Context, keeper *conditions.Keeper, node string, size, limit int64) {
|
||||
if keeper == nil || limit <= 0 {
|
||||
return
|
||||
}
|
||||
obs := declarationRoomObservation(node, size, limit)
|
||||
if float64(size) < link.NearTheLimit*float64(limit) {
|
||||
if _, err := keeper.Clear(ctx, obs.Key(), fmt.Sprintf("%s's declaration is %d bytes, under the limit's margin", node, size)); err != nil {
|
||||
fmt.Printf("the condition %s could not be cleared: %v\n", obs.Key(), err)
|
||||
}
|
||||
return
|
||||
}
|
||||
fmt.Printf("NEAR THE LIMIT: %s\n", obs.Summary)
|
||||
if _, err := keeper.Observe(ctx, obs); err != nil {
|
||||
fmt.Printf("the condition %s could not be raised: %v\n", obs.Key(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// compactKept writes the vault's export compact. The composition (the SDK's keptFile) writes it indented, and its 334
|
||||
// sealed secrets were 164,763 bytes of it against 138,474 compact: a fifth of the declaration's room, in whitespace,
|
||||
// base64-encoded again on the wire. Every reader parses JSON, so the spacing is nobody's. A resource that is not
|
||||
// the export, or whose content is not JSON, is left as it is. The resources are copied, never changed in place.
|
||||
func compactKept(resources []map[string]any) []map[string]any {
|
||||
out := make([]map[string]any, len(resources))
|
||||
for i, r := range resources {
|
||||
out[i] = r
|
||||
id, _ := r["id"].(string)
|
||||
path, _ := r["path"].(string)
|
||||
content, ok := r["content"].(string)
|
||||
if !ok || (id != "kept" && !strings.HasSuffix(id, ".kept")) || !strings.HasSuffix(path, "/export.json") {
|
||||
continue
|
||||
}
|
||||
var dst bytes.Buffer
|
||||
if json.Compact(&dst, []byte(content)) != nil {
|
||||
continue
|
||||
}
|
||||
if strings.HasSuffix(content, "\n") {
|
||||
dst.WriteByte('\n')
|
||||
}
|
||||
if dst.String() == content {
|
||||
continue
|
||||
}
|
||||
c := make(map[string]any, len(r))
|
||||
for k, v := range r {
|
||||
c[k] = v
|
||||
}
|
||||
c["content"] = dst.String()
|
||||
out[i] = c
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1,71 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// The vault's export is sent compact (issue 1689): same document, no indentation, one rewrite.
|
||||
func TestTheVaultsExportIsSentCompactAndNothingElseIsTouched(t *testing.T) {
|
||||
indented := "{\n \"export\": 1,\n \"kept\": [\n {\n \"name\": \"a\"\n }\n ]\n}\n"
|
||||
other := "{\n \"x\": 1\n}\n"
|
||||
in := []map[string]any{
|
||||
{"id": "mesh-vault.kept", "type": "file", "path": "/srv/vault/export.json", "mode": "0600", "content": indented},
|
||||
{"id": "mesh-vault.config", "type": "file", "path": "/srv/vault/config.json", "content": other},
|
||||
{"id": "other.kept", "type": "file", "path": "/srv/other/notes.json", "content": other},
|
||||
}
|
||||
out := compactKept(in)
|
||||
want := "{\"export\":1,\"kept\":[{\"name\":\"a\"}]}\n"
|
||||
if out[0]["content"] != want || out[0]["mode"] != "0600" || out[0]["path"] != "/srv/vault/export.json" {
|
||||
t.Fatalf("the export is %q (%v), expected %q with its mode and path kept", out[0]["content"], out[0], want)
|
||||
}
|
||||
if out[1]["content"] != other || out[2]["content"] != other {
|
||||
t.Fatalf("a file that is not the export was rewritten: %v %v", out[1], out[2])
|
||||
}
|
||||
if in[0]["content"] != indented {
|
||||
t.Fatalf("the composed resource was changed in place")
|
||||
}
|
||||
if again := compactKept(out); again[0]["content"] != want {
|
||||
t.Fatalf("compacting twice changed it: %q", again[0]["content"])
|
||||
}
|
||||
}
|
||||
|
||||
// From 75% of the bus's largest message (ADR 0486 §8, issue 1698) the declaration is a warning; over it, urgent; back under 75%, cleared.
|
||||
func TestADeclarationNearTheBusLimitRaisesAConditionAndClearsWhenSmaller(t *testing.T) {
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||
Teller: &conditions.Told{}, Now: func() time.Time { return time.Date(2026, 10, 21, 14, 0, 0, 0, time.UTC) }})
|
||||
t.Cleanup(func() { k.Close(t.Context()) })
|
||||
const limit = 1000
|
||||
key := declarationRoomObservation("novox", 0, limit).Key()
|
||||
read := func() (conditions.Condition, bool) {
|
||||
t.Helper()
|
||||
c, found, err := conditions.ReadOne(t.Context(), store, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c, found
|
||||
}
|
||||
|
||||
settleDeclarationRoom(t.Context(), k, "novox", 749, limit)
|
||||
if _, found := read(); found {
|
||||
t.Fatal("a declaration at 74.9% raised a condition")
|
||||
}
|
||||
settleDeclarationRoom(t.Context(), k, "novox", 750, limit)
|
||||
c, found := read()
|
||||
if !found || c.Severity != conditions.Warning || !strings.Contains(c.Summary, "750 bytes") ||
|
||||
!strings.Contains(c.Summary, "1000") || c.Subject.Machine != "novox" {
|
||||
t.Fatalf("at 75%% the condition is %v %+v", found, c)
|
||||
}
|
||||
settleDeclarationRoom(t.Context(), k, "novox", 1001, limit)
|
||||
if c, _ := read(); c.Severity != conditions.Urgent {
|
||||
t.Fatalf("over the limit the condition is %q, expected urgent", c.Severity)
|
||||
}
|
||||
settleDeclarationRoom(t.Context(), k, "novox", 700, limit)
|
||||
if _, found := read(); found {
|
||||
t.Fatal("the condition stayed after the declaration fell under 75%")
|
||||
}
|
||||
}
|
||||
@@ -1,273 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **An object is removed a day after no node's latest declaration names it** (novox/hq ADR 0486 §3, issue 1696), so a node that is
|
||||
// asleep and receives its latest declaration later still finds every object that declaration names. What each node's latest
|
||||
// declaration names is read from the bus itself, the last message of that node's subject of the stream NODES: nothing else
|
||||
// is kept of it. When an object is first found named by none, the time is written in its own metadata (declarationUnnamed);
|
||||
// found named again, that is erased; found unnamed for a day, it is removed. Each node has a bucket of its own, judged against that
|
||||
// node's latest declaration alone. Objects are immutable and regenerable from the
|
||||
// controller's state, so a removal loses nothing.
|
||||
|
||||
// declarationUnnamed is the key of an object's metadata holding when it was first found named by no node's latest declaration.
|
||||
const declarationUnnamed = "unnamed-since"
|
||||
|
||||
// sweepEvery is how often the bucket is judged: the removal is exact to within this.
|
||||
const sweepEvery = time.Hour
|
||||
|
||||
// bucketObject is one object of the bucket as the sweep sees it.
|
||||
type bucketObject struct {
|
||||
Name string
|
||||
// UnnamedSince is when it was first found named by none, zero when it was named at the last look.
|
||||
UnnamedSince time.Time
|
||||
}
|
||||
|
||||
// sweptBucket is what the sweep needs of the bucket.
|
||||
type sweptBucket interface {
|
||||
Objects(ctx context.Context) ([]bucketObject, error)
|
||||
MarkUnnamed(ctx context.Context, name string, since time.Time) error
|
||||
ClearUnnamed(ctx context.Context, name string) error
|
||||
// Delete removes the object if it still carries the mark it was judged by, and says whether it did: a send that put it again
|
||||
// since has erased the mark, and the object is left.
|
||||
Delete(ctx context.Context, name string, markedAt time.Time) (bool, error)
|
||||
}
|
||||
|
||||
// filesSweep says what a sweep did.
|
||||
type filesSweep struct{ Kept, Marked, Cleared, Removed int }
|
||||
|
||||
// sweepDeclarationFiles judges every object against the names of the latest declarations: named ones are kept (and a
|
||||
// mark of being unnamed is erased), unnamed ones are marked at the first look and removed once the mark is a day old.
|
||||
func sweepDeclarationFiles(ctx context.Context, bucket sweptBucket, named map[string]bool, now time.Time) (filesSweep, error) {
|
||||
var r filesSweep
|
||||
objects, err := bucket.Objects(ctx)
|
||||
if err != nil {
|
||||
return r, err
|
||||
}
|
||||
sort.Slice(objects, func(i, j int) bool { return objects[i].Name < objects[j].Name })
|
||||
for _, o := range objects {
|
||||
switch {
|
||||
case named[o.Name]:
|
||||
r.Kept++
|
||||
if !o.UnnamedSince.IsZero() {
|
||||
if err := bucket.ClearUnnamed(ctx, o.Name); err != nil {
|
||||
return r, fmt.Errorf("the object %s: %w", o.Name[:12], err)
|
||||
}
|
||||
r.Cleared++
|
||||
}
|
||||
case o.UnnamedSince.IsZero():
|
||||
if err := bucket.MarkUnnamed(ctx, o.Name, now); err != nil {
|
||||
return r, fmt.Errorf("the object %s: %w", o.Name[:12], err)
|
||||
}
|
||||
r.Marked++
|
||||
case now.Sub(o.UnnamedSince) >= broker.DeclarationFilesKeptFor:
|
||||
gone, err := bucket.Delete(ctx, o.Name, o.UnnamedSince)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("the object %s: %w", o.Name[:12], err)
|
||||
}
|
||||
if gone {
|
||||
r.Removed++
|
||||
} else {
|
||||
r.Kept++
|
||||
}
|
||||
default:
|
||||
r.Kept++
|
||||
}
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// namedByLatest is the name of every object this latest declaration of a node names. A body that cannot be read is an error, never
|
||||
// "names none": an object the node still needs would be marked for removal.
|
||||
func namedByLatest(node string, signed []byte) (map[string]bool, error) {
|
||||
var s link.Signed
|
||||
if err := json.Unmarshal(signed, &s); err != nil {
|
||||
return nil, fmt.Errorf("%s's latest declaration is not readable: %w", node, err)
|
||||
}
|
||||
names, err := namedObjects(s.Declaration)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s's latest declaration is not readable: %w", node, err)
|
||||
}
|
||||
named := make(map[string]bool, len(names))
|
||||
for name := range names {
|
||||
named[name] = true
|
||||
}
|
||||
return named, nil
|
||||
}
|
||||
|
||||
// latestDeclarations are the last signed declarations of the nodes, by node, as the stream NODES holds them. A node with none has
|
||||
// no entry.
|
||||
func latestDeclarations(ctx context.Context, conn *nats.Conn, nodes []string) (map[string][]byte, error) {
|
||||
js, err := conn.JetStream(nats.Context(ctx))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string][]byte{}
|
||||
for _, node := range nodes {
|
||||
msg, err := js.GetLastMsg("NODES", link.DeclareSubject(node), nats.Context(ctx))
|
||||
if errors.Is(err, nats.ErrMsgNotFound) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the latest declaration of %s: %w", node, err)
|
||||
}
|
||||
out[node] = msg.Data
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// busSweptBucket is the bucket on the controller's connection.
|
||||
type busSweptBucket struct{ store jetstream.ObjectStore }
|
||||
|
||||
func (b busSweptBucket) Objects(ctx context.Context) ([]bucketObject, error) {
|
||||
infos, err := b.store.List(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoObjectsFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]bucketObject, 0, len(infos))
|
||||
for _, info := range infos {
|
||||
o := bucketObject{Name: info.Name}
|
||||
if at, err := time.Parse(time.RFC3339, info.Metadata[declarationUnnamed]); err == nil {
|
||||
o.UnnamedSince = at
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (b busSweptBucket) meta(ctx context.Context, name string, change func(map[string]string)) error {
|
||||
info, err := b.store.GetInfo(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
meta := info.ObjectMeta
|
||||
kept := map[string]string{}
|
||||
for k, v := range meta.Metadata {
|
||||
kept[k] = v
|
||||
}
|
||||
change(kept)
|
||||
meta.Metadata = kept
|
||||
return b.store.UpdateMeta(ctx, name, meta)
|
||||
}
|
||||
|
||||
func (b busSweptBucket) MarkUnnamed(ctx context.Context, name string, since time.Time) error {
|
||||
return b.meta(ctx, name, func(m map[string]string) { m[declarationUnnamed] = since.UTC().Format(time.RFC3339) })
|
||||
}
|
||||
|
||||
func (b busSweptBucket) ClearUnnamed(ctx context.Context, name string) error {
|
||||
return b.meta(ctx, name, func(m map[string]string) { delete(m, declarationUnnamed) })
|
||||
}
|
||||
|
||||
func (b busSweptBucket) Delete(ctx context.Context, name string, markedAt time.Time) (bool, error) {
|
||||
info, err := b.store.GetInfo(ctx, name)
|
||||
if errors.Is(err, jetstream.ErrObjectNotFound) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if at, perr := time.Parse(time.RFC3339, info.Metadata[declarationUnnamed]); perr != nil || !at.Equal(markedAt) {
|
||||
return false, nil
|
||||
}
|
||||
return true, b.store.Delete(ctx, name)
|
||||
}
|
||||
|
||||
// sweepingDeclarationFiles judges the bucket every sweepEvery for as long as this controller serves and holds the lease.
|
||||
func sweepingDeclarationFiles(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn) {
|
||||
tick := time.NewTicker(sweepEvery)
|
||||
defer tick.Stop()
|
||||
failing := ""
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
if epoch, err := theLease.epoch(ctx); err != nil || epoch == 0 {
|
||||
continue
|
||||
}
|
||||
why := sweepOnce(ctx, inv, conn)
|
||||
if why != failing {
|
||||
if why != "" {
|
||||
fmt.Printf("the bucket of declaration files cannot be swept: %s\n", why)
|
||||
} else {
|
||||
fmt.Printf("the bucket of declaration files is swept again\n")
|
||||
}
|
||||
failing = why
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sweepOnce judges the bucket of every node against that node's latest declaration, one bucket at a time under the node's lock, with
|
||||
// the declaration read again under it: a send that put an object and published its declaration is seen, and one that has not yet
|
||||
// is waiting for the lock.
|
||||
func sweepOnce(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn) string {
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return fmt.Sprintf("the nodes cannot be read: %v", err)
|
||||
}
|
||||
var problems []string
|
||||
var total filesSweep
|
||||
for _, n := range nodes {
|
||||
r, err := sweepNode(ctx, conn, n.Name)
|
||||
total.Kept, total.Marked, total.Cleared, total.Removed = total.Kept+r.Kept, total.Marked+r.Marked, total.Cleared+r.Cleared, total.Removed+r.Removed
|
||||
if err != nil {
|
||||
problems = append(problems, fmt.Sprintf("%s: %v", n.Name, err))
|
||||
}
|
||||
}
|
||||
if total.Removed+total.Marked+total.Cleared > 0 {
|
||||
fmt.Printf("the buckets of declaration files: %d object(s) kept, %d found unnamed, %d named again, %d removed\n",
|
||||
total.Kept, total.Marked, total.Cleared, total.Removed)
|
||||
}
|
||||
return strings.Join(problems, "; ")
|
||||
}
|
||||
|
||||
func sweepNode(ctx context.Context, conn *nats.Conn, node string) (filesSweep, error) {
|
||||
working, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
||||
defer cancel()
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return filesSweep{}, err
|
||||
}
|
||||
bucket, err := broker.DeclarationFilesBucketOf(node)
|
||||
if err != nil {
|
||||
return filesSweep{}, fmt.Errorf("the bucket of node %q: %w", node, err)
|
||||
}
|
||||
store, err := js.ObjectStore(working, bucket)
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
return filesSweep{}, nil // a node never sent a reference has no bucket yet
|
||||
}
|
||||
if err != nil {
|
||||
return filesSweep{}, fmt.Errorf("the bucket cannot be read: %w", err)
|
||||
}
|
||||
defer lockFilesOf(node)()
|
||||
latest, err := latestDeclarations(working, conn, []string{node})
|
||||
if err != nil {
|
||||
return filesSweep{}, err
|
||||
}
|
||||
named := map[string]bool{}
|
||||
if body, ok := latest[node]; ok {
|
||||
if named, err = namedByLatest(node, body); err != nil {
|
||||
return filesSweep{}, err
|
||||
}
|
||||
}
|
||||
return sweepDeclarationFiles(working, busSweptBucket{store}, named, time.Now())
|
||||
}
|
||||
@@ -1,89 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// 1840/AC5 (ADR 0507 §5): the source of the declared-packages mirror composes each node and names, for a package two modules
|
||||
// declare and one declared absent, the modules that declare it; the same records give the same bytes on a second pass, so
|
||||
// the mirror writes nothing; a node with no package has a key that says so.
|
||||
func TestTheDeclaredPackagesMirrorNamesWhatEachNodesModulesDeclare(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "viewer", Version: "1", Resources: []map[string]any{
|
||||
{"id": "pkg-curl", "type": "package", "package": "curl"},
|
||||
{"id": "pkg-nano", "type": "package", "package": "nano", "absent": true}}})
|
||||
register(t, open, catalogue.Manifest{Module: "fetcher", Version: "1", Resources: []map[string]any{
|
||||
{"id": "pkg-curl", "type": "package", "package": "curl"}}})
|
||||
for _, module := range []string{"viewer", "fetcher"} {
|
||||
if _, err := assign(ctx, open, "anchor", module); err != nil {
|
||||
t.Fatalf("assign anchor %s: %v", module, err)
|
||||
}
|
||||
}
|
||||
source := declaredPackagesMirror(open)
|
||||
first, err := source(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got inventory.DeclaredPackagesMirrored
|
||||
if err := json.Unmarshal(first["anchor"], &got); err != nil {
|
||||
t.Fatalf("anchor has no value: %v", err)
|
||||
}
|
||||
// Every node also runs the mesh's own modules, which declare packages of their own: curl and nano are looked
|
||||
// up by name, and no package names viewer or fetcher but those they declare.
|
||||
byName := map[string]inventory.PackageDeclared{}
|
||||
for _, p := range got.Packages {
|
||||
byName[p.Package] = p
|
||||
}
|
||||
same := func(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
for _, p := range []inventory.PackageDeclared{{Package: "curl", Modules: []string{"fetcher", "viewer"}},
|
||||
{Package: "nano", Modules: []string{"viewer"}, Absent: true}} {
|
||||
if g, ok := byName[p.Package]; !ok || g.Absent != p.Absent || !same(g.Modules, p.Modules) {
|
||||
t.Errorf("anchor declares %s as %+v, want %+v (all: %+v)", p.Package, g, p, got.Packages)
|
||||
}
|
||||
}
|
||||
for _, p := range got.Packages {
|
||||
if p.Package == "curl" || p.Package == "nano" {
|
||||
continue
|
||||
}
|
||||
for _, m := range p.Modules {
|
||||
if m == "viewer" || m == "fetcher" {
|
||||
t.Errorf("%s is said to be declared by %s, which does not declare it", p.Package, m)
|
||||
}
|
||||
}
|
||||
}
|
||||
var laptop inventory.DeclaredPackagesMirrored
|
||||
if err := json.Unmarshal(first["laptop"], &laptop); err != nil || laptop.Node != "laptop" || laptop.Packages == nil {
|
||||
t.Errorf("a node assigned neither module has no value of its own: %s (%v)", first["laptop"], err)
|
||||
}
|
||||
for _, p := range laptop.Packages {
|
||||
if p.Package == "curl" || p.Package == "nano" {
|
||||
t.Errorf("laptop is assigned neither module and declares %+v", p)
|
||||
}
|
||||
}
|
||||
second, err := source(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for node, value := range first {
|
||||
if string(second[node]) != string(value) {
|
||||
t.Errorf("the value of %s changed between two passes over the same records:\n%s\n%s", node, value, second[node])
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1177,12 +1177,6 @@ func askGroupCheck(ctx context.Context, open *stores, group string, members []or
|
||||
return "", err
|
||||
}
|
||||
request.Check.Group = group
|
||||
// **The values held for the keys the members introduce are told and carried** (novox/hq ADR 0502, issue 1826).
|
||||
var pulls []pullAt
|
||||
for i, s := range changeSourcesOf(members, entries, reach) {
|
||||
pulls = append(pulls, pullAt{changeSource: s, Number: members[i].pull().Number})
|
||||
}
|
||||
withHeldConfig(ctx, os.Stderr, &request, pulls, true)
|
||||
// A group's check is queued as ready when any member's pull request is marked ready.
|
||||
request.Check.Ready = slices.ContainsFunc(members, func(m orderMember) bool { return m.Ready })
|
||||
world, err := restOfTheMesh(ctx, open, shelfOf(entries), "")
|
||||
@@ -1210,7 +1204,7 @@ func askGroupCheck(ctx context.Context, open *stores, group string, members []or
|
||||
return "", fmt.Errorf("%s cannot be cloned for the group's check: %w", m.ID, err)
|
||||
}
|
||||
request.Check.Members = append(request.Check.Members, link.GroupMember{Owner: mp.Owner, Repo: mp.Repo,
|
||||
Number: mp.Number, Repository: url, Ref: mp.Commit, Paths: mp.Paths, Base: mp.Base, Trunk: mp.BaseSHA})
|
||||
Number: mp.Number, Repository: url, Ref: mp.Commit, Paths: mp.Paths, Base: mp.Base})
|
||||
}
|
||||
seat := buildSeatHeld(ctx)
|
||||
ask, err := askOverOn(seat)
|
||||
|
||||
@@ -58,17 +58,8 @@ type stalledLine struct {
|
||||
// Severity is "urgent" for a group held or half merged and a refused member (novox/hq issues 1275, 1278); empty
|
||||
// or anything else is a warning, as before.
|
||||
Severity string `json:"severity,omitempty"`
|
||||
// Needs is "new-head" for a delivery whose cure is its author's new head (novox/hq ADR 0505 §5), Author that author
|
||||
// as the operator reads it ("the author of feat/x", with the session that pushed it when its pull request names one)
|
||||
// and Branch its branch. A mesh-delivery from before says none, and the words are as before.
|
||||
Needs string `json:"needs,omitempty"`
|
||||
Author string `json:"author,omitempty"`
|
||||
Branch string `json:"branch,omitempty"`
|
||||
}
|
||||
|
||||
// needsNewHead is what a stalled line needs when its cure is its author's new head (ADR 0505 §5).
|
||||
const needsNewHead = "new-head"
|
||||
|
||||
// severity is the condition's severity a stalled line asks: urgent when it says so, a warning otherwise.
|
||||
func (l stalledLine) severity() conditions.Severity {
|
||||
if l.Severity == string(conditions.Urgent) {
|
||||
|
||||
@@ -59,18 +59,6 @@ func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testin
|
||||
}
|
||||
}
|
||||
|
||||
// heardOf has this mesh's own view of the plans take in the plan as the bucket holds it. A verb of the
|
||||
// controller (deliveryCommand, plansCommand) opens an inventory of its own and writes the plan there; this
|
||||
// mesh's inventory learns of the write from the bucket's watch, a moment later, and a loaded machine makes the
|
||||
// moment long enough for advanceHeld to read the plan as it was, still waiting for its word, and ask nothing
|
||||
// (issue 1399). Reading the plan from the bucket puts it into the view at once, with no wait to bound.
|
||||
heardOf := func(id string) {
|
||||
t.Helper()
|
||||
if _, err := inv.PlanByID(ctx, id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// No holder on record: the merge starts its walk, as before.
|
||||
p := merge("c1aaaaaaaa", "modules/app/index.ts")
|
||||
if p.Waiting() || len(*asked) != 1 {
|
||||
@@ -102,7 +90,6 @@ func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testin
|
||||
if err := deliveryCommand(ctx, []string{"go", p.ID}); err == nil {
|
||||
t.Fatal("a walk was let go twice")
|
||||
}
|
||||
heardOf(p.ID)
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) != 2 {
|
||||
t.Fatalf("the word did not start the walk: %v", *asked)
|
||||
@@ -132,7 +119,6 @@ func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testin
|
||||
if err := plansCommand(ctx, []string{"go", p.ID, "--why", "mesh-delivery is down"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
heardOf(p.ID)
|
||||
advanceHeld(ctx, open)
|
||||
got, _ = inv.PlanByID(ctx, p.ID)
|
||||
if got.Waiting() || !strings.HasPrefix(got.Delivery.By, "a person") || len(*asked) < 4 {
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -103,7 +102,7 @@ func (f following) resumeFirstBuilds(ctx context.Context, d deliveryMerge) error
|
||||
}
|
||||
for _, w := range waiting {
|
||||
k := announcedOf(w)
|
||||
if !k.FirstBuild || len(k.Held) > 0 || !strings.EqualFold(k.Owner+"/"+k.Repo, d.Owner+"/"+d.Repo) || k.Number != d.Number ||
|
||||
if !k.FirstBuild || !strings.EqualFold(k.Owner+"/"+k.Repo, d.Owner+"/"+d.Repo) || k.Number != d.Number ||
|
||||
k.HeadSHA != d.HeadSHA {
|
||||
continue
|
||||
}
|
||||
@@ -173,122 +172,12 @@ func firstBuildsOf(ctx context.Context, inv catalogueReader, m link.SourceMoved,
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
// heldFirstBuild says a first build was refused because its manifest uses a word the running build-agent does not read,
|
||||
// and its merge is kept waiting for a build-agent that reads it (novox/hq ADR 0482 §5, issue 1173).
|
||||
type heldFirstBuild struct {
|
||||
Module string `json:"module"`
|
||||
Path string `json:"path"`
|
||||
Word string `json:"word"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// judgeRefusedFirstBuild is the hold a first build's outcome puts its merge under: every module the merge holds already,
|
||||
// and this one, unless it is among them; nil unless the build failed on a word of its manifest the running build-agent
|
||||
// does not know, and was asked for a first merge, answered by the record of merges no plan answers or put back to waiting
|
||||
// by an earlier hold (a second refusal in one merge is held too, review of issue 1173).
|
||||
func judgeRefusedFirstBuild(result link.BuildResult, req inventory.BuildRequest, merge inventory.BatchedMerge) []heldFirstBuild {
|
||||
word := refusedWord(result.Failed)
|
||||
if word == "" || req.For != "merge" {
|
||||
return nil
|
||||
}
|
||||
var held []heldFirstBuild
|
||||
switch k := announcedOf(merge); {
|
||||
case merge.Plan == inventory.WaitingRecord:
|
||||
// Answered when its builds were asked: a hold from before (a build asked again) is over, and starts anew.
|
||||
case merge.Plan == "" && len(k.Held) > 0:
|
||||
held = k.Held
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
this := heldFirstBuild{Module: req.Name(), Path: strings.Trim(req.Path, "/"), Word: word, At: time.Now().UTC()}
|
||||
for i, h := range held {
|
||||
if strings.Trim(h.Path, "/") == this.Path {
|
||||
out := append([]heldFirstBuild(nil), held...)
|
||||
out[i] = this
|
||||
return out
|
||||
}
|
||||
}
|
||||
return append(append([]heldFirstBuild(nil), held...), this)
|
||||
}
|
||||
|
||||
// heldDirsOf is the directories a held merge asks again: its held modules' own, each once, "." for the repository's root.
|
||||
func heldDirsOf(k keptMerge) []string {
|
||||
var dirs []string
|
||||
seen := map[string]bool{}
|
||||
for _, h := range k.Held {
|
||||
dir := strings.Trim(h.Path, "/")
|
||||
if dir == "" {
|
||||
dir = "."
|
||||
}
|
||||
if !seen[dir] {
|
||||
seen[dir] = true
|
||||
dirs = append(dirs, dir)
|
||||
}
|
||||
}
|
||||
return dirs
|
||||
}
|
||||
|
||||
// unknownWord is how the strict reading of a manifest says a word is not one it knows: encoding/json's own words, which
|
||||
// the build-agent's answer carries whole. The build's answer has no typed field for it (Failed is the builder's words).
|
||||
var unknownWord = regexp.MustCompile(`json: unknown field "([^"]+)"`)
|
||||
|
||||
// refusedWord is the word a build's failure says the reader did not know; empty when it says none.
|
||||
func refusedWord(failed string) string {
|
||||
if m := unknownWord.FindStringSubmatch(failed); m != nil {
|
||||
return m[1]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// holdRefusedFirstBuild keeps a first build's merge waiting when the build was refused for a word the running build-agent
|
||||
// does not read (novox/hq ADR 0482 §5, issue 1173): no failed build is recorded, nothing expires the merge, and it is
|
||||
// named by a condition until a delivery of build-agent asks the build again. Answers whether the outcome was dealt with.
|
||||
func holdRefusedFirstBuild(ctx context.Context, inv *inventory.Inventory, result link.BuildResult) (bool, error) {
|
||||
if result.Failed == "" || refusedWord(result.Failed) == "" {
|
||||
return false, nil
|
||||
}
|
||||
req, found, err := inv.BuildRequestByID(ctx, result.ID)
|
||||
if err != nil || !found {
|
||||
return false, err
|
||||
}
|
||||
merge, found, err := inv.MergeOf(ctx, req.Repository, req.Commit)
|
||||
if err != nil || !found {
|
||||
return false, err
|
||||
}
|
||||
held := judgeRefusedFirstBuild(result, req, merge)
|
||||
if len(held) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
k := announcedOf(merge)
|
||||
k.Held = held
|
||||
event, err := json.Marshal(k)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := inv.HoldMerge(ctx, merge.Repository, merge.Commit, event); err != nil {
|
||||
return false, err
|
||||
}
|
||||
this := held[len(held)-1]
|
||||
for _, h := range held {
|
||||
if h.Path == strings.Trim(req.Path, "/") {
|
||||
this = h
|
||||
}
|
||||
}
|
||||
fmt.Printf("%s: the first build of %s was refused for the word %q, which the running build-agent does not read; the "+
|
||||
"merge %.8s of %s is kept waiting for a build-agent that reads it (holding %s), and the build is asked again when "+
|
||||
"one is delivered\n", result.ID, this.Module, this.Word, merge.Commit, merge.Repository, heldNames(held))
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// probeFirstBuildsID is the probe that reads the first merges waiting for mesh-delivery's word.
|
||||
const probeFirstBuildsID = "D18"
|
||||
|
||||
// kindFirstBuildUnasked is what a first merge waiting too long for mesh-delivery's word raises.
|
||||
const kindFirstBuildUnasked = "first-build-unasked"
|
||||
|
||||
// kindFirstBuildHeld is what a first merge held for a build-agent that reads its manifest raises.
|
||||
const kindFirstBuildHeld = "first-build-held"
|
||||
|
||||
// firstBuildWaitBound is how long a first merge waits for mesh-delivery's announcement of the operator's Approve before
|
||||
// it is said: a bound in the sense of ADR 0426 §3, not a delay. Nothing waits on it: the word's arrival asks the builds
|
||||
// at once, and the merge is said, never dropped, once it is past this (issue 1146).
|
||||
@@ -300,10 +189,6 @@ func firstBuildObservations(waiting []inventory.BatchedMerge, now time.Time) []c
|
||||
var out []conditions.Observation
|
||||
for _, w := range waiting {
|
||||
k := announcedOf(w)
|
||||
if len(k.Held) > 0 {
|
||||
out = append(out, heldFirstBuildObservation(k))
|
||||
continue
|
||||
}
|
||||
if !k.FirstBuild || now.Sub(w.Heard) <= firstBuildWaitBound {
|
||||
continue
|
||||
}
|
||||
@@ -326,37 +211,6 @@ func firstBuildObservations(waiting []inventory.BatchedMerge, now time.Time) []c
|
||||
return out
|
||||
}
|
||||
|
||||
// heldNames is the modules a merge holds, each with the word it waits for: "watchman (key-ttl), view (per-machine)".
|
||||
func heldNames(held []heldFirstBuild) string {
|
||||
names := make([]string, 0, len(held))
|
||||
for _, h := range held {
|
||||
names = append(names, fmt.Sprintf("%s (%q)", h.Module, h.Word))
|
||||
}
|
||||
return strings.Join(names, ", ")
|
||||
}
|
||||
|
||||
// heldFirstBuildObservation names a first merge held for a build-agent that reads its manifests: at once, since nothing
|
||||
// is lost and nothing times out, but the operator can see what the merge waits for. One condition names every module held.
|
||||
func heldFirstBuildObservation(k keptMerge) conditions.Observation {
|
||||
repository := k.Owner + "/" + k.Repo
|
||||
modules := make([]string, 0, len(k.Held))
|
||||
for _, h := range k.Held {
|
||||
modules = append(modules, h.Module)
|
||||
}
|
||||
held := heldNames(k.Held)
|
||||
return conditions.Observation{Scope: conditions.ScopeMerge, ID: fmt.Sprintf("%s_%.8s", repository, k.Commit),
|
||||
Kind: kindFirstBuildHeld, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the merge %.8s of %s adds modules whose manifests use words the running build-agent does not "+
|
||||
"read: %s; the merge waits for a build-agent that reads them", k.Commit, repository, held),
|
||||
Said: fmt.Sprintf("%s at %s waits for a build-agent that reads the words of %s", repository, k.Commit, held),
|
||||
Headline: fmt.Sprintf("The first build of %s waits for a newer build-agent", strings.Join(modules, ", ")),
|
||||
Explanation: fmt.Sprintf("The merge %.8s into %s adds modules whose manifests use words the build-agent now "+
|
||||
"running does not know: %s. The merge is kept, not failed: their first builds are asked again when a delivery "+
|
||||
"of build-agent ends.", k.Commit, repository, held),
|
||||
Resolved: "a delivery of build-agent ends and the first builds register the modules, or a hand `build` does",
|
||||
}
|
||||
}
|
||||
|
||||
// probeFirstBuilds is D18: no first merge waits past its bound for mesh-delivery's word unsaid.
|
||||
func probeFirstBuilds(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
waiting, err := d.open.inventory.WaitingMerges(ctx)
|
||||
@@ -365,61 +219,3 @@ func probeFirstBuilds(ctx context.Context, d *doctor) ([]conditions.Observation,
|
||||
}
|
||||
return firstBuildObservations(waiting, time.Now().UTC()), nil
|
||||
}
|
||||
|
||||
// askHeldFirstBuildsAgain asks again the first build of each first merge held for a build-agent that reads its manifest,
|
||||
// when the plan that ended delivered build-agent (novox/hq issue 1173): once for each end, since the ask answers the merge
|
||||
// and a refusal that comes again holds it anew. Answers how many merges it asked for.
|
||||
func askHeldFirstBuildsAgain(ctx context.Context, open *stores, p inventory.Plan) int {
|
||||
if !deliversBuildAgent(p) {
|
||||
return 0
|
||||
}
|
||||
actingOnMerges.Lock()
|
||||
defer actingOnMerges.Unlock()
|
||||
inv := open.inventory
|
||||
waiting, err := inv.WaitingMerges(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("%s delivered build-agent, and the first merges held for it could not be read: %v\n", p.ID, err)
|
||||
return 0
|
||||
}
|
||||
asked := 0
|
||||
for _, w := range waiting {
|
||||
k := announcedOf(w)
|
||||
if len(k.Held) == 0 {
|
||||
continue
|
||||
}
|
||||
// Its own held modules, not the rule of a first merge: a module of the merge that passed is registered by now,
|
||||
// and the repository no longer counts as one the mesh builds nothing from (review of issue 1173).
|
||||
why, err := approvedAnnounced(ctx, inv, k.Owner, k.Repo, keptMerge{SourceMoved: k.SourceMoved})
|
||||
if err != nil {
|
||||
fmt.Printf("%s delivered build-agent, and the merge %.8s of %s held for it asks no build now (%v): it stays "+
|
||||
"waiting\n", p.ID, w.Commit, w.Repository, err)
|
||||
continue
|
||||
}
|
||||
dirs := heldDirsOf(k)
|
||||
fmt.Printf("%s delivered build-agent: the first builds of %s are asked again\n", p.ID, heldNames(k.Held))
|
||||
paths := make([]string, 0, len(dirs))
|
||||
for _, dir := range dirs {
|
||||
if dir == "." {
|
||||
dir = ""
|
||||
}
|
||||
paths = append(paths, dir)
|
||||
}
|
||||
repository := k.Owner + "/" + k.Repo
|
||||
askBuilds(withFirstBuildWarrant(ctx, repository, paths, why), inv, k.SourceMoved,
|
||||
buildSource{Repository: repository, Seat: gitSeat}, dirs)
|
||||
// Answered: it no longer waits, so this end asks nothing twice; a refusal that comes again holds it anew.
|
||||
if err := inv.AnswerMerge(ctx, w.Repository, w.Commit, inventory.WaitingRecord, false); err != nil {
|
||||
fmt.Printf(" and the merge could not be marked answered: %v\n", err)
|
||||
continue
|
||||
}
|
||||
asked++
|
||||
}
|
||||
return asked
|
||||
}
|
||||
|
||||
// deliversBuildAgent says a delivery plan ended delivered with build-agent built in it: the reader a held first build
|
||||
// waits for is the one now running.
|
||||
func deliversBuildAgent(p inventory.Plan) bool {
|
||||
m := p.Modules["build-agent"]
|
||||
return p.WalkSaid == inventory.WalkEnded && p.Outcome() == inventory.WalkDelivered && m != nil && m.State == "built"
|
||||
}
|
||||
|
||||
@@ -1,221 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// 1173/AC1, 1173/AC3 whole against the store: a first merge's build refused for a word the running build-agent does not
|
||||
// read keeps the merge waiting with no failed build recorded; a delivery plan that does not deliver build-agent asks
|
||||
// nothing; the one that ends delivering it asks the first build again once, and its pass registers the module.
|
||||
func TestAHeldFirstMergeIsAskedAgainWhenABuildAgentIsDelivered(t *testing.T) {
|
||||
const head, commit = "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", "0177ea7a00000000"
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
|
||||
a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": head})
|
||||
wasKept, wasAsk, realAsk := keptDeliveryMerge, routerAskOf, askABuild
|
||||
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||
return a.merge, true, nil
|
||||
}
|
||||
routerAskOf = func(context.Context, catalogueReader, string, string) (*asks.Ask, *asks.Warrant, error) {
|
||||
return &a.ask, &a.warrant, nil
|
||||
}
|
||||
var asked []string
|
||||
askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) {
|
||||
id := "build-" + string(rune('1'+len(asked)))
|
||||
asked = append(asked, id)
|
||||
return id, nil
|
||||
}
|
||||
t.Cleanup(func() { keptDeliveryMerge, routerAskOf, askABuild = wasKept, wasAsk, realAsk })
|
||||
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main", HeadSHA: head, Commit: commit, Number: 7,
|
||||
MergedAt: time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano), Paths: []string{"module.json", "main.go"},
|
||||
ModuleDirs: []string{"."}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(asked) != 1 {
|
||||
t.Fatalf("1173/AC3: the first merge asked %v, want the one first build", asked)
|
||||
}
|
||||
|
||||
// The running build-agent refuses the manifest's word.
|
||||
refused := onTrunk("build-1", "novox/mesh-delivery", "git", "", nil)
|
||||
refused.Failed, refused.Manifest, refused.Commit = refusedKeyTTL, nil, commit
|
||||
if err := (builds{inv: inv, open: open}).Built(ctx, refused); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, recorded, _ := inv.BuildByID(ctx, "build-1"); recorded {
|
||||
t.Fatal("1173/AC1: a refused first build was recorded as a failed build")
|
||||
}
|
||||
waiting, err := inv.WaitingMerges(ctx)
|
||||
if err != nil || len(waiting) != 1 || len(announcedOf(waiting[0]).Held) == 0 {
|
||||
t.Fatalf("1173/AC1: the merges waiting are %+v (%v), want the first merge, held", waiting, err)
|
||||
}
|
||||
if obs := firstBuildObservations(waiting, time.Now()); len(obs) != 1 || obs[0].Kind != kindFirstBuildHeld {
|
||||
t.Fatalf("1173/AC2: the held merge is named by %+v", obs)
|
||||
}
|
||||
|
||||
// A plan that ended without delivering build-agent asks nothing; one that delivered it asks once.
|
||||
other := inventory.Plan{ID: "p1", State: inventory.PlanDone, WalkSaid: inventory.WalkEnded,
|
||||
Modules: map[string]*inventory.PlanModule{"mesh-delivery": {State: "built"}}}
|
||||
if n := askHeldFirstBuildsAgain(ctx, open, other); n != 0 || len(asked) != 1 {
|
||||
t.Fatalf("1173/AC3: a plan that delivered no build-agent asked %d builds again (%v)", n, asked)
|
||||
}
|
||||
delivered := inventory.Plan{ID: "p2", State: inventory.PlanDone, WalkSaid: inventory.WalkEnded,
|
||||
Modules: map[string]*inventory.PlanModule{"build-agent": {State: "built"}}}
|
||||
if n := askHeldFirstBuildsAgain(ctx, open, delivered); n != 1 || len(asked) != 2 {
|
||||
t.Fatalf("1173/AC3: the plan that delivered build-agent asked %d builds again (%v), want one", n, asked)
|
||||
}
|
||||
if n := askHeldFirstBuildsAgain(ctx, open, delivered); n != 0 || len(asked) != 2 {
|
||||
t.Fatalf("1173/AC3: the same end asked again: %d (%v)", n, asked)
|
||||
}
|
||||
|
||||
// Its pass registers the module.
|
||||
pass := onTrunk("build-2", "novox/mesh-delivery", "git", "", map[string]any{"module": "mesh-delivery-watchman", "version": "1"})
|
||||
pass.Commit = commit
|
||||
if err := (builds{inv: inv, open: open}).Built(ctx, pass); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, known := inventoryHolds(t, inv, "mesh-delivery-watchman"); !known {
|
||||
t.Fatal("1173/AC3: the module is not registered after the first build passed")
|
||||
}
|
||||
}
|
||||
|
||||
// aTwoModuleFirstMerge is a first merge into novox/mesh-delivery adding the modules watchman and view, made on the
|
||||
// operator's Approve of its head, whose first builds are asked; asked says the build id asked for each path, in order.
|
||||
func aTwoModuleFirstMerge(t *testing.T, open *stores, commit string) *[][2]string {
|
||||
t.Helper()
|
||||
const head = "b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2"
|
||||
a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@b2b2b2b2b2b2": head})
|
||||
wasKept, wasAsk, realAsk := keptDeliveryMerge, routerAskOf, askABuild
|
||||
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||
return a.merge, true, nil
|
||||
}
|
||||
routerAskOf = func(context.Context, catalogueReader, string, string) (*asks.Ask, *asks.Warrant, error) {
|
||||
return &a.ask, &a.warrant, nil
|
||||
}
|
||||
asked := &[][2]string{}
|
||||
askABuild = func(_ context.Context, _ buildSource, path, _ string) (string, error) {
|
||||
id := fmt.Sprintf("build-%d", len(*asked)+1)
|
||||
*asked = append(*asked, [2]string{path, id})
|
||||
return id, nil
|
||||
}
|
||||
t.Cleanup(func() { keptDeliveryMerge, routerAskOf, askABuild = wasKept, wasAsk, realAsk })
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main", HeadSHA: head, Commit: commit, Number: 7,
|
||||
MergedAt: time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano),
|
||||
Paths: []string{"view/module.json", "view/main.go", "watchman/module.json", "watchman/main.go"},
|
||||
ModuleDirs: []string{"view", "watchman"}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(t.Context(), m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(*asked) != 2 {
|
||||
t.Fatalf("the first merge asked %v, want the two first builds", *asked)
|
||||
}
|
||||
return asked
|
||||
}
|
||||
|
||||
// idOf is the build asked last for path.
|
||||
func idOf(t *testing.T, asked [][2]string, path string) string {
|
||||
t.Helper()
|
||||
for i := len(asked) - 1; i >= 0; i-- {
|
||||
if asked[i][0] == path {
|
||||
return asked[i][1]
|
||||
}
|
||||
}
|
||||
t.Fatalf("no build asked for %s: %v", path, asked)
|
||||
return ""
|
||||
}
|
||||
|
||||
// built hands the controller a build's outcome: refused for the word, or passing as module.
|
||||
func built(t *testing.T, open *stores, id, path, commit, refusedFor, module string) {
|
||||
t.Helper()
|
||||
r := onTrunk(id, "novox/mesh-delivery", "git", path, map[string]any{"module": module, "version": "1"})
|
||||
r.Commit = commit
|
||||
if refusedFor != "" {
|
||||
r.Failed, r.Manifest = `this is not a module manifest: json: unknown field "`+refusedFor+`"`, nil
|
||||
}
|
||||
if err := (builds{inv: open.inventory, open: open}).Built(t.Context(), r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
var buildAgentDelivered = inventory.Plan{ID: "p-ba", State: inventory.PlanDone, WalkSaid: inventory.WalkEnded,
|
||||
Modules: map[string]*inventory.PlanModule{"build-agent": {State: "built"}}}
|
||||
|
||||
// 1173/AC3 (review round 1, item 1): in a first merge of two modules where view passes and watchman is refused, the
|
||||
// held watchman is asked again when build-agent is delivered, though the repository builds view by then.
|
||||
func TestAHeldModuleIsAskedAgainAfterItsSiblingRegistered(t *testing.T) {
|
||||
const commit = "0177ea7b00000000"
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := aTwoModuleFirstMerge(t, open, commit)
|
||||
|
||||
built(t, open, idOf(t, *asked, "view"), "view", commit, "", "mesh-delivery-view")
|
||||
if _, known := inventoryHolds(t, open.inventory, "mesh-delivery-view"); !known {
|
||||
t.Fatal("view is not registered after its first build passed")
|
||||
}
|
||||
refused := idOf(t, *asked, "watchman")
|
||||
built(t, open, refused, "watchman", commit, "key-ttl", "")
|
||||
if _, recorded, _ := open.inventory.BuildByID(ctx, refused); recorded {
|
||||
t.Fatal("1173/AC1: the refused first build of watchman was recorded as a failed build")
|
||||
}
|
||||
waiting, err := open.inventory.WaitingMerges(ctx)
|
||||
if err != nil || len(waiting) != 1 || len(announcedOf(waiting[0]).Held) != 1 {
|
||||
t.Fatalf("1173/AC1: the merges waiting are %+v (%v), want the first merge, holding watchman", waiting, err)
|
||||
}
|
||||
|
||||
if n := askHeldFirstBuildsAgain(ctx, open, buildAgentDelivered); n != 1 || len(*asked) != 3 || (*asked)[2][0] != "watchman" {
|
||||
t.Fatalf("1173/AC3: the delivery of build-agent asked %d merges again (%v), want watchman's first build alone", n, *asked)
|
||||
}
|
||||
built(t, open, idOf(t, *asked, "watchman"), "watchman", commit, "", "mesh-delivery-watchman")
|
||||
if _, known := inventoryHolds(t, open.inventory, "mesh-delivery-watchman"); !known {
|
||||
t.Fatal("1173/AC3: watchman is not registered after the first build asked again passed")
|
||||
}
|
||||
}
|
||||
|
||||
// 1173/AC1, AC2, AC3 (review round 1, item 2): in a first merge of two modules both refused, neither is recorded as a
|
||||
// failed build, the condition names both, and the delivery of build-agent asks both again.
|
||||
func TestTwoRefusedModulesOfOneFirstMergeAreBothHeld(t *testing.T) {
|
||||
const commit = "0177ea7c00000000"
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := aTwoModuleFirstMerge(t, open, commit)
|
||||
|
||||
for _, path := range []string{"watchman", "view"} {
|
||||
id := idOf(t, *asked, path)
|
||||
built(t, open, id, path, commit, "key-ttl", "")
|
||||
if _, recorded, _ := open.inventory.BuildByID(ctx, id); recorded {
|
||||
t.Fatalf("1173/AC1: the refused first build of %s was recorded as a failed build", path)
|
||||
}
|
||||
}
|
||||
waiting, err := open.inventory.WaitingMerges(ctx)
|
||||
if err != nil || len(waiting) != 1 || len(announcedOf(waiting[0]).Held) != 2 {
|
||||
t.Fatalf("1173/AC1: the merges waiting are %+v (%v), want the first merge, holding both", waiting, err)
|
||||
}
|
||||
obs := firstBuildObservations(waiting, time.Now())
|
||||
if len(obs) != 1 || !strings.Contains(obs[0].Summary, "watchman") || !strings.Contains(obs[0].Summary, "view") {
|
||||
t.Fatalf("1173/AC2: the held merge is named by %+v, want one condition naming both", obs)
|
||||
}
|
||||
|
||||
if n := askHeldFirstBuildsAgain(ctx, open, buildAgentDelivered); n != 1 || len(*asked) != 4 {
|
||||
t.Fatalf("1173/AC3: the delivery of build-agent asked %d merges again (%v), want both first builds", n, *asked)
|
||||
}
|
||||
built(t, open, idOf(t, *asked, "watchman"), "watchman", commit, "", "mesh-delivery-watchman")
|
||||
built(t, open, idOf(t, *asked, "view"), "view", commit, "", "mesh-delivery-view")
|
||||
for _, module := range []string{"mesh-delivery-watchman", "mesh-delivery-view"} {
|
||||
if _, known := inventoryHolds(t, open.inventory, module); !known {
|
||||
t.Fatalf("1173/AC3: %s is not registered after the first build asked again passed", module)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,141 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
const refusedKeyTTL = `this is not a module manifest: a state is either a name or {name, history, ttl-seconds, per-machine}: ` +
|
||||
`json: unknown field "key-ttl"`
|
||||
|
||||
// aFirstMergeAnswered is a first merge as askFirstBuilds leaves it: answered by the record of merges no plan answers.
|
||||
func aFirstMergeAnswered() (inventory.BuildRequest, inventory.BatchedMerge) {
|
||||
req := inventory.BuildRequest{ID: "build-1", Repository: "novox/mesh-delivery", Seat: "git", Path: "watchman", Ref: "main",
|
||||
Commit: "0177ea7a00", For: "merge"}
|
||||
event, _ := json.Marshal(keptMerge{SourceMoved: link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main",
|
||||
Commit: "0177ea7a00", Number: 3, HeadSHA: "a1a1a1a1a1a1"}, FirstBuild: true})
|
||||
return req, inventory.BatchedMerge{Repository: "novox/mesh-delivery", Branch: "main", Commit: "0177ea7a00",
|
||||
Plan: inventory.WaitingRecord, Event: event}
|
||||
}
|
||||
|
||||
// 1173/AC1: a first build refused for a word its manifest uses and the running build-agent does not know holds its merge.
|
||||
func TestAFirstBuildRefusedForAnUnknownWordHoldsItsMerge(t *testing.T) {
|
||||
req, merge := aFirstMergeAnswered()
|
||||
held := judgeRefusedFirstBuild(link.BuildResult{ID: "build-1", Failed: refusedKeyTTL}, req, merge)
|
||||
if len(held) != 1 || held[0].Word != "key-ttl" || held[0].Module != "watchman" || held[0].Path != "watchman" {
|
||||
t.Fatalf("1173/AC1: a first build refused for \"key-ttl\" is judged %+v, want a hold on that word for watchman", held)
|
||||
}
|
||||
}
|
||||
|
||||
// 1173/AC2: a held first merge is named at once by one condition: the merge, the module, the word and what it waits for.
|
||||
func TestAHeldFirstMergeIsNamedByACondition(t *testing.T) {
|
||||
now := time.Date(2026, 10, 22, 14, 0, 0, 0, time.UTC)
|
||||
_, merge := aFirstMergeAnswered()
|
||||
var k keptMerge
|
||||
_ = json.Unmarshal(merge.Event, &k)
|
||||
k.Held = []heldFirstBuild{{Module: "watchman", Path: "watchman", Word: "key-ttl", At: now},
|
||||
{Module: "view", Path: "view", Word: "per-machine", At: now}}
|
||||
merge.Event, _ = json.Marshal(k)
|
||||
merge.Heard = now.Add(-time.Second)
|
||||
got := firstBuildObservations([]inventory.BatchedMerge{merge}, now)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("1173/AC2: %d conditions for a held first merge, want one: %+v", len(got), got)
|
||||
}
|
||||
o := got[0]
|
||||
if o.Kind != kindFirstBuildHeld || o.Scope != conditions.ScopeMerge {
|
||||
t.Fatalf("1173/AC2: the condition is %+v", o)
|
||||
}
|
||||
for _, want := range []string{"0177ea7a", "watchman", "key-ttl", "view", "per-machine", "build-agent"} {
|
||||
if !strings.Contains(o.Summary, want) {
|
||||
t.Fatalf("1173/AC2: the summary %q does not name %q", o.Summary, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1173/AC4: a first build that fails for any other reason is not held.
|
||||
func TestAFirstBuildFailingForAnyOtherReasonIsNotHeld(t *testing.T) {
|
||||
req, merge := aFirstMergeAnswered()
|
||||
if judgeRefusedFirstBuild(link.BuildResult{Failed: refusedKeyTTL}, req, merge) == nil {
|
||||
t.Fatal("1173/AC4: the control, a refusal for an unknown word, is not held")
|
||||
}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
failed string
|
||||
change func(*inventory.BuildRequest, *inventory.BatchedMerge)
|
||||
}{
|
||||
{what: "a compile error", failed: "main.go:3: undefined: foo"},
|
||||
{what: "no failure", failed: ""},
|
||||
{what: "a missing field", failed: `json: cannot unmarshal string into Go struct field`},
|
||||
{what: "a build a person asked", failed: refusedKeyTTL, change: func(r *inventory.BuildRequest, _ *inventory.BatchedMerge) { r.For = "build" }},
|
||||
{what: "a merge a plan answers", failed: refusedKeyTTL, change: func(_ *inventory.BuildRequest, m *inventory.BatchedMerge) { m.Plan = "plan-7" }},
|
||||
{what: "a waiting merge held for nothing", failed: refusedKeyTTL, change: func(_ *inventory.BuildRequest, m *inventory.BatchedMerge) { m.Plan = "" }},
|
||||
} {
|
||||
r, m := req, merge
|
||||
if c.change != nil {
|
||||
c.change(&r, &m)
|
||||
}
|
||||
if held := judgeRefusedFirstBuild(link.BuildResult{Failed: c.failed}, r, m); held != nil {
|
||||
t.Fatalf("1173/AC4: %s is held: %+v", c.what, held)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1173/AC1, AC2 (review round 1, item 2): a second refusal in a merge already held is held too, its module added to the
|
||||
// hold, so no failed build is recorded and the condition names both.
|
||||
func TestASecondRefusalInAHeldMergeIsAddedToTheHold(t *testing.T) {
|
||||
req, merge := aFirstMergeAnswered()
|
||||
first := judgeRefusedFirstBuild(link.BuildResult{Failed: refusedKeyTTL}, req, merge)
|
||||
k := announcedOf(merge)
|
||||
k.Held = first
|
||||
merge.Event, _ = json.Marshal(k)
|
||||
merge.Plan = "" // HoldMerge put it back to waiting
|
||||
req.ID, req.Path = "build-2", "view"
|
||||
held := judgeRefusedFirstBuild(link.BuildResult{Failed: `json: unknown field "per-machine"`}, req, merge)
|
||||
if len(held) != 2 || held[0].Module != "watchman" || held[1].Module != "view" || held[1].Word != "per-machine" {
|
||||
t.Fatalf("1173/AC1: the second refusal in a held merge is judged %+v, want watchman and view held", held)
|
||||
}
|
||||
// The same module refused again is held once.
|
||||
k.Held = held
|
||||
merge.Event, _ = json.Marshal(k)
|
||||
if again := judgeRefusedFirstBuild(link.BuildResult{Failed: refusedKeyTTL}, req, merge); len(again) != 2 {
|
||||
t.Fatalf("1173/AC1: view refused again is held as %+v, want it once", again)
|
||||
}
|
||||
}
|
||||
|
||||
// 1173/AC3 (review round 1, item 1): a held merge is asked again for its own held modules' directories, not by the rule of
|
||||
// a first merge, which finds nothing once another module of the repository is registered.
|
||||
func TestAHeldMergeIsAskedAgainForItsHeldModulesAlone(t *testing.T) {
|
||||
k := keptMerge{Held: []heldFirstBuild{{Module: "watchman", Path: "watchman"}, {Module: "mesh-delivery", Path: ""},
|
||||
{Module: "watchman", Path: "/watchman/"}}}
|
||||
if got := heldDirsOf(k); strings.Join(got, ",") != "watchman,." {
|
||||
t.Fatalf("1173/AC3: a held merge asks again %v, want watchman and the root once each", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1173/AC3 (the part that needs no store): only a plan that ended delivered with build-agent built in it is the end a held
|
||||
// first build waits for.
|
||||
func TestOnlyADeliveredPlanOfBuildAgentIsWhatAHeldFirstBuildWaitsFor(t *testing.T) {
|
||||
plan := func(state, walk string, modules map[string]*inventory.PlanModule) inventory.Plan {
|
||||
return inventory.Plan{ID: "p", State: state, WalkSaid: walk, Modules: modules}
|
||||
}
|
||||
built := map[string]*inventory.PlanModule{"build-agent": {State: "built"}}
|
||||
if !deliversBuildAgent(plan(inventory.PlanDone, inventory.WalkEnded, built)) {
|
||||
t.Fatal("1173/AC3: a plan that delivered build-agent is not the end a held first build waits for")
|
||||
}
|
||||
for what, p := range map[string]inventory.Plan{
|
||||
"a plan still running": plan(inventory.PlanBuilding, "", built),
|
||||
"a plan that failed": plan(inventory.PlanFailed, inventory.WalkEnded, built),
|
||||
"a failed build-agent": plan(inventory.PlanDone, inventory.WalkEnded, map[string]*inventory.PlanModule{"build-agent": {State: "failed"}}),
|
||||
"another module's plan": plan(inventory.PlanDone, inventory.WalkEnded, map[string]*inventory.PlanModule{"mailu": {State: "built"}}),
|
||||
} {
|
||||
if deliversBuildAgent(p) {
|
||||
t.Fatalf("1173/AC3: %s is the end a held first build waits for", what)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -68,7 +68,7 @@ func TestTheForgeNotAnsweringInTimeIsNotAskedAgain(t *testing.T) {
|
||||
// An ask that hangs past the bound is cut off at the bound, not at its own wait, and the failure still says it
|
||||
// waited: the receive loop is held no longer than the bound (novox/hq issue 486).
|
||||
func TestAnAskHangingPastTheBoundIsCutOffAtIt(t *testing.T) {
|
||||
w := waitOut{bound: 100 * time.Millisecond, first: 10 * time.Millisecond, most: 50 * time.Millisecond,
|
||||
w := waitOut{bound: 200 * time.Millisecond, first: 10 * time.Millisecond, most: 50 * time.Millisecond,
|
||||
now: time.Now, sleep: sleepOrDone}
|
||||
asked := 0
|
||||
began := time.Now()
|
||||
|
||||
@@ -1,198 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A move held for a person is asked of the person (novox/hq issue 1732, issue 1320; ADR 0377, ADR 0259).
|
||||
//
|
||||
// **A wait only a person can end is said to that person at once.** ADR 0377 holds a recorded module's move that
|
||||
// recreates a container or touches its data until a person pushes the machine. The walk said so in its note and
|
||||
// ended "done", `status` named the machine behind, and nobody was asked: on 2026-10-21 mailu's new build waited on
|
||||
// novox for 40 minutes behind a plan that read "done, 3 tier(s)", and on 2026-10-19 mssql waited on ace the same way.
|
||||
// Row S24 raises one condition per machine that has such a move, for the operator, with the answer **Release**: the
|
||||
// push of that machine, which the controller's own `push` verb performs on the operator's warrant (ADR 0259). It ends
|
||||
// when the machine no longer waits, whoever pushed it.
|
||||
//
|
||||
// A hold a person chose is not asked again — a module they set to `record` with a why, and the bus, whose step has
|
||||
// its own row (S17).
|
||||
|
||||
// kindMovesHeld is S24's kind: machine.<node>.moves-held.
|
||||
const kindMovesHeld = "moves-held"
|
||||
|
||||
// heldFacts is one machine and the moves of recorded modules that wait there for a person.
|
||||
type heldFacts struct {
|
||||
node string
|
||||
moves []heldMove
|
||||
}
|
||||
|
||||
// gatherHeldMoves is, per machine that holds at least one move no person chose to hold, all its held moves, by machine.
|
||||
func gatherHeldMoves(ctx context.Context, inv *inventory.Inventory) ([]heldFacts, error) {
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the machines cannot be read: %w", err)
|
||||
}
|
||||
f, err := readMoveFacts(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []heldFacts
|
||||
for _, n := range nodes {
|
||||
moves, err := heldMovesOn(ctx, inv, f, n.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
h := heldFacts{node: n.Name, moves: moves}
|
||||
if len(h.asked()) > 0 {
|
||||
out = append(out, h)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// asked are the moves no person chose to hold, and chosen the ones a person did (a module set to record with a why, the bus).
|
||||
func (h heldFacts) asked() []heldMove { return h.byChoice(false) }
|
||||
func (h heldFacts) chosen() []heldMove { return h.byChoice(true) }
|
||||
|
||||
func (h heldFacts) byChoice(chosen bool) []heldMove {
|
||||
var out []heldMove
|
||||
for _, m := range h.moves {
|
||||
if m.Chosen == chosen {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// modules are the held moves' modules, in order, of all of them.
|
||||
func (h heldFacts) modules() []string { return moduleNames(h.moves) }
|
||||
|
||||
func moduleNames(moves []heldMove) []string {
|
||||
out := make([]string, 0, len(moves))
|
||||
for _, m := range moves {
|
||||
out = append(out, m.Module)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// releaseAction is the one answer a held move offers: the push of the machine, approved by the operator. The push
|
||||
// names every module whose move waits there as one that may move its data: the operator's Approve is the word
|
||||
// ADR 0217 asks for, and the ask says so. Offered only where no person's own hold is on the machine too: a push
|
||||
// composes every recorded module at its current build, so it would send that move as well, which the ask never named
|
||||
// (ADR 0259 §6), and hold the whole machine if that module's data would move.
|
||||
func (h heldFacts) releaseAction() conditions.Action {
|
||||
return conditions.Action{Label: "Release", Verb: "mesh-controller.push", Level: conditions.LevelApprove,
|
||||
Arguments: map[string]string{"node": h.node, "move": strings.Join(moduleNames(h.asked()), ","), "why": "",
|
||||
"cause": conditions.CauseOperatorAnswer}}
|
||||
}
|
||||
|
||||
// heldWords are the plain words of a machine's held moves: which modules wait, why in general, and what an Approve does.
|
||||
func heldWords(h heldFacts) (headline, explanation, needs string, resolved string) {
|
||||
names := namesWords(moduleNames(h.asked()), 3)
|
||||
headline = clipWords(conditions.Capital(names)+" waits for you on "+h.node, conditions.HeadlineMax)
|
||||
recreates, data := false, false
|
||||
for _, m := range h.asked() {
|
||||
switch {
|
||||
case strings.Contains(m.Why, "recreates"):
|
||||
recreates = true
|
||||
default:
|
||||
data = true
|
||||
}
|
||||
}
|
||||
why := "would change what it runs"
|
||||
switch {
|
||||
case recreates && data:
|
||||
why = "would restart a container or touch its data"
|
||||
case recreates:
|
||||
why = "would restart one of its containers"
|
||||
case data:
|
||||
why = "would touch its files or data"
|
||||
}
|
||||
explanation = clipWords(fmt.Sprintf("The new version of %s %s on %s, so it does not go out with its delivery; "+
|
||||
"it waits for you. Release sends %s everything it is owed, these among it, and lets their data move where it "+
|
||||
"has to. Nothing is sent until you answer.", names, why, h.node, h.node), conditions.ExplanationMax)
|
||||
needs = "release them " + FromMeshMCPServer
|
||||
if chosen := h.chosen(); len(chosen) > 0 {
|
||||
// A push of the machine would send the move a person chose to hold as well: no answer is offered here.
|
||||
explanation = clipWords(fmt.Sprintf("The new version of %s %s on %s, so it does not go out with its delivery; "+
|
||||
"it waits for you. %s is also held there by your own choice, and a push of %s would send that too, so "+
|
||||
"nothing is offered here: the push is yours at the controller's terminal.",
|
||||
names, why, h.node, conditions.Capital(namesWords(moduleNames(chosen), 3)), h.node), conditions.ExplanationMax)
|
||||
needs = fmt.Sprintf("push %s at the controller's terminal, once you have decided about %s.", h.node,
|
||||
namesWords(moduleNames(chosen), 3))
|
||||
}
|
||||
resolved = "Resolved: " + names + " is no longer waiting on " + h.node
|
||||
return
|
||||
}
|
||||
|
||||
// watchMovesHeld is S24: a machine that has moves of recorded modules waiting for a person, said to the operator with the
|
||||
// answer that sends them. The condition ends when the machine no longer waits: it is no longer among the facts.
|
||||
func watchMovesHeld(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, h := range f.held {
|
||||
var said []string
|
||||
for _, m := range h.moves {
|
||||
said = append(said, fmt.Sprintf("%s %s → %s %s", m.Module, short(m.From), short(m.To), m.Why))
|
||||
}
|
||||
headline, explanation, needs, resolved := heldWords(h)
|
||||
if c := h.chosen(); len(c) > 0 {
|
||||
said = append(said, "held by a person's own choice: "+strings.Join(moduleNames(c), ", "))
|
||||
}
|
||||
var actions []conditions.Action
|
||||
if len(h.chosen()) == 0 {
|
||||
actions = []conditions.Action{h.releaseAction()}
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: h.node, Kind: kindMovesHeld,
|
||||
Machine: h.node, Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s has %d move(s) of recorded modules waiting for a person's push: %s — `push %s` sends them",
|
||||
h.node, len(h.moves), strings.Join(h.modules(), ", "), h.node),
|
||||
Said: strings.Join(said, "; "),
|
||||
Headline: headline, Explanation: explanation, Needs: needs, Resolved: resolved,
|
||||
Actions: actions})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// planHeld is, for a plan that ended or is running, the moves its modules carried that a machine still holds for a
|
||||
// person: "module on machine", sorted. The plan's own record says the move waited when it was judged
|
||||
// (PlanModule.Recorded); the machines say whether it still does, and for the build this plan carried, so a plan older
|
||||
// than the build now held is not named.
|
||||
func planHeld(p inventory.Plan, held []heldFacts) []string {
|
||||
var out []string
|
||||
for _, h := range held {
|
||||
for _, m := range h.moves {
|
||||
s := p.Modules[m.Module]
|
||||
if s == nil || !strings.HasPrefix(s.Recorded[h.node], "waits:") || !sameCommit(s.Commit, m.To) {
|
||||
continue
|
||||
}
|
||||
out = append(out, m.Module+" on "+h.node)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// heldSaid is what a plan's line adds when it carries a move that a machine holds for a person (novox/hq issue 1732): a
|
||||
// plan whose module waits for a push is not plainly "done".
|
||||
func heldSaid(p inventory.Plan, held []heldFacts) string {
|
||||
waiting := planHeld(p, held)
|
||||
if len(waiting) == 0 {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf(" — %d held for a person: %s (the operator is asked)", len(waiting), strings.Join(waiting, ", "))
|
||||
}
|
||||
|
||||
// heldNow reads the machines' held moves for a plan's line, nil when they cannot be read: the line then reads as before.
|
||||
func heldNow(ctx context.Context, inv *inventory.Inventory) []heldFacts {
|
||||
held, err := gatherHeldMoves(ctx, inv)
|
||||
if err != nil {
|
||||
return nil // empty-on-error: a plan's line then reads as it did before; the watchdog (S24) says it cannot read
|
||||
}
|
||||
return held
|
||||
}
|
||||
@@ -1,92 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
func inventoryUpgrade(why string) inventory.Upgrade { return inventory.Upgrade{Why: why, By: "jochen"} }
|
||||
|
||||
func mustMoveFacts(t *testing.T, rm *recordedMesh) moveFacts {
|
||||
t.Helper()
|
||||
f, err := readMoveFacts(t.Context(), rm.open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// 1732/AC8 (integration): a mesh running mailu at c1 on two machines, whose registered c2 recreates a container, is read
|
||||
// as two machines each holding mailu's move for a person; a bundle-only c3 on top changes nothing of it; after a machine
|
||||
// is sent c2 (as a push records it) it is no longer among them, and plans that carried c2 say nothing more of it.
|
||||
func TestMachinesHoldingAMoveForAPersonAreReadFromTheRecords(t *testing.T) {
|
||||
rm := aRecordedMesh(t, aMailu(true))
|
||||
c2 := aMailu(true)
|
||||
resourceOf(&c2, "imap")["image"] = "registry.invalid/imap@sha256:2"
|
||||
rm.register(t, c2, "c2", time.Now().Add(-time.Minute))
|
||||
ctx := t.Context()
|
||||
held, err := gatherHeldMoves(ctx, rm.open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(held) != 2 || held[0].node != "anchor" || held[1].node != "laptop" || len(held[0].moves) != 1 ||
|
||||
held[0].moves[0].Module != "mailu" || held[0].moves[0].To != "c2" || !strings.Contains(held[0].moves[0].Why, "recreates") {
|
||||
t.Fatalf("held %+v", held)
|
||||
}
|
||||
if err := rm.open.inventory.RecordSent(ctx, nodeID(t, rm.open, "anchor"), "d-anchor-c2", map[string]string{"mailu": "c2"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held, err = gatherHeldMoves(ctx, rm.open.inventory)
|
||||
if err != nil || len(held) != 1 || held[0].node != "laptop" {
|
||||
t.Fatalf("after anchor was sent c2: %+v (%v)", held, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1732/AC9 (integration): a module a person set to record is no ask: they chose the hold.
|
||||
func TestAHoldAPersonChoseIsNotAsked(t *testing.T) {
|
||||
rm := aRecordedMesh(t, aMailu(false))
|
||||
c2 := aMailu(false)
|
||||
c2.Bundles[0].Digest = "sha256:bbb"
|
||||
rm.register(t, c2, "c2", time.Now().Add(-time.Minute))
|
||||
if err := rm.open.inventory.SetUpgradeOf(t.Context(), "mailu", inventoryUpgrade("the migration week")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held, err := gatherHeldMoves(t.Context(), rm.open.inventory)
|
||||
if err != nil || len(held) != 0 {
|
||||
t.Fatalf("a machine holding only a person's own hold is asked: %+v (%v)", held, err)
|
||||
}
|
||||
if moves, _ := heldMovesOn(t.Context(), rm.open.inventory, mustMoveFacts(t, rm), "anchor"); len(moves) != 1 || !moves[0].Chosen {
|
||||
t.Fatalf("status still names it, marked as chosen: %+v", moves)
|
||||
}
|
||||
}
|
||||
|
||||
// 1742/AC4 (integration): a registered build naming an image of the store that the store lacks is read as a missing
|
||||
// image on each machine that runs the module; one the store holds is not; and what the store holds is not asked twice.
|
||||
func TestARegisteredBuildNamingAnImageTheStoreLacksIsRead(t *testing.T) {
|
||||
m := aMailu(true)
|
||||
resourceOf(&m, "imap")["image"] = "artifact-store://mailu/imap@sha256:" + strings.Repeat("e", 64)
|
||||
rm := aRecordedMesh(t, m)
|
||||
asked := 0
|
||||
lacking := func(_ context.Context, reference string) (bool, error) { asked++; return false, nil }
|
||||
imagesHeld.Lock()
|
||||
imagesHeld.held = map[string]bool{}
|
||||
imagesHeld.Unlock()
|
||||
got, err := gatherMissingImages(t.Context(), rm.open.inventory, lacking)
|
||||
if err != nil || len(got) != 2 || got[0].Module != "mailu" || got[0].Node != "anchor" || got[1].Node != "laptop" {
|
||||
t.Fatalf("missing %+v (%v)", got, err)
|
||||
}
|
||||
holding := func(_ context.Context, reference string) (bool, error) { asked++; return true, nil }
|
||||
if got, err := gatherMissingImages(t.Context(), rm.open.inventory, holding); err != nil || len(got) != 0 {
|
||||
t.Fatalf("an image the store holds is missing: %+v (%v)", got, err)
|
||||
}
|
||||
before := asked
|
||||
if _, err := gatherMissingImages(t.Context(), rm.open.inventory, lacking); err != nil || asked != before {
|
||||
t.Fatalf("an image the store was found to hold was asked about again (%d, %v)", asked-before, err)
|
||||
}
|
||||
}
|
||||
@@ -1,237 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **A move held for a person is asked of the person** (novox/hq issue 1732, issue 1320; ADR 0377, ADR 0259).
|
||||
|
||||
// aHeldMailu is novox holding mailu's new build because it would recreate a container: the walk's own words.
|
||||
func aHeldMailu() heldFacts {
|
||||
return heldFacts{node: "novox", moves: []heldMove{{Module: "mailu", From: "b2f5ebb2aaaa", To: "3259765dbbbb",
|
||||
Why: "waits: recreates 1 of mailu's 12 container(s) (with a new image: automx)"}}}
|
||||
}
|
||||
|
||||
func heldConditionOn(f *signalFacts) conditions.Condition {
|
||||
o := watchMovesHeld(f)[0]
|
||||
return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline,
|
||||
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}
|
||||
}
|
||||
|
||||
// 1732/AC1: a machine holding a move of a recorded module for a person raises one condition for the operator, with
|
||||
// Release as its only answer: the machine's push, approved, naming the modules whose moves wait there.
|
||||
func TestAHeldMoveIsRaisedWithReleaseAsItsOnlyAction(t *testing.T) {
|
||||
f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC))
|
||||
f.held = []heldFacts{aHeldMailu()}
|
||||
got := watchMovesHeld(f)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
o := got[0]
|
||||
if o.Kind != "moves-held" || o.Scope != conditions.ScopeMachine || o.ID != "novox" || o.Severity != conditions.Warning ||
|
||||
o.Resolver != conditions.ResolverOperator || o.Key() != "machine.novox.moves-held" {
|
||||
t.Fatalf("kind, scope, subject, severity or resolver: %+v (%s)", o, o.Key())
|
||||
}
|
||||
if len(o.Actions) != 1 {
|
||||
t.Fatalf("actions: %+v", o.Actions)
|
||||
}
|
||||
a := o.Actions[0]
|
||||
if a.Label != "Release" || a.Verb != "mesh-controller.push" || a.Level != conditions.LevelApprove ||
|
||||
a.Arguments["node"] != "novox" || a.Arguments["move"] != "mailu" || a.Arguments["cause"] != conditions.CauseOperatorAnswer {
|
||||
t.Fatalf("the action is not the push of novox: %+v", a)
|
||||
}
|
||||
if _, takes := a.Arguments["why"]; !takes {
|
||||
t.Fatal("the push takes a why, which the asker fills in with the operator's answer")
|
||||
}
|
||||
for _, want := range []string{"mailu", "novox"} {
|
||||
if !strings.Contains(o.Headline+o.Explanation, want) {
|
||||
t.Errorf("the words do not name %q: %q / %q", want, o.Headline, o.Explanation)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(o.Said, "recreates 1 of mailu's 12") {
|
||||
t.Errorf("the evidence does not carry the walk's own words: %q", o.Said)
|
||||
}
|
||||
if w := (conditions.Observation{Headline: o.Headline, Explanation: o.Explanation, Needs: o.Needs, Resolved: o.Resolved}); w.Headline == "" || len(o.Headline) > conditions.HeadlineMax {
|
||||
t.Errorf("headline: %q", o.Headline)
|
||||
}
|
||||
}
|
||||
|
||||
// 1732/AC2: a machine with nothing held raises nothing, so the condition ends the look after the move is sent.
|
||||
func TestAMachineHoldingNothingRaisesNoHeldMoveCondition(t *testing.T) {
|
||||
f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC))
|
||||
if got := watchMovesHeld(f); len(got) != 0 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1732/AC3: the condition is asked on a channel at the approve level in words that say what Release does, and the
|
||||
// operator's warrant calls the controller's own push with the machine and the modules, once.
|
||||
func TestAHeldMovesReleaseIsAskedAndAWarrantPushesTheMachine(t *testing.T) {
|
||||
f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC))
|
||||
f.held = []heldFacts{aHeldMailu()}
|
||||
c := heldConditionOn(f)
|
||||
r := newAskerRig(t)
|
||||
r.open = []conditions.Condition{c}
|
||||
if err := r.a.reconcile(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sent := r.asksSent(t)
|
||||
if len(sent) != 1 || len(sent[0].Options) != 1 {
|
||||
t.Fatalf("asked %+v", sent)
|
||||
}
|
||||
o := sent[0].Options[0]
|
||||
if o.Label != "Release" || o.Level != asks.Approve || o.Does != "the held builds are sent to novox" {
|
||||
t.Errorf("the option is %+v", o)
|
||||
}
|
||||
if err := sent[0].Check(r.now); err != nil {
|
||||
t.Errorf("the ask is refused as made: %v", err)
|
||||
}
|
||||
w := r.warrantFor(t, c.Key, "Release")
|
||||
answerWith(t, r, w)
|
||||
answerWith(t, r, w)
|
||||
if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "mesh-controller.push@ ") ||
|
||||
!strings.Contains(r.called[0], `"node":"novox"`) || !strings.Contains(r.called[0], `"move":"mailu"`) ||
|
||||
!strings.Contains(r.called[0], `"cause":"`+conditions.CauseOperatorAnswer+`"`) ||
|
||||
!strings.Contains(r.called[0], "chose Release") {
|
||||
t.Fatalf("called %v", r.called)
|
||||
}
|
||||
}
|
||||
|
||||
// 1732/AC4: the controller may call its own push and upgrade verbs on a warrant: the grant names them.
|
||||
func TestTheControllerMayActOnAWarrantForPushAndUpgrade(t *testing.T) {
|
||||
for _, verb := range []string{"push", "upgrade"} {
|
||||
granted := false
|
||||
for _, v := range broker.VerbsTheControllerActsOnAWarrant {
|
||||
granted = granted || (v.Seat == broker.ControllerSeat && v.Verb == verb)
|
||||
}
|
||||
if !granted {
|
||||
t.Errorf("the controller may not act on a warrant for %s", verb)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func aDoneMailuPlan() inventory.Plan {
|
||||
return inventory.Plan{ID: "plan-1792585970446304628", Repository: "novox/mesh-catalog", State: inventory.PlanDone,
|
||||
Tiers: [][]string{{"a"}, {"b"}, {"mailu"}},
|
||||
Modules: map[string]*inventory.PlanModule{"mailu": {State: "built", Commit: "3259765dbbbbcccc",
|
||||
Recorded: map[string]string{"novox": "waits: recreates 1 of mailu's 12 container(s)"}}}}
|
||||
}
|
||||
|
||||
// 1732/AC5: a done plan that carries a move a machine still holds for a person says so on its line; once the machine
|
||||
// no longer holds it, or holds another build of the module, the line says nothing more.
|
||||
func TestADonePlanSaysWhatItHoldsForAPerson(t *testing.T) {
|
||||
p := aDoneMailuPlan()
|
||||
line := planLine(p, time.Date(2026, 10, 21, 16, 0, 0, 0, time.UTC)) + heldSaid(p, []heldFacts{aHeldMailu()})
|
||||
if !strings.Contains(line, "done, 3 tier(s)") || !strings.Contains(line, "1 held for a person: mailu on novox") {
|
||||
t.Errorf("the line: %q", line)
|
||||
}
|
||||
if said := heldSaid(p, nil); said != "" {
|
||||
t.Errorf("a plan whose moves were sent says %q", said)
|
||||
}
|
||||
other := aHeldMailu()
|
||||
other.moves[0].To = "9999999ffff"
|
||||
if said := heldSaid(p, []heldFacts{other}); said != "" {
|
||||
t.Errorf("a plan older than the build now held says %q", said)
|
||||
}
|
||||
p.Modules["mailu"].Recorded = map[string]string{"novox": recordedRollsWhy}
|
||||
if said := heldSaid(p, []heldFacts{aHeldMailu()}); said != "" {
|
||||
t.Errorf("a move the plan sent says %q", said)
|
||||
}
|
||||
}
|
||||
|
||||
// 1732/AC6: the builds held after a failed release are asked of the operator: the held-release condition offers
|
||||
// Release, which calls the controller's own `upgrade` with release-backlog and nothing else, once, on a warrant.
|
||||
func TestTheBuildsHeldAfterAFailedReleaseAreAskedAndReleasedOnAWarrant(t *testing.T) {
|
||||
saved := backlogNow
|
||||
t.Cleanup(func() { backlogNow = saved })
|
||||
backlogNow.held = "release-1792595552499982311 failed (failed on novox); what waits is released again by a person"
|
||||
backlogNow.waiting = map[string][]inventory.CarriedMove{"novox": {{Module: "indexer"}, {Module: "nextcloud"}}}
|
||||
got := backlogObservation()
|
||||
if len(got) != 1 || len(got[0].Actions) != 1 {
|
||||
t.Fatalf("raised %+v", got)
|
||||
}
|
||||
a := got[0].Actions[0]
|
||||
if a.Label != "Release" || a.Verb != "mesh-controller.upgrade" || a.Level != conditions.LevelApprove ||
|
||||
a.Arguments["release-backlog"] != "true" || a.Arguments["cause"] != conditions.CauseOperatorAnswer {
|
||||
t.Fatalf("the action: %+v", a)
|
||||
}
|
||||
o := got[0]
|
||||
c := conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline,
|
||||
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}
|
||||
r := newAskerRig(t)
|
||||
r.open = []conditions.Condition{c}
|
||||
if err := r.a.reconcile(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sent := r.asksSent(t)
|
||||
if len(sent) != 1 || len(sent[0].Options) != 1 || sent[0].Options[0].Does != "the builds held after the failed delivery are released" {
|
||||
t.Fatalf("asked %+v", sent)
|
||||
}
|
||||
w := r.warrantFor(t, c.Key, "Release")
|
||||
answerWith(t, r, w)
|
||||
answerWith(t, r, w)
|
||||
if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "mesh-controller.upgrade@ ") ||
|
||||
!strings.Contains(r.called[0], `"release-backlog":"true"`) || !strings.Contains(r.called[0], "chose Release") {
|
||||
t.Fatalf("called %v", r.called)
|
||||
}
|
||||
}
|
||||
|
||||
// 1732/AC7: a plan whose send waits for a gate while the builds are held after a failed release reads as waiting for
|
||||
// a person to release them, not "building ... LATE"; with nothing held it still reads as the wait it is.
|
||||
func TestAPlanWaitingForAGateBehindAFailedReleaseIsNotLate(t *testing.T) {
|
||||
saved := backlogNow
|
||||
t.Cleanup(func() { backlogNow = saved })
|
||||
now := time.Date(2026, 10, 21, 18, 50, 0, 0, time.UTC)
|
||||
p := inventory.Plan{ID: "plan-1", Repository: "novox/mesh-sdk", Commit: "c0ffee00", State: inventory.PlanBuilding,
|
||||
Tiers: [][]string{{"indexer"}}, TierEntered: now.Add(-100 * time.Minute), Created: now.Add(-110 * time.Minute),
|
||||
Modules: map[string]*inventory.PlanModule{"indexer": {State: "built"}},
|
||||
Note: refusalNote(0, errors.New("a build waits there for its gate: novox"))}
|
||||
backlogNow.held = ""
|
||||
if line := planLine(p, now); !strings.Contains(line, "a build waits there for its gate: novox") {
|
||||
t.Errorf("the wait is not said: %q", line)
|
||||
}
|
||||
backlogNow.held = "release-1 failed (failed on novox); what waits is released again by a person"
|
||||
line := planLine(p, now)
|
||||
if strings.Contains(line, "LATE") || strings.Contains(line, "building") ||
|
||||
!strings.Contains(line, "until a person releases it") {
|
||||
t.Errorf("a plan held behind a failed release reads %q", line)
|
||||
}
|
||||
}
|
||||
|
||||
// 1732/AC10: a machine that also holds a move a person chose to hold gets no Release, since a push would send that move
|
||||
// too; the condition names both and says the push is the controller's terminal. A machine with only asked moves keeps it.
|
||||
func TestAMachineAlsoHoldingAPersonsOwnHoldIsOfferedNoRelease(t *testing.T) {
|
||||
f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC))
|
||||
h := aHeldMailu()
|
||||
h.moves = append(h.moves, heldMove{Module: "nftables", From: "n1aaaa", To: "n2bbbb",
|
||||
Why: "waits: a person set it to record, and holds every move of it until they push", Chosen: true})
|
||||
f.held = []heldFacts{h}
|
||||
got := watchMovesHeld(f)
|
||||
if len(got) != 1 || len(got[0].Actions) != 0 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
o := got[0]
|
||||
text := o.Headline + o.Explanation + o.Needs + o.Summary + o.Said
|
||||
for _, want := range []string{"mailu", "nftables", "terminal", "push novox"} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Errorf("does not say %q: %q", want, text)
|
||||
}
|
||||
}
|
||||
w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs}
|
||||
if why, ok := conditions.PlainWords(w, "novox"); !ok {
|
||||
t.Errorf("not plain: %s", why)
|
||||
}
|
||||
f.held = []heldFacts{aHeldMailu()}
|
||||
if got := watchMovesHeld(f); len(got) != 1 || len(got[0].Actions) != 1 || got[0].Actions[0].Arguments["move"] != "mailu" {
|
||||
t.Fatalf("a machine with only asked moves: %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -1,245 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// No declaration is sent that names an image the artifact store does not hold (novox/hq issue 1742).
|
||||
//
|
||||
// On 2026-10-21 at 17:13 a send to novox named `mailu/automx@sha256:c62150b8…`; the build record keeps that digest
|
||||
// ("recent-build") and the store did not hold it. The node-engine removed the running container, then could not pull
|
||||
// the replacement, and every later apply on novox failed for the next hour while the mail clients' autoconfiguration
|
||||
// had no backend. The node-engine's half (pull first, replace after) is its own change; this is the controller's: **a
|
||||
// module whose image the store lacks is left out of the declaration**, as a module whose setting nobody gave is
|
||||
// (ADR 0163, rule 6) — the node-engine keeps what it has of it and touches none of its containers — and **it is said**,
|
||||
// row S25: a condition naming the module, the machine and the image, until the store holds the image again (a new build
|
||||
// of the module, or the copy put back).
|
||||
//
|
||||
// Judged only for an image the store's own address names, from the send's own resources; an image of another registry
|
||||
// is no business of the store's. A store that cannot be asked judges nothing: a send is not refused for a question
|
||||
// that did not answer.
|
||||
|
||||
// kindImageMissing is S25's kind: module.<module>.<machine>.image-missing.
|
||||
const kindImageMissing = "image-missing"
|
||||
|
||||
// missingImage is a module held out of a machine's declaration because the store lacks an image it names.
|
||||
type missingImage struct {
|
||||
Node, Module, Image string
|
||||
}
|
||||
|
||||
// storeHolds is whether the store holds the manifest an image reference at its address names.
|
||||
type storeHolds func(ctx context.Context, reference string) (bool, error)
|
||||
|
||||
// imageInStore is the artifact-store reference of an image the store's address names, "" for any other.
|
||||
func imageInStore(image, address string) string {
|
||||
if address == "" || !strings.HasPrefix(image, address+"/") || !strings.Contains(image, "@sha256:") {
|
||||
return ""
|
||||
}
|
||||
reference := catalogue.Recorded(image)
|
||||
if _, ours := catalogue.InArtifactStore(reference); !ours {
|
||||
return ""
|
||||
}
|
||||
return reference
|
||||
}
|
||||
|
||||
// holdBackMissingImages leaves out of the declaration every module that names an image the store does not hold,
|
||||
// and answers what it left out. The module's resources go, it joins LeftOut with the reason, and its build stays the
|
||||
// one the machine was last sent (`was`).
|
||||
func holdBackMissingImages(ctx context.Context, node string, s *sendable, address string, holds storeHolds,
|
||||
was map[string]string) []missingImage {
|
||||
lacks := map[string]bool{} // reference → the store does not hold it; absent: not asked, or no answer
|
||||
asked := map[string]bool{}
|
||||
lacking := map[string]string{} // module → the image
|
||||
for _, r := range s.Resources {
|
||||
if fmt.Sprint(r["type"]) != "container" {
|
||||
continue
|
||||
}
|
||||
image, _ := r["image"].(string)
|
||||
reference := imageInStore(image, address)
|
||||
module := s.moduleOf[fmt.Sprint(r["id"])]
|
||||
if reference == "" || module == "" || lacking[module] != "" {
|
||||
continue
|
||||
}
|
||||
if !asked[reference] {
|
||||
asked[reference] = true
|
||||
held, err := holds(ctx, reference)
|
||||
// empty-on-error: the store did not answer, so nothing is held back for a question unanswered
|
||||
lacks[reference] = err == nil && !held
|
||||
}
|
||||
if lacks[reference] {
|
||||
lacking[module] = image
|
||||
}
|
||||
}
|
||||
if len(lacking) == 0 {
|
||||
return nil
|
||||
}
|
||||
kept := s.Resources[:0:0]
|
||||
for _, r := range s.Resources {
|
||||
if _, out := lacking[s.moduleOf[fmt.Sprint(r["id"])]]; !out {
|
||||
kept = append(kept, r)
|
||||
}
|
||||
}
|
||||
s.Resources = kept
|
||||
if s.leftOutWhy == nil {
|
||||
s.leftOutWhy = map[string]string{}
|
||||
}
|
||||
var found []missingImage
|
||||
for module, image := range lacking {
|
||||
s.leftOutWhy[module] = "its image " + image + " is not in the artifact store"
|
||||
if !slices.Contains(s.LeftOut, module) {
|
||||
s.LeftOut = append(s.LeftOut, module)
|
||||
}
|
||||
if s.Builds != nil {
|
||||
if prior, sent := was[module]; sent {
|
||||
s.Builds[module] = prior
|
||||
} else {
|
||||
delete(s.Builds, module)
|
||||
}
|
||||
}
|
||||
found = append(found, missingImage{Node: node, Module: module, Image: image})
|
||||
}
|
||||
sort.Strings(s.LeftOut)
|
||||
sort.Slice(found, func(i, j int) bool { return found[i].Module < found[j].Module })
|
||||
return found
|
||||
}
|
||||
|
||||
// withoutMissingImages is composeSend's last step: the store asked about the images the declaration names, the modules
|
||||
// whose image it lacks left out and kept in the ledger S25 reads. A store that cannot be found or asked leaves the
|
||||
// declaration as composed.
|
||||
func withoutMissingImages(ctx context.Context, open *stores, node string, s sendable) sendable {
|
||||
inv := open.inventory
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return s
|
||||
}
|
||||
address, err := artifactStoreAddress(ctx, inv, shelf, node)
|
||||
if err != nil || address == "" {
|
||||
return s
|
||||
}
|
||||
sent, known, err := inv.SentBuilds(ctx, node)
|
||||
if err != nil || !known {
|
||||
sent = nil
|
||||
}
|
||||
store := artifacts.Store{Address: address}
|
||||
found := holdBackMissingImages(ctx, node, &s, address, store.HoldsManifest, sent)
|
||||
for _, m := range found {
|
||||
fmt.Printf("%s: %s is left out of the declaration: its image %s is not in the artifact store\n", node, m.Module, m.Image)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// imageFacts are the images S25 raises: a module whose registered build names an image the store lacks, on each machine
|
||||
// that runs it.
|
||||
type imageFacts struct {
|
||||
missingImage
|
||||
}
|
||||
|
||||
// imagesHeld remembers the images the store was found to hold: a stored image is named by its digest and stays what it
|
||||
// is, so it is asked about once; one it lacks is asked again at every look.
|
||||
var imagesHeld = struct {
|
||||
sync.Mutex
|
||||
held map[string]bool
|
||||
}{held: map[string]bool{}}
|
||||
|
||||
// gatherMissingImages is every machine's module whose registered build — the one a send would carry — names an image of
|
||||
// the artifact store that the store does not hold. Read from the catalogue and the store, never from what a send did:
|
||||
// a push is a command of its own, and the serving controller's watchdog hears nothing of it.
|
||||
func gatherMissingImages(ctx context.Context, inv *inventory.Inventory, holds storeHolds) ([]imageFacts, error) {
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the catalogue cannot be read: %w", err)
|
||||
}
|
||||
var out []imageFacts
|
||||
for _, module := range sortedKeysOf(modulesOf(shelf)) {
|
||||
for _, r := range shelf[module].Resources {
|
||||
if fmt.Sprint(r["type"]) != "container" {
|
||||
continue
|
||||
}
|
||||
image, _ := r["image"].(string)
|
||||
if _, ours := catalogue.InArtifactStore(image); !ours || !strings.Contains(image, "@sha256:") {
|
||||
continue
|
||||
}
|
||||
imagesHeld.Lock()
|
||||
known := imagesHeld.held[image]
|
||||
imagesHeld.Unlock()
|
||||
if !known {
|
||||
held, err := holds(ctx, image)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the artifact store cannot be asked about %s: %w", image, err)
|
||||
}
|
||||
if held {
|
||||
imagesHeld.Lock()
|
||||
imagesHeld.held[image] = true
|
||||
imagesHeld.Unlock()
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
running, err := inv.Running(ctx, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, node := range running {
|
||||
out = append(out, imageFacts{missingImage{Node: node, Module: module, Image: image}})
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// modulesOf is the shelf's modules as a map of names, for sortedKeysOf.
|
||||
func modulesOf(shelf map[string]catalogue.Manifest) map[string]string {
|
||||
out := make(map[string]string, len(shelf))
|
||||
for m := range shelf {
|
||||
out[m] = m
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// watchMissingImages is S25: a module left out of a machine's declaration because the store lacks its image, said at
|
||||
// once. The condition ends when the store holds the image, or a send names another.
|
||||
func watchMissingImages(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, m := range f.images {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeModule, ID: m.Module + "." + m.Node, Kind: kindImageMissing,
|
||||
Machine: m.Node, Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s is left out of every send to %s: its declaration names an image that the artifact store does "+
|
||||
"not hold (the evidence names it); the running container is untouched", m.Module, m.Node),
|
||||
Said: "image " + m.Image + " is not in the store",
|
||||
Headline: clipWords(conditions.Capital(m.Module)+" is not updated on "+m.Node, conditions.HeadlineMax),
|
||||
Explanation: clipWords(fmt.Sprintf("The new version of %s names a container image that the mesh's store does not "+
|
||||
"hold, so it was not sent to %s. What runs there is untouched, and the rest of %s is sent as usual.",
|
||||
m.Module, m.Node, m.Node), conditions.ExplanationMax),
|
||||
Needs: fmt.Sprintf("build %s again; the details name the image.", m.Module),
|
||||
Resolved: "Resolved: the store holds the image of " + m.Module + " again"})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// gatherMissingImagesHere is gatherMissingImages asking the mesh's own artifact store; nothing to ask where the mesh has
|
||||
// none on its network yet.
|
||||
func gatherMissingImagesHere(ctx context.Context, inv *inventory.Inventory) ([]imageFacts, error) {
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the catalogue cannot be read: %w", err)
|
||||
}
|
||||
address, err := artifactStoreAddress(ctx, inv, shelf, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if address == "" {
|
||||
return nil, nil
|
||||
}
|
||||
return gatherMissingImages(ctx, inv, artifacts.Store{Address: address}.HoldsManifest)
|
||||
}
|
||||
@@ -1,125 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// **No declaration is sent that names an image the artifact store does not hold** (novox/hq issue 1742).
|
||||
|
||||
const storeAt = "novox.internal:5100"
|
||||
|
||||
// automx is the image the send of 2026-10-21 17:13 named and the store lacked; the others are held.
|
||||
const (
|
||||
automxImage = storeAt + "/mailu/automx@sha256:c62150b8e2cbcc18cb911b098c2877f928761a1e6251667f39ea18251ea35591"
|
||||
imapImage = storeAt + "/mailu/imap@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
)
|
||||
|
||||
func aMailSend() sendable {
|
||||
return sendable{
|
||||
Resources: []map[string]any{
|
||||
{"id": "mailu.automx", "type": "container", "image": automxImage},
|
||||
{"id": "mailu.automx-config", "type": "file", "path": "/etc/automx.conf"},
|
||||
{"id": "mailu.imap", "type": "container", "image": imapImage},
|
||||
{"id": "other.web", "type": "container", "image": "docker.io/library/nginx@sha256:" + strings.Repeat("b", 64)},
|
||||
{"id": "dns.dnsmasq", "type": "container", "image": storeAt + "/dnsmasq/main@sha256:" + strings.Repeat("c", 64)},
|
||||
},
|
||||
moduleOf: map[string]string{"mailu.automx": "automx", "mailu.automx-config": "automx", "mailu.imap": "mailu",
|
||||
"other.web": "other", "dns.dnsmasq": "dnsmasq"},
|
||||
Builds: map[string]string{"automx": "b2f5ebb2new", "mailu": "m2", "other": "o1", "dnsmasq": "d1"},
|
||||
}
|
||||
}
|
||||
|
||||
func storeHolding(missing ...string) (storeHolds, *[]string) {
|
||||
var asked []string
|
||||
return func(_ context.Context, reference string) (bool, error) {
|
||||
asked = append(asked, reference)
|
||||
for _, m := range missing {
|
||||
if strings.Contains(reference, m) {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}, &asked
|
||||
}
|
||||
|
||||
// 1742/AC1: given a send naming automx's image, which the store does not hold, when the declaration is judged, then
|
||||
// every resource of automx is out of it, automx is among the modules left out with the reason, its build is the one the
|
||||
// machine was last sent, and every other module — an image of another registry among them — is as composed.
|
||||
func TestAModuleWhoseImageTheStoreLacksIsLeftOutOfTheSend(t *testing.T) {
|
||||
s := aMailSend()
|
||||
holds, asked := storeHolding("mailu/automx@")
|
||||
found := holdBackMissingImages(t.Context(), "novox", &s, storeAt, holds, map[string]string{"automx": "b2f5ebb2old"})
|
||||
if len(found) != 1 || found[0].Module != "automx" || found[0].Node != "novox" || found[0].Image != automxImage {
|
||||
t.Fatalf("found %+v", found)
|
||||
}
|
||||
var ids []string
|
||||
for _, r := range s.Resources {
|
||||
ids = append(ids, r["id"].(string))
|
||||
}
|
||||
if got := strings.Join(ids, ","); got != "mailu.imap,other.web,dns.dnsmasq" {
|
||||
t.Errorf("the declaration keeps %s", got)
|
||||
}
|
||||
if len(s.LeftOut) != 1 || s.LeftOut[0] != "automx" || !strings.Contains(s.leftOutWhy["automx"], "not in the artifact store") {
|
||||
t.Errorf("left out %v because %v", s.LeftOut, s.leftOutWhy)
|
||||
}
|
||||
if s.Builds["automx"] != "b2f5ebb2old" || s.Builds["mailu"] != "m2" || s.Builds["other"] != "o1" {
|
||||
t.Errorf("builds %v", s.Builds)
|
||||
}
|
||||
for _, a := range *asked {
|
||||
if strings.Contains(a, "docker.io") {
|
||||
t.Errorf("the store was asked about an image of another registry: %s", a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1742/AC2: given a module never sent to the machine, when its image is missing, then it has no build in the record of the
|
||||
// send; given the store does not answer, then nothing is held back for a question unanswered.
|
||||
func TestAnUnansweredStoreHoldsNothingBackAndANewModuleHasNoBuild(t *testing.T) {
|
||||
s := aMailSend()
|
||||
holds, _ := storeHolding("mailu/automx@")
|
||||
found := holdBackMissingImages(t.Context(), "novox", &s, storeAt, holds, nil)
|
||||
if len(found) != 1 {
|
||||
t.Fatalf("%+v", found)
|
||||
}
|
||||
if _, has := s.Builds["automx"]; has {
|
||||
t.Errorf("a module never sent is recorded as sent: %v", s.Builds)
|
||||
}
|
||||
s = aMailSend()
|
||||
silent := func(context.Context, string) (bool, error) { return false, errors.New("registry: connection refused") }
|
||||
if found := holdBackMissingImages(t.Context(), "novox", &s, storeAt, silent, nil); len(found) != 0 || len(s.LeftOut) != 0 ||
|
||||
len(s.Resources) != 5 {
|
||||
t.Errorf("an unanswered store held back %+v, left out %v", found, s.LeftOut)
|
||||
}
|
||||
}
|
||||
|
||||
// 1742/AC3: a module left out for its image raises a condition for the operator naming the module, the machine and the
|
||||
// image in the evidence, plain words, and no answer to give; a mesh whose images are all held raises nothing.
|
||||
func TestAMissingImageIsRaisedForTheOperator(t *testing.T) {
|
||||
f := calm(time.Date(2026, 10, 21, 17, 40, 0, 0, time.UTC))
|
||||
if got := watchMissingImages(f); len(got) != 0 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
f.images = []imageFacts{{missingImage{Node: "novox", Module: "automx", Image: automxImage}}}
|
||||
got := watchMissingImages(f)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
o := got[0]
|
||||
if o.Kind != "image-missing" || o.Key() != "module.automx.novox.image-missing" || o.Resolver != conditions.ResolverOperator ||
|
||||
o.Severity != conditions.Warning || o.Machine != "novox" {
|
||||
t.Fatalf("%+v (%s)", o, o.Key())
|
||||
}
|
||||
if !strings.Contains(o.Said, "c62150b8") || !strings.Contains(o.Headline+o.Explanation, "automx") {
|
||||
t.Errorf("words: %q / %q / %q", o.Headline, o.Explanation, o.Said)
|
||||
}
|
||||
w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs}
|
||||
if why, ok := conditions.PlainWords(w, "novox"); !ok {
|
||||
t.Errorf("not plain: %s", why)
|
||||
}
|
||||
}
|
||||
@@ -35,7 +35,7 @@ func TestAMergeSaysTheModulesWhoseBuildSourceItLeftUnchanged(t *testing.T) {
|
||||
{Own: true, Paths: []string{"modules/build-agent/Dockerfile", "modules/build-agent/module.json"}},
|
||||
},
|
||||
"route-proxy": {
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"go.mod", "modules/route-proxy/cmd/route-proxy/"}},
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"go.mod", "examples/route-proxy/"}},
|
||||
{Own: true, Paths: []string{"modules/route-proxy/Dockerfile", "modules/route-proxy/module.json"}},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -408,13 +408,6 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
||||
fmt.Printf("%s: a dry run of %s on %s, not taken in\n", result.ID, result.Repository, result.Ref)
|
||||
return nil
|
||||
}
|
||||
// **A first build refused for a word the running build-agent does not read waits for one that does** (novox/hq ADR
|
||||
// 0482 §5, issue 1173): its merge is kept, named, and never failed; no failed build is recorded.
|
||||
if held, err := holdRefusedFirstBuild(ctx, b.inv, result); err != nil {
|
||||
return err
|
||||
} else if held {
|
||||
return nil
|
||||
}
|
||||
manifest, _, err := takeIn(ctx, b.inv, result)
|
||||
// The assignments pending on this build, made or ended (novox/hq issue 325), said in the daemon's log
|
||||
// after what became of the build.
|
||||
|
||||
@@ -65,9 +65,6 @@ type mergeCheckInput struct {
|
||||
// group are the other heads of a delivery group composed with this one (novox/hq ADR 0239), each laid
|
||||
// over the mesh after the one before: the group judged as one future state.
|
||||
group []groupChange
|
||||
// held are the approved config values held for the keys the pull requests introduce (novox/hq ADR 0502 clause 4),
|
||||
// laid over the composition with the change only.
|
||||
held []heldValue
|
||||
// admin is a PostgreSQL the gate may create and drop databases in.
|
||||
admin string
|
||||
// beside is the directory holding the core clones beside the check (MESH_CHECK_BESIDE), for finding moves.
|
||||
@@ -192,8 +189,6 @@ func mergeGateCommand(ctx context.Context, args []string) error {
|
||||
"--changed, for a list too long for an argument (novox/hq issue 1247)")
|
||||
group := set.String("group", "", "a delivery group's other heads, as JSON [{repository, tree, changed}], composed "+
|
||||
"with this one as one future state (novox/hq ADR 0239)")
|
||||
heldFile := set.String("held", "", "a file of the config values approved for keys the change's pull requests introduce, as JSON "+
|
||||
"[{pull, node, module, key, value, approved_by}], laid over the composition with the change (novox/hq ADR 0502)")
|
||||
asJSON := set.Bool("json", false, "print the verdict as JSON")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
@@ -213,10 +208,6 @@ func mergeGateCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
held, err := readHeld(*heldFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out := io.Writer(os.Stdout)
|
||||
stdout := os.Stdout
|
||||
if *asJSON {
|
||||
@@ -232,7 +223,7 @@ func mergeGateCommand(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
v, err := judgeChange(ctx, mergeCheckInput{facts: f, repository: *repository, tree: *tree, changed: changedList,
|
||||
group: others, held: held, admin: *admin, say: out, beside: os.Getenv("MESH_CHECK_BESIDE")})
|
||||
group: others, admin: *admin, say: out, beside: os.Getenv("MESH_CHECK_BESIDE")})
|
||||
os.Stdout = stdout
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -396,16 +387,7 @@ func judgeChange(ctx context.Context, in mergeCheckInput) (mergeVerdict, error)
|
||||
}
|
||||
}
|
||||
sort.Strings(added)
|
||||
// **The held values are laid in this composition only** (novox/hq ADR 0502 clause 4): the mesh as it is does not
|
||||
// declare their keys, so the base composition would refuse them.
|
||||
withHeld := in
|
||||
var heldNotes []string
|
||||
withHeld.facts, heldNotes = layHeld(in.facts, in.held, change)
|
||||
for _, n := range heldNotes {
|
||||
say("%s", n)
|
||||
}
|
||||
v.Notes = append(v.Notes, heldNotes...)
|
||||
after, tried, notes, err := composeEveryMachine(ctx, withHeld, change, changeSources, added)
|
||||
after, tried, notes, err := composeEveryMachine(ctx, in, change, changeSources, added)
|
||||
if err != nil {
|
||||
return v, fmt.Errorf("the mesh with the change could not be raised: %w", err)
|
||||
}
|
||||
|
||||
@@ -1,135 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
snapshot "git.novox.be/novox/mesh-sdk/go/facts"
|
||||
)
|
||||
|
||||
// Config values held for the keys an open pull request introduces (novox/hq ADR 0502 clause 4, issue 1826): the gate
|
||||
// lays each approved one over its module's values on its node in the composition with the change, and in that one
|
||||
// only. In the mesh as it is the key is not declared, so the value would be refused there.
|
||||
|
||||
// heldValue is one approved held value as the build agent writes it (--held): the value of Key for Module on Node,
|
||||
// a machine's real name, which the snapshot holds only as a pseudonym.
|
||||
type heldValue struct {
|
||||
Pull string `json:"pull"`
|
||||
Head string `json:"head,omitempty"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
Approved string `json:"approved,omitempty"`
|
||||
By string `json:"by,omitempty"`
|
||||
At string `json:"at,omitempty"`
|
||||
}
|
||||
|
||||
// readHeld reads the file --held names; none named, none held.
|
||||
func readHeld(path string) ([]heldValue, error) {
|
||||
if path == "" {
|
||||
return nil, nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var held []heldValue
|
||||
if err := json.Unmarshal(raw, &held); err != nil {
|
||||
return nil, fmt.Errorf("the held values are not readable: %w", err)
|
||||
}
|
||||
for _, h := range held {
|
||||
if h.Node == "" || h.Module == "" || h.Key == "" || len(h.Value) == 0 {
|
||||
return nil, fmt.Errorf("a held value of %s names no node, module, key or value", h.Pull)
|
||||
}
|
||||
}
|
||||
return held, nil
|
||||
}
|
||||
|
||||
// layHeld is the snapshot with each held value laid as the value of its key in its node's layer of its module, and what
|
||||
// it said of them. The snapshot given is not changed. **A held value for a machine the snapshot does not hold, or a
|
||||
// module the change does not define, is said and not laid**, never dropped silently; one for a module not assigned to
|
||||
// its machine is laid and said. The notes name machines as the snapshot does (roles and pseudonym), not by their names.
|
||||
func layHeld(f snapshot.Facts, held []heldValue, shelf map[string]catalogue.Manifest) (snapshot.Facts, []string) {
|
||||
if len(held) == 0 {
|
||||
return f, nil
|
||||
}
|
||||
out := f
|
||||
out.Machines = make([]snapshot.Machine, len(f.Machines))
|
||||
copy(out.Machines, f.Machines)
|
||||
var notes, laid []string
|
||||
pulls := map[string]bool{}
|
||||
for _, h := range held {
|
||||
at := -1
|
||||
pseudonym := snapshot.Pseudonym("machine", h.Node)
|
||||
for i, m := range out.Machines {
|
||||
if m.Name == pseudonym {
|
||||
at = i
|
||||
}
|
||||
}
|
||||
if at < 0 {
|
||||
notes = append(notes, fmt.Sprintf("the value held for %s of %s (%s=%s) is not composed: the snapshot holds no machine %s",
|
||||
h.Module, h.Pull, h.Key, h.Value, pseudonym))
|
||||
continue
|
||||
}
|
||||
if _, defined := shelf[h.Module]; !defined {
|
||||
notes = append(notes, fmt.Sprintf("the value held for %s of %s (%s=%s) is not composed: the change defines no module %s",
|
||||
h.Module, h.Pull, h.Key, h.Value, h.Module))
|
||||
continue
|
||||
}
|
||||
var value any
|
||||
if err := json.Unmarshal(h.Value, &value); err != nil {
|
||||
notes = append(notes, fmt.Sprintf("the value held for %s of %s (%s) is not composed: it is not a value: %v",
|
||||
h.Module, h.Pull, h.Key, err))
|
||||
continue
|
||||
}
|
||||
m := &out.Machines[at]
|
||||
layers := make([]snapshot.Settings, len(m.Settings))
|
||||
copy(layers, m.Settings)
|
||||
found := false
|
||||
for i := range layers {
|
||||
if layers[i].Module == h.Module {
|
||||
values := make(map[string]any, len(layers[i].Values)+1)
|
||||
for k, v := range layers[i].Values {
|
||||
values[k] = v
|
||||
}
|
||||
values[h.Key] = value
|
||||
layers[i].Values = values
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
layers = append(layers, snapshot.Settings{Module: h.Module, Values: map[string]any{h.Key: value}})
|
||||
sort.Slice(layers, func(a, b int) bool { return layers[a].Module < layers[b].Module })
|
||||
}
|
||||
m.Settings = layers
|
||||
assigned := false
|
||||
for _, a := range m.Assigned {
|
||||
assigned = assigned || a == h.Module
|
||||
}
|
||||
by := ""
|
||||
if h.Approved != "" {
|
||||
by = ", approved by " + h.Approved
|
||||
}
|
||||
line := fmt.Sprintf("%s on %s %s=%s%s", h.Module, m.Described(), h.Key, h.Value, by)
|
||||
if !assigned {
|
||||
line += " (not assigned there in the snapshot)"
|
||||
}
|
||||
laid = append(laid, line)
|
||||
pulls[h.Pull] = true
|
||||
}
|
||||
if len(laid) > 0 {
|
||||
var names []string
|
||||
for p := range pulls {
|
||||
names = append(names, p)
|
||||
}
|
||||
sort.Strings(names)
|
||||
notes = append([]string{fmt.Sprintf("composed with %d value(s) held for %s: %s", len(laid), strings.Join(names, ", "),
|
||||
strings.Join(laid, "; "))}, notes...)
|
||||
}
|
||||
return out, notes
|
||||
}
|
||||
@@ -1,85 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
snapshot "git.novox.be/novox/mesh-sdk/go/facts"
|
||||
)
|
||||
|
||||
func heldGateFacts() snapshot.Facts {
|
||||
return snapshot.Facts{Machines: []snapshot.Machine{
|
||||
{Name: snapshot.Pseudonym("machine", "ace"), Roles: []string{"the build node"}, Assigned: []string{"n8n"},
|
||||
Settings: []snapshot.Settings{{Module: "n8n", Values: map[string]any{"port": 5678.0}}}},
|
||||
{Name: snapshot.Pseudonym("machine", "g14")},
|
||||
}}
|
||||
}
|
||||
|
||||
var heldShelf = map[string]catalogue.Manifest{"n8n": {Module: "n8n"}}
|
||||
|
||||
// 1826/AC3: a held value is laid in its node's layer of its module, beside what the layer holds, and said.
|
||||
func TestAHeldValueIsLaidOverItsNodesLayerAndSaid(t *testing.T) {
|
||||
f := heldGateFacts()
|
||||
got, notes := layHeld(f, []heldValue{{Pull: "novox/mesh-catalog#434", Node: "ace", Module: "n8n", Key: "media-gid",
|
||||
Value: []byte("2000"), Approved: "the operator"}}, heldShelf)
|
||||
layer := got.Machines[0].Settings[0].Values
|
||||
if layer["media-gid"] != 2000.0 || layer["port"] != 5678.0 {
|
||||
t.Fatalf("the layer holds %v", layer)
|
||||
}
|
||||
if _, has := f.Machines[0].Settings[0].Values["media-gid"]; has {
|
||||
t.Fatal("the snapshot given was changed")
|
||||
}
|
||||
if len(notes) != 1 || !strings.Contains(notes[0], "composed with 1 value(s) held for novox/mesh-catalog#434") ||
|
||||
!strings.Contains(notes[0], "media-gid=2000") || !strings.Contains(notes[0], "approved by the operator") ||
|
||||
strings.Contains(notes[0], "ace") {
|
||||
t.Fatalf("said %q", notes)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC3: a machine with no layer of the module gets one.
|
||||
func TestAHeldValueMakesTheLayerAMachineLacks(t *testing.T) {
|
||||
got, _ := layHeld(heldGateFacts(), []heldValue{{Pull: "p", Node: "g14", Module: "n8n", Key: "k", Value: []byte(`"x"`)}}, heldShelf)
|
||||
s := got.Machines[1].Settings
|
||||
if len(s) != 1 || s[0].Module != "n8n" || s[0].Values["k"] != "x" {
|
||||
t.Fatalf("layers %+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC3: a held value for a machine or a module the snapshot or the change does not hold is said, never laid.
|
||||
func TestAHeldValueForWhatIsNotHeldIsSaidNotLaid(t *testing.T) {
|
||||
f := heldGateFacts()
|
||||
got, notes := layHeld(f, []heldValue{
|
||||
{Pull: "p", Node: "nowhere", Module: "n8n", Key: "k", Value: []byte("1")},
|
||||
{Pull: "p", Node: "ace", Module: "ghost", Key: "k", Value: []byte("1")},
|
||||
}, heldShelf)
|
||||
if len(notes) != 2 || !strings.Contains(notes[0], "holds no machine") || !strings.Contains(notes[1], "defines no module ghost") {
|
||||
t.Fatalf("said %q", notes)
|
||||
}
|
||||
if _, has := got.Machines[0].Settings[0].Values["k"]; has {
|
||||
t.Fatal("laid")
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC3: none held lays nothing and says nothing; the file is read and refuses a value without a node.
|
||||
func TestNoHeldValuesLayNothingAndTheFileIsRead(t *testing.T) {
|
||||
if got, notes := layHeld(heldGateFacts(), nil, heldShelf); len(notes) != 0 || len(got.Machines) != 2 {
|
||||
t.Fatalf("%v %v", got, notes)
|
||||
}
|
||||
dir := t.TempDir()
|
||||
good := filepath.Join(dir, "good.json")
|
||||
os.WriteFile(good, []byte(`[{"pull":"p","node":"ace","module":"n8n","key":"k","value":1}]`), 0o644)
|
||||
if h, err := readHeld(good); err != nil || len(h) != 1 || h[0].Key != "k" {
|
||||
t.Fatalf("%v %v", h, err)
|
||||
}
|
||||
bad := filepath.Join(dir, "bad.json")
|
||||
os.WriteFile(bad, []byte(`[{"pull":"p","module":"n8n","key":"k","value":1}]`), 0o644)
|
||||
if _, err := readHeld(bad); err == nil {
|
||||
t.Fatal("a value without a node was read")
|
||||
}
|
||||
if h, err := readHeld(""); h != nil || err != nil {
|
||||
t.Fatalf("%v %v", h, err)
|
||||
}
|
||||
}
|
||||
@@ -526,57 +526,3 @@ func TestAMemberJudgedAloneSeesAMoveThroughTheClonesBeside(t *testing.T) {
|
||||
t.Errorf("nothing beside, yet moves %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC3 (integration level, a throwaway store): a change whose container says ${config:media-gid} fails the gate for
|
||||
// want of the value, passes it with the value the operator approved held for its pull request, and the mesh as it is
|
||||
// composes the same either way.
|
||||
func TestAChangeNeedingAHeldValueFailsWithoutItAndPassesWithIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
trunk := `{"module":"n8n","version":"1","resources":[{"id":"server","type":"container","name":"n8n",
|
||||
"image":"n8nio/n8n@sha256:4846eb2f4b874ab04cde7fc1e249d2ddaec66e9aea64439beb2972cfea88e3c0"}]}`
|
||||
m, err := catalogue.ParseManifest([]byte(trunk))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog",
|
||||
Path: "modules/n8n", BuiltFrom: "c0ffee"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "n8n"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := gatherFacts(ctx, open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := range f.Machines {
|
||||
f.Machines[i].Capabilities = append(f.Machines[i].Capabilities, snapshot.Capability{Name: "container-groups", Present: true})
|
||||
}
|
||||
tree := aTree(t, map[string]string{"n8n": n8nWithAGroup})
|
||||
judge := func(held []heldValue) mergeVerdict {
|
||||
v, err := judgeChange(t.Context(), mergeCheckInput{facts: f, admin: os.Getenv("MESH_TEST_POSTGRES"),
|
||||
repository: "novox/mesh-catalog", tree: tree, changed: []string{"modules/n8n/module.json"}, held: held})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
without := judge(nil)
|
||||
if without.Verdict != "fail" || !strings.Contains(without.Report(), "media-gid") {
|
||||
t.Fatalf("a required value nothing sets passed, or failed for another reason:\n%s", without.Report())
|
||||
}
|
||||
with := judge([]heldValue{{Pull: "novox/mesh-catalog#434", Node: "laptop", Module: "n8n", Key: "media-gid",
|
||||
Value: []byte("2000"), Approved: "the operator"}})
|
||||
if with.Verdict == "fail" || with.Verdict == "error" {
|
||||
t.Fatalf("the held value did not make it compose:\n%s", with.Report())
|
||||
}
|
||||
if !strings.Contains(with.Report(), "composed with 1 value(s) held for novox/mesh-catalog#434") {
|
||||
t.Fatalf("the value laid is not said:\n%s", with.Report())
|
||||
}
|
||||
for i := range with.Machines {
|
||||
if with.Machines[i].Base.Composes != without.Machines[i].Base.Composes {
|
||||
t.Fatalf("the mesh as it is composes differently with a held value: %+v / %+v", with.Machines[i].Base, without.Machines[i].Base)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0505 §5: a delivery whose cure is its author's new head reaches the operator in words that name its
|
||||
// author and say a new head is needed, offer no Stop and never call it the operator's; the line as mesh-delivery-gate
|
||||
// says it (ready.go readyLines: needs "new-head", author, branch).
|
||||
func TestADeliveryThatNeedsANewHeadNamesItsAuthorAndOffersNoStop(t *testing.T) {
|
||||
l := stalledLine{ID: "novox/mesh-controller@499ff5c21861", Number: 432, State: "rejected", For: "2h1m0s", Bound: "2h0m0s",
|
||||
H2: "none: a new head from its author, or a recheck that passes", Needs: needsNewHead,
|
||||
Author: "the author of feat/quiet-deliveries (quiet-deliveries on shanks)", Branch: "feat/quiet-deliveries",
|
||||
Says: "novox/mesh-controller#432 passed its checks at 499ff5c21861 and is not merged: a conflict"}
|
||||
o := stalledObservations([]stalledLine{l})[0]
|
||||
for _, words := range []string{o.Headline, o.Explanation, o.Resolved, o.Needs} {
|
||||
if why, ok := conditions.Plain(words); !ok {
|
||||
t.Errorf("%q is not plain: %s", words, why)
|
||||
}
|
||||
for _, never := range []string{"stop it", "operator's"} {
|
||||
if strings.Contains(words, never) {
|
||||
t.Errorf("%q says %q", words, never)
|
||||
}
|
||||
}
|
||||
}
|
||||
if o.Headline != "Pull request 432 of mesh-controller needs a new head from its author" ||
|
||||
!strings.Contains(o.Explanation, "until the author of feat/quiet-deliveries (quiet-deliveries on shanks) pushes a new head") ||
|
||||
!strings.Contains(o.Explanation, "for 2 hours") || len(o.Actions) != 0 ||
|
||||
o.Needs != "ask the author of feat/quiet-deliveries (quiet-deliveries on shanks) for a new head." {
|
||||
t.Fatalf("words: %q / %q / %q / %v", o.Headline, o.Explanation, o.Needs, o.Actions)
|
||||
}
|
||||
// A branch in code is not said; the author is then "its author".
|
||||
l.Author = "the author of fix/a1b2c3d4e5f6"
|
||||
if o := stalledObservations([]stalledLine{l})[0]; !strings.Contains(o.Needs, "ask its author") {
|
||||
t.Fatalf("a dotted branch was said: %q", o.Needs)
|
||||
}
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// 1617/AC2 (unit): a new module whose first build a merge's hearing asked is not asked again when its batch is cut,
|
||||
// and one whose ask failed, was never handed over or is long lost is.
|
||||
func TestAModuleWhoseBuildIsComingIsNotAskedAgain(t *testing.T) {
|
||||
now := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)
|
||||
source := buildSource{Repository: "http://forge/novox/mesh-catalog.git", Seat: "git"}
|
||||
asked := func(path string, mod func(*inventory.RequestOutcome)) inventory.RequestOutcome {
|
||||
r := inventory.RequestOutcome{BuildRequest: inventory.BuildRequest{ID: "b-" + path, Repository: source.Repository,
|
||||
Seat: "git", Path: path, For: "merge", At: now.Add(-3 * time.Minute)}}
|
||||
if mod != nil {
|
||||
mod(&r)
|
||||
}
|
||||
return r
|
||||
}
|
||||
requests := []inventory.RequestOutcome{
|
||||
asked("modules/go", nil),
|
||||
asked("modules/failed", func(r *inventory.RequestOutcome) { r.Heard, r.Failed = true, "no module.json" }),
|
||||
asked("modules/notasked", func(r *inventory.RequestOutcome) { r.NotAsked = "the bus is silent" }),
|
||||
asked("modules/lost", func(r *inventory.RequestOutcome) { r.At = now.Add(-2 * time.Hour) }),
|
||||
asked("modules/byhand", func(r *inventory.RequestOutcome) { r.For = "build" }),
|
||||
}
|
||||
got := unaskedDirs([]string{"modules/go", "modules/failed", "modules/notasked", "modules/lost", "modules/byhand",
|
||||
"modules/nodejs"}, requests, source, now)
|
||||
want := []string{"modules/failed", "modules/notasked", "modules/lost", "modules/byhand", "modules/nodejs"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("asked again %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
@@ -100,38 +99,3 @@ func TestAMergeBuildsItsNewModuleBesideItsPlan(t *testing.T) {
|
||||
t.Fatalf("a merge adding no module asked for %v", *asked)
|
||||
}
|
||||
}
|
||||
|
||||
// 1617/AC1 (integration): a merge adding a module to a repository the mesh builds modules from, heard while a batch of
|
||||
// that repository is still assembling (its window running), asks the module's first build when it is heard, so the
|
||||
// module is registered while the batch waits; cutting the batch asks it no second time (1617/AC2).
|
||||
func TestAMergeAddingAModuleAsksItsBuildBeforeItsBatchIsCut(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "pr/c1")
|
||||
asked := asksWithPaths(t)
|
||||
catalog := func(commit string, made time.Time, paths ...string) link.SourceMoved {
|
||||
return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit,
|
||||
Head: "pr/" + commit, MergedAt: made.Format(time.RFC3339Nano), Paths: paths}
|
||||
}
|
||||
hear(t, open, catalog("c1", t0, "modules/app/index.ts"), t0)
|
||||
if len(*asked) != 0 {
|
||||
t.Fatalf("a merge adding no module asked %v", *asked)
|
||||
}
|
||||
hear(t, open, lone(catalog("c2", t0.Add(10*time.Second), "modules/go/packages.json", "modules/go/module.json")), t0.Add(10*time.Second))
|
||||
if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 1 {
|
||||
t.Fatalf("the batch should still be assembling: %d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
want := [3]string{"novox/mesh-catalog", "modules/go", "main"}
|
||||
if len(*asked) != 1 || (*asked)[0] != want {
|
||||
t.Fatalf("heard with its batch assembling, the merge asked %v, not %v", *asked, want)
|
||||
}
|
||||
cutAt(t, open, t0.Add(10*time.Second+90*time.Second))
|
||||
n := 0
|
||||
for _, a := range *asked {
|
||||
if a[1] == "modules/go" {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("modules/go was asked %d times: %v", n, *asked)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,198 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// heldElsewhere is the plans held by another act for longer than any test runs: waiting for them ends only with the
|
||||
// context, and a look that does not wait is told at once that they are busy.
|
||||
func heldElsewhere(ctx context.Context, wait bool) (func(), error) {
|
||||
if !wait {
|
||||
return nil, inventory.ErrPlansBusy
|
||||
}
|
||||
<-ctx.Done()
|
||||
return nil, inventory.ErrPlansBusy
|
||||
}
|
||||
|
||||
// 1708/AC1: given the plans held by another act, when the event loop takes a build's outcome, then it returns at
|
||||
// once, without waiting the plans' two minutes (on 2026-10-21 at 14:20:45 nodered's outcome held the loop until
|
||||
// 14:22:45, and the operator was told the controller stopped listening).
|
||||
func TestAnOutcomeDoesNotWaitForThePlansAnotherActHolds(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)
|
||||
defer cancel()
|
||||
start := time.Now()
|
||||
release, err := holdPlansForOutcome(ctx, heldElsewhere)
|
||||
if took := time.Since(start); took > time.Second {
|
||||
t.Fatalf("taking an outcome waited %s for plans another act holds; the event loop takes nothing meanwhile", took)
|
||||
}
|
||||
if err == nil || release != nil {
|
||||
t.Fatalf("plans another act holds were said to be held: release %v, err %v", release != nil, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1708/AC4: given the plans free, when the event loop takes a build's outcome, then it holds them and marks the plan
|
||||
// from the outcome directly, as before.
|
||||
func TestAnOutcomeHoldsThePlansWhenTheyAreFree(t *testing.T) {
|
||||
let := false
|
||||
release, err := holdPlansForOutcome(t.Context(), func(context.Context, bool) (func(), error) {
|
||||
return func() { let = true }, nil
|
||||
})
|
||||
if err != nil || release == nil {
|
||||
t.Fatalf("free plans were not held: %v", err)
|
||||
}
|
||||
release()
|
||||
if !let {
|
||||
t.Fatal("the hold given was not the one taken")
|
||||
}
|
||||
}
|
||||
|
||||
// withLooker installs a looker that is never run, and answers it, so a test can read whether a look was asked.
|
||||
func withLooker(t *testing.T) *looker {
|
||||
t.Helper()
|
||||
l := newLooker(func(context.Context) bool { return false })
|
||||
lookerMu.Lock()
|
||||
before := theLooker
|
||||
theLooker = l
|
||||
lookerMu.Unlock()
|
||||
t.Cleanup(func() {
|
||||
lookerMu.Lock()
|
||||
theLooker = before
|
||||
lookerMu.Unlock()
|
||||
})
|
||||
return l
|
||||
}
|
||||
|
||||
// 1708/AC2: given an outcome left to the build records because the plans were held, when it is left, then the look
|
||||
// at the plans is asked for, so the records settle it once the holder lets go rather than at the next event.
|
||||
func TestAnOutcomeLeftToTheRecordsAsksForTheLook(t *testing.T) {
|
||||
l := withLooker(t)
|
||||
leaveToRecords("nodered", "build-1708-ac2", "", "", inventory.ErrPlansBusy)
|
||||
select {
|
||||
case <-l.signal:
|
||||
default:
|
||||
t.Fatal("an outcome left to the build records asked for no look: the plan waits for whatever event comes next")
|
||||
}
|
||||
}
|
||||
|
||||
// 1708/AC3: given an outcome left to the records that said what the build did (ADR 0350 §4), when the look settles
|
||||
// the plan from the build records by that build's id, then the module keeps those words, once.
|
||||
func TestTheRecordsSettleAnOutcomeWithWhatTheBuildDid(t *testing.T) {
|
||||
withLooker(t)
|
||||
asked := time.Date(2026, 10, 21, 12, 15, 0, 0, time.UTC)
|
||||
plan := func() inventory.Plan {
|
||||
return inventory.Plan{ID: "plan-1708", Tiers: [][]string{{"nodered"}},
|
||||
Modules: map[string]*inventory.PlanModule{"nodered": {State: "asked", AskedAt: &asked, Build: "build-1708-ac3"}}}
|
||||
}
|
||||
byID := map[string]inventory.Build{"build-1708-ac3": {ID: "build-1708-ac3", Module: "nodered", Commit: "4efac366",
|
||||
At: asked.Add(5 * time.Minute)}}
|
||||
leaveToRecords("nodered", "build-1708-ac3", "reused from build-1708-check", "", inventory.ErrPlansBusy)
|
||||
|
||||
p := plan()
|
||||
if !settleFromRecords(&p, p.Tiers[0], nil, byID) || p.Modules["nodered"].State != "built" {
|
||||
t.Fatalf("the records did not settle the outcome: %+v", p.Modules["nodered"])
|
||||
}
|
||||
if got := p.Modules["nodered"].Made; got != "reused from build-1708-check" {
|
||||
t.Fatalf("settled from the records, the module says %q of what its build did, not the outcome's words", got)
|
||||
}
|
||||
again := plan()
|
||||
settleFromRecords(&again, again.Tiers[0], nil, byID)
|
||||
if got := again.Modules["nodered"].Made; got != "" {
|
||||
t.Fatalf("an outcome's words were kept for a second settle: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1708/AC5: given a serving controller (its look at the plans running), when an outcome was taken into the plans,
|
||||
// then moving them (a release's sends, which may retry for minutes) is left to the look, off the event loop
|
||||
// (2026-10-21 14:28:40 to 14:32:10: the loop sent okioki's release to g14 after mesh-delivery-builder's outcome).
|
||||
func TestAnOutcomeLeavesMovingThePlansToTheLook(t *testing.T) {
|
||||
l := withLooker(t)
|
||||
moved := false
|
||||
advanceAfterOutcome(func() { moved = true })
|
||||
if moved {
|
||||
t.Fatal("the plans were moved on the event loop after an outcome, where a send that retries holds every message")
|
||||
}
|
||||
select {
|
||||
case <-l.signal:
|
||||
default:
|
||||
t.Fatal("the look was not asked to move the plans after an outcome")
|
||||
}
|
||||
}
|
||||
|
||||
// 1708/AC5: given no look running (a command, which has no event loop), when an outcome was taken, then the plans are
|
||||
// moved at once, as before.
|
||||
func TestAnOutcomeMovesThePlansWhereNoLookRuns(t *testing.T) {
|
||||
lookerMu.Lock()
|
||||
before := theLooker
|
||||
theLooker = nil
|
||||
lookerMu.Unlock()
|
||||
t.Cleanup(func() {
|
||||
lookerMu.Lock()
|
||||
theLooker = before
|
||||
lookerMu.Unlock()
|
||||
})
|
||||
moved := false
|
||||
advanceAfterOutcome(func() { moved = true })
|
||||
if !moved {
|
||||
t.Fatal("with no look running, an outcome left the plans unmoved")
|
||||
}
|
||||
}
|
||||
|
||||
// 1708/AC6: given a registration the mesh refused (its build recorded as worked, the outcome failed), left to the
|
||||
// records because the plans were held, when the look settles the plan from the records, then the module and the plan
|
||||
// are failed, never built: settled as built, the tier would be sent the version from before (reviewer-1708@shanks).
|
||||
func TestARefusedRegistrationLeftToTheRecordsFailsThePlan(t *testing.T) {
|
||||
withLooker(t)
|
||||
asked := time.Date(2026, 10, 21, 12, 15, 0, 0, time.UTC)
|
||||
p := inventory.Plan{ID: "plan-1708-ac6", State: inventory.PlanBuilding, Tiers: [][]string{{"nodered"}},
|
||||
Modules: map[string]*inventory.PlanModule{"nodered": {State: "asked", AskedAt: &asked, Build: "build-1708-ac6"}}}
|
||||
byID := map[string]inventory.Build{"build-1708-ac6": {ID: "build-1708-ac6", Module: "nodered", Commit: "4efac366",
|
||||
At: asked.Add(5 * time.Minute)}}
|
||||
leaveToRecords("nodered", "build-1708-ac6", "", "the mesh does not register it: a port is taken", inventory.ErrPlansBusy)
|
||||
|
||||
overlayRefusals(nil, byID, refusalsKeptFor([]string{"build-1708-ac6"}))
|
||||
settleFromRecords(&p, p.Tiers[0], nil, byID)
|
||||
if s := p.Modules["nodered"]; s.State != "failed" || p.State != inventory.PlanFailed {
|
||||
t.Fatalf("a refused registration left to the records settled as %q, the plan %q", s.State, p.State)
|
||||
}
|
||||
if !strings.Contains(p.Modules["nodered"].Why, "a port is taken") {
|
||||
t.Fatalf("the refusal's words were lost: %q", p.Modules["nodered"].Why)
|
||||
}
|
||||
}
|
||||
|
||||
// 1708/AC6: a refusal kept in the store (a controller replaced before the settle) fails the record as one kept in
|
||||
// the process does.
|
||||
func TestARefusalFromTheStoreFailsTheRecord(t *testing.T) {
|
||||
recorded := map[string][]inventory.Build{"nodered": {{ID: "b-store", Module: "nodered"}, {ID: "b-other", Module: "nodered"}}}
|
||||
byID := map[string]inventory.Build{"b-store": {ID: "b-store", Module: "nodered"}}
|
||||
overlayRefusals(recorded, byID, map[string]string{"b-store": "not registered"})
|
||||
if recorded["nodered"][0].Worked() || byID["b-store"].Worked() {
|
||||
t.Fatal("a build whose registration was refused still reads as worked")
|
||||
}
|
||||
if !recorded["nodered"][1].Worked() {
|
||||
t.Fatal("a build nobody refused was failed")
|
||||
}
|
||||
}
|
||||
|
||||
// 1708/AC7: given the records settle a module as deleted at its source, or fail the plan, when the look settles, then
|
||||
// it does what taking the outcome directly does: the deleted module is named to be forgotten, and the plan's failure
|
||||
// is said (reviewer-1708@shanks).
|
||||
func TestSettlingFromTheRecordsDoesWhatTakingTheOutcomeDoes(t *testing.T) {
|
||||
p := inventory.Plan{State: inventory.PlanFailed, Modules: map[string]*inventory.PlanModule{
|
||||
"gone": {State: planDeleted}, "x": {State: "failed"}, "y": {State: "built"}}}
|
||||
before := map[string]string{"gone": "asked", "x": "asked", "y": "built"}
|
||||
deleted, failed := settledEffects(before, &p, inventory.PlanBuilding)
|
||||
if len(deleted) != 1 || deleted[0] != "gone" {
|
||||
t.Fatalf("the module the records found deleted is not forgotten: %v", deleted)
|
||||
}
|
||||
if !failed {
|
||||
t.Fatal("the plan the records failed is not said as failed")
|
||||
}
|
||||
if d, f := settledEffects(map[string]string{"gone": planDeleted}, &p, inventory.PlanFailed); len(d) != 0 || f {
|
||||
t.Fatalf("what was deleted or failed before the settle is acted on again: %v %v", d, f)
|
||||
}
|
||||
}
|
||||
@@ -195,29 +195,6 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
||||
"there yet is sealed by the controller until the vault holds it.", m, m),
|
||||
Resolved: "Resolved: the vault gave " + m + " the core's credentials"}
|
||||
}),
|
||||
kindFilesNotPut: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "The big files for " + m + " cannot be put on the bus",
|
||||
Needs: "look at the bus's permissions for the controller; the details give the reason.",
|
||||
Explanation: fmt.Sprintf("The mesh could not put the large files of %s's instructions in their place on the bus, "+
|
||||
"send after send, so %s is sent them inside the instructions instead, which may make them too big.", m, m),
|
||||
Resolved: "Resolved: the big files for " + m + " reach the bus again"}
|
||||
}),
|
||||
kindDeclarationNearLimit: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
if o.Severity == conditions.Urgent {
|
||||
return words{Headline: m + " cannot be sent its instructions",
|
||||
Needs: "make the instructions for " + m + " smaller, for example by taking a module off it.",
|
||||
Explanation: fmt.Sprintf("The instructions for %s have grown past the largest message the bus takes, so "+
|
||||
"nothing new reaches %s until they are smaller.", m, m),
|
||||
Resolved: "Resolved: the instructions for " + m + " fit the bus again"}
|
||||
}
|
||||
return words{Headline: "The instructions for " + m + " are nearly too big",
|
||||
Needs: "do not add more to " + m + " until this is dealt with; the details give the size.",
|
||||
Explanation: fmt.Sprintf("The instructions for %s are close to the largest message the bus takes. "+
|
||||
"One more module or secret may stop anything new from reaching %s.", m, m),
|
||||
Resolved: "Resolved: the instructions for " + m + " have room again"}
|
||||
}),
|
||||
kindCoreDowngrade: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
component := idPart(o, 0)
|
||||
@@ -457,18 +434,6 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
||||
Explanation: "A delivery did not finish. Nothing is tried again until you say so.",
|
||||
Resolved: "Resolved: the delivery is no longer failed"}
|
||||
}),
|
||||
kindMovesHeld: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "An update waits for you",
|
||||
Explanation: "A new version of a module would restart a container or touch its data, so it waits for you.",
|
||||
Resolved: "Resolved: the update is no longer waiting"}
|
||||
}),
|
||||
kindImageMissing: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A module is not updated",
|
||||
Explanation: "The new version of a module names a container image that the mesh's store does not hold, so it " +
|
||||
"was not sent. What runs is untouched.",
|
||||
Needs: "build the module again; the details name the image.",
|
||||
Resolved: "Resolved: the store holds the image again"}
|
||||
}),
|
||||
"ask-lost": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A build has no result",
|
||||
Explanation: "A build was asked for and never answered. It is asked again with the next delivery.",
|
||||
@@ -966,9 +931,6 @@ func stalledWords(l stalledLine, o conditions.Observation) (headline, explanatio
|
||||
if l.State == "unanswered" {
|
||||
return unansweredWords(l, o)
|
||||
}
|
||||
if l.Needs == needsNewHead {
|
||||
return newHeadWords(l, name)
|
||||
}
|
||||
held := l.State
|
||||
if held == "" {
|
||||
held = "held"
|
||||
@@ -1002,29 +964,6 @@ func stalledWords(l stalledLine, o conditions.Observation) (headline, explanatio
|
||||
fmt.Sprintf("Delivery of %s is no longer %s", name, held), needs, actions
|
||||
}
|
||||
|
||||
// newHeadWords are the words of a delivery whose cure is its author's new head (novox/hq ADR 0505 §5): it names the
|
||||
// author and says a new head is needed; it offers no Stop and never calls it the operator's, because the operator
|
||||
// hears of it only once the author's own watch has failed, after two hours with no new head.
|
||||
func newHeadWords(l stalledLine, name string) (headline, explanation, resolved, needs string, actions []conditions.Action) {
|
||||
pull := "A pull request of " + name
|
||||
if l.Number > 0 {
|
||||
pull = fmt.Sprintf("Pull request %d of %s", l.Number, name)
|
||||
}
|
||||
author := l.Author
|
||||
if _, plain := conditions.Plain(author); !plain {
|
||||
author = "its author" // a branch named in code (a dotted name, a number) is not said
|
||||
}
|
||||
long := "for a long time"
|
||||
if d, err := time.ParseDuration(l.For); err == nil {
|
||||
long = "for " + humanDuration(d)
|
||||
}
|
||||
return pull + " needs a new head from its author",
|
||||
fmt.Sprintf("%s cannot merge until %s pushes a new head: its check failed after its Approve, and no new head "+
|
||||
"came %s.", pull, author, long),
|
||||
pull + " has a new head, or is closed",
|
||||
"ask " + author + " for a new head.", nil
|
||||
}
|
||||
|
||||
// stoppedNeeds is what ends a delivery that is already stopped, from the stalled line's healer words ("none: a
|
||||
// merge that delivers its modules ends it — the operator's"): the part that says what ends it.
|
||||
func stoppedNeeds(l stalledLine) string {
|
||||
|
||||
@@ -260,7 +260,7 @@ func TestUpdatesHeldNameWhatWaitsAndWhereItIsReleased(t *testing.T) {
|
||||
plainExample(t, got[0], "Updates of openrazer and sensors wait for your release",
|
||||
"Needs you: release them from the mesh MCP server; this notification cannot do it. Updates of openrazer and "+
|
||||
"sensors on g14 and shanks wait for a person to release them, because the last walk failed. "+
|
||||
"They are not delivered until then, and stay held if you leave them.", "Release")
|
||||
"They are not delivered until then, and stay held if you leave them.")
|
||||
}
|
||||
|
||||
// **What held them is said in the glossary's words** (2026-10-08 review): the backlog is held after any
|
||||
|
||||
@@ -78,50 +78,3 @@ func TestAPlansLineCountsFromItsTierNotFromItsLastSave(t *testing.T) {
|
||||
|
||||
// novox/hq issue 296: a plan whose tier stands still is not written again by every advance — its revision,
|
||||
// its last save and its `plan-moved` stay where its last change left them.
|
||||
|
||||
// novox/hq issue 920: a building plan whose current tier is all built waits on its note (a send refused at the
|
||||
// gate), and its line, `status` and the stalled condition say that, not "building for ... LATE".
|
||||
func TestAPlanLineSaysTheRefusalOnceItsTierIsBuilt(t *testing.T) {
|
||||
now := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC)
|
||||
built := now.Add(-time.Hour)
|
||||
refusal := "tier 1: a build waits there for its gate: novox (mesh-delivery-indexer d68d7ef3→a5aa69a4) — tried again"
|
||||
mk := func(second string) inventory.Plan {
|
||||
return inventory.Plan{ID: "plan-r", Repository: "novox/mesh-sdk", Commit: "b2f5ebb2", State: inventory.PlanBuilding,
|
||||
Tier: 1, Created: now.Add(-2 * time.Hour), Updated: now, TierEntered: now.Add(-time.Hour), Note: refusal,
|
||||
Tiers: [][]string{{"a"}, {"b", "c"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "built", BuiltAt: &built},
|
||||
"b": {State: "built", BuiltAt: &built}, "c": {State: second}}}
|
||||
}
|
||||
|
||||
// 920/AC1
|
||||
p := mk("built")
|
||||
line := planLine(p, now)
|
||||
if !strings.Contains(line, "a build waits there for its gate: novox") || strings.Contains(line, "building for") {
|
||||
t.Errorf("920/AC1: a plan with its tier built and a refusal reads %q", line)
|
||||
}
|
||||
st := planStatuses([]inventory.Plan{p}, now, pauseView{}, nil)
|
||||
if len(st) != 1 || !st[0].Late {
|
||||
t.Errorf("920/AC1: status: %+v", st)
|
||||
}
|
||||
|
||||
// 920/AC2
|
||||
if line := planLine(mk("asked"), now); !strings.Contains(line, "building for 1h0m0s") {
|
||||
t.Errorf("920/AC2: a plan with a tier module not built reads %q", line)
|
||||
}
|
||||
}
|
||||
|
||||
// 920/AC3: a tier move writes the previous tier's summary into the note; with the new tier all built, the line still
|
||||
// says "building for", for that note is no refusal.
|
||||
func TestAPlanLineDoesNotHoldOnATiersSummary(t *testing.T) {
|
||||
now := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC)
|
||||
built := now.Add(-time.Hour)
|
||||
for _, note := range []string{"failed or refused its send (apply-failing says it): novox", "tier 0 built; sent mesh-sdk to novox first, in one send"} {
|
||||
p := inventory.Plan{ID: "plan-s", Repository: "novox/mesh-sdk", Commit: "b2f5ebb2", State: inventory.PlanBuilding,
|
||||
Tier: 1, Created: now.Add(-2 * time.Hour), Updated: now, TierEntered: now.Add(-time.Hour), Note: note,
|
||||
Tiers: [][]string{{"a"}, {"b"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "built", BuiltAt: &built}, "b": {State: "built", BuiltAt: &built}}}
|
||||
if line := planLine(p, now); !strings.Contains(line, "building for 1h0m0s") || strings.Contains(line, "held") {
|
||||
t.Errorf("920/AC3: a plan with the note %q reads %q", note, line)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -136,18 +136,6 @@ func askToLook() {
|
||||
}
|
||||
}
|
||||
|
||||
// askedToLook asks the serving controller's look at the plans, and says whether there was one to ask.
|
||||
func askedToLook() bool {
|
||||
lookerMu.RLock()
|
||||
l := theLooker
|
||||
lookerMu.RUnlock()
|
||||
if l == nil {
|
||||
return false
|
||||
}
|
||||
l.ask()
|
||||
return true
|
||||
}
|
||||
|
||||
// startPlanLooks starts the look at the open plans, and asks for the first: the controller took the lease, and a
|
||||
// plan left by a replaced controller resumes from there (ADR 0406 §6). It replaces the rollout's 30 second ticker.
|
||||
// It answers a channel that is closed once the look has returned after the context ended, for a test that restores what
|
||||
@@ -192,7 +180,7 @@ func startRolloutEvents(ctx context.Context, open *stores, js *broker.JetStream)
|
||||
if err := startProvAppliedLooks(ctx, js); err != nil {
|
||||
fmt.Printf("plans: the provisioner clients' words could not be watched (%v); a walk waits for them until the next look\n", err)
|
||||
}
|
||||
if err := startPlanMovedLooks(ctx, js, open); err != nil {
|
||||
if err := startPlanMovedLooks(ctx, js); err != nil {
|
||||
fmt.Printf("plans: the plans moved by other processes could not be followed (%v)\n", err)
|
||||
}
|
||||
done := startPlanLooks(ctx, open)
|
||||
@@ -206,13 +194,8 @@ func servingPlanSaved(ctx context.Context, bus link.Bus, open *stores) func(*inv
|
||||
return func(_ *inventory.Inventory, p inventory.Plan) {
|
||||
sayPlanMoved(ctx, bus, open.inventory, p)
|
||||
planKept(p)
|
||||
keepBuildingKey(ctx, p)
|
||||
if p.WalkSaid == inventory.WalkEnded {
|
||||
go forgetChecksOfWalk(context.WithoutCancel(ctx), open.inventory, p)
|
||||
// A first build refused for a word only a newer build-agent reads is asked again (novox/hq issue 1173).
|
||||
if deliversBuildAgent(p) {
|
||||
go askHeldFirstBuildsAgain(context.WithoutCancel(ctx), open, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -567,7 +550,7 @@ func startProvAppliedLooks(ctx context.Context, js *broker.JetStream) error {
|
||||
// `plan-moved`, so a plan a person moved with a verb at the control node's terminal — a process this one did not spawn,
|
||||
// whose saves it does not hear — is looked at all the same (novox/hq ADR 0406 §2, §5, issue 974). New messages only; the
|
||||
// first look reads the rest. A look its own save asks is the same look: asks are one.
|
||||
func startPlanMovedLooks(ctx context.Context, js *broker.JetStream, open *stores) error {
|
||||
func startPlanMovedLooks(ctx context.Context, js *broker.JetStream) error {
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -582,18 +565,7 @@ func startPlanMovedLooks(ctx context.Context, js *broker.JetStream, open *stores
|
||||
if err != nil {
|
||||
return fmt.Errorf("following the plans moved: %w", err)
|
||||
}
|
||||
var bus link.Bus
|
||||
if core, err := js.Conn().JetStream(); err == nil {
|
||||
bus = link.OverNATS{Conn: js.Conn(), JS: core}
|
||||
}
|
||||
consuming, err := consumer.Consume(func(m jetstream.Msg) {
|
||||
askToLook()
|
||||
if bus != nil && open != nil {
|
||||
// A walk's start or end, said by this controller or a verb: said again with the machines whose
|
||||
// declaration it changes (novox/hq ADR 0505 §2, walk_declared.go).
|
||||
go sayDeclared(context.WithoutCancel(ctx), bus, open, m.Data())
|
||||
}
|
||||
})
|
||||
consuming, err := consumer.Consume(func(jetstream.Msg) { askToLook() })
|
||||
if err != nil {
|
||||
return fmt.Errorf("following the plans moved: %w", err)
|
||||
}
|
||||
|
||||
@@ -174,7 +174,7 @@ func TestAPlanMovedByAnotherProcessAsksForALook(t *testing.T) {
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
go l.run(ctx)
|
||||
if err := startPlanMovedLooks(ctx, js, nil); err != nil {
|
||||
if err := startPlanMovedLooks(ctx, js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
legacy, err := js.Conn().JetStream()
|
||||
|
||||
@@ -31,8 +31,7 @@ func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) {
|
||||
return err
|
||||
}
|
||||
text := string(raw)
|
||||
// The regular expression is the costly part of reading every source: only a file that names the table can match.
|
||||
if strings.Contains(strings.ToLower(text), "release_plan") && write.MatchString(text) {
|
||||
if write.MatchString(text) {
|
||||
found = append(found, path+" writes the plan table")
|
||||
}
|
||||
// A plan record is written by the bucket's raw create and update, which only the plans' own file calls.
|
||||
|
||||
@@ -52,7 +52,7 @@ func TestASharedRepositoryIsPlannedFromTheRecordedBuildSources(t *testing.T) {
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: append([]string{"cmd/mesh-builder/", "internal/broker/"}, gomod...)}}
|
||||
sourcesOf["route-proxy"] = []inventory.BuildSource{
|
||||
{Paths: []string{"modules/route-proxy/Dockerfile", "modules/route-proxy/module.json"}},
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: append([]string{"modules/route-proxy/cmd/route-proxy/", "internal/broker/"}, gomod...)}}
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: append([]string{"examples/route-proxy/", "internal/broker/"}, gomod...)}}
|
||||
register(catalogue.Manifest{Module: "mesh-controller", Version: "1"}, "novox/mesh-controller", "", nil, nil)
|
||||
register(agent, "novox/mesh-catalog", "modules/build-agent", nil, controllerRead)
|
||||
register(catalogue.Manifest{Module: "route-proxy", Version: "1"}, "novox/mesh-catalog", "modules/route-proxy", nil, controllerRead)
|
||||
@@ -133,7 +133,7 @@ func TestASharedRepositoryIsPlannedFromTheRecordedBuildSources(t *testing.T) {
|
||||
{"a package only the controller builds from: the controller alone", "mesh-controller",
|
||||
[]string{"internal/conditions/condition.go"}, "mesh-controller", true},
|
||||
{"the route proxy's program: the route proxy alone", "mesh-controller",
|
||||
[]string{"modules/route-proxy/cmd/route-proxy/main.go"}, "route-proxy", true},
|
||||
[]string{"examples/route-proxy/main.go"}, "route-proxy", true},
|
||||
{"a package all three build from: all three", "mesh-controller",
|
||||
[]string{"internal/broker/broker.go"}, "mesh-controller,route-proxy | build-agent", false},
|
||||
{"the route proxy's directory in the catalogue: it alone", "mesh-catalog",
|
||||
|
||||
@@ -259,7 +259,7 @@ func TestASharedRepositoryMovesOnlyWhatItsBuildSourceHolds(t *testing.T) {
|
||||
{Own: true, Paths: []string{"modules/build-agent/Dockerfile", "modules/build-agent/module.json"}},
|
||||
},
|
||||
"route-proxy": {
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: with("modules/route-proxy/cmd/route-proxy/", "internal/broker/",
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: with("examples/route-proxy/", "internal/broker/",
|
||||
"vendor/github.com/nats-io/nats.go/")},
|
||||
{Own: true, Paths: []string{"modules/route-proxy/Dockerfile", "modules/route-proxy/module.json"}},
|
||||
},
|
||||
@@ -288,7 +288,7 @@ func TestASharedRepositoryMovesOnlyWhatItsBuildSourceHolds(t *testing.T) {
|
||||
{"a README of the controller's repository: no module (338)", "mesh-controller",
|
||||
[]string{"README.md"}, read, "", "README.md"},
|
||||
{"the route proxy's program alone: the route proxy alone (338)", "mesh-controller",
|
||||
[]string{"modules/route-proxy/cmd/route-proxy/main.go"}, read, "route-proxy", ""},
|
||||
[]string{"examples/route-proxy/main.go"}, read, "route-proxy", ""},
|
||||
{"the build seat's program alone: its holder alone", "mesh-controller",
|
||||
[]string{"cmd/mesh-builder/main.go"}, read, "build-agent", ""},
|
||||
{"a package the build seat's program and the controller build from: both", "mesh-controller",
|
||||
@@ -305,7 +305,7 @@ func TestASharedRepositoryMovesOnlyWhatItsBuildSourceHolds(t *testing.T) {
|
||||
{"go.sum: all three", "mesh-controller",
|
||||
[]string{"go.sum"}, read, "mesh-controller,route-proxy | build-agent", ""},
|
||||
{"a file added to the proxy's package: the proxy", "mesh-controller",
|
||||
[]string{"modules/route-proxy/cmd/route-proxy/new.go"}, read, "route-proxy", ""},
|
||||
[]string{"examples/route-proxy/new.go"}, read, "route-proxy", ""},
|
||||
// Before any build source is said: as before.
|
||||
{"no build source said: a README moves all three, as before", "mesh-controller",
|
||||
[]string{"README.md"}, unsaid, "mesh-controller,route-proxy | build-agent", ""},
|
||||
@@ -345,7 +345,7 @@ func TestAModuleAPlanOvertookIsReadWhole(t *testing.T) {
|
||||
built := time.Date(2026, 10, 10, 1, 0, 0, 0, time.UTC)
|
||||
read := map[string][]inventory.ReadRepository{
|
||||
"route-proxy": {
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"modules/route-proxy/cmd/route-proxy/", "go.mod"}, Built: built},
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"examples/route-proxy/", "go.mod"}, Built: built},
|
||||
{Own: true, Paths: []string{"modules/route-proxy/module.json"}, Built: built},
|
||||
},
|
||||
"mesh-controller": {{Own: true, Paths: []string{"module.json", "cmd/mesh-controller/"}, Built: built}},
|
||||
@@ -396,11 +396,11 @@ func TestAMissedMergeMovingOnlyAPackagingModuleIsActedOnOnce(t *testing.T) {
|
||||
}
|
||||
read := map[string][]inventory.ReadRepository{
|
||||
"mesh-controller": {{Own: true, Paths: []string{"module.json", "cmd/mesh-controller/"}, Built: merged.Add(-time.Hour)}},
|
||||
"route-proxy": {{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"modules/route-proxy/cmd/route-proxy/"},
|
||||
"route-proxy": {{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"examples/route-proxy/"},
|
||||
Built: merged.Add(-time.Hour), Looked: merged.Add(-time.Hour)}},
|
||||
}
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main", Commit: "c1",
|
||||
MergedAt: merged.Format(time.RFC3339), Paths: []string{"modules/route-proxy/cmd/route-proxy/main.go"}}
|
||||
MergedAt: merged.Format(time.RFC3339), Paths: []string{"examples/route-proxy/main.go"}}
|
||||
if got := wouldMove(m, entries, planningView(read, nil)); len(got) != 1 || got[0].Manifest.Module != "route-proxy" {
|
||||
t.Fatalf("a missed merge of the proxy's program would move %v", got)
|
||||
}
|
||||
@@ -599,14 +599,14 @@ func TestTheGatePlansFromTheBuildSourcesTheSnapshotCarries(t *testing.T) {
|
||||
Sources: []snapshot.BuildSource{{Own: true, Paths: []string{"module.json", "cmd/mesh-controller/", "internal/broker/"}}}},
|
||||
{Name: "route-proxy", Repository: "novox/mesh-catalog", Path: "modules/route-proxy", Manifest: manifest("route-proxy"),
|
||||
Reads: []string{"novox/mesh-controller"},
|
||||
Sources: []snapshot.BuildSource{{Repository: "novox/mesh-controller", Paths: []string{"modules/route-proxy/cmd/route-proxy/", "internal/broker/"}},
|
||||
Sources: []snapshot.BuildSource{{Repository: "novox/mesh-controller", Paths: []string{"examples/route-proxy/", "internal/broker/"}},
|
||||
{Own: true, Paths: []string{"modules/route-proxy/module.json"}}}},
|
||||
}}
|
||||
for paths, want := range map[string]string{
|
||||
"cmd/mesh-controller/main.go": "mesh-controller",
|
||||
"modules/route-proxy/cmd/route-proxy/main.go": "route-proxy",
|
||||
"internal/broker/broker.go": "mesh-controller,route-proxy",
|
||||
"README.md": "",
|
||||
"cmd/mesh-controller/main.go": "mesh-controller",
|
||||
"examples/route-proxy/main.go": "route-proxy",
|
||||
"internal/broker/broker.go": "mesh-controller,route-proxy",
|
||||
"README.md": "",
|
||||
} {
|
||||
r, err := reachOfChange(facts, "novox/mesh-controller", []string{paths}, "")
|
||||
if err != nil {
|
||||
|
||||
@@ -1,238 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The plans building (novox/hq ADR 0483 clause 2, issue 1687).
|
||||
//
|
||||
// **A key per delivery plan in its build phase, so that the build seat takes no new check while a delivery builds.** The phase
|
||||
// runs from the ask of the plan's first build until every build of its last tier is built, or the plan fails, stops or waits
|
||||
// on the operator. It is derived from the plan alone (buildPhaseOf), so a controller that restarts finds the same answer. The
|
||||
// key is put when the first build is asked (sayBuilding), kept current at every save of the plan (keepBuildingKey) and by a
|
||||
// look every link.PlansBuildingRefresh while this controller holds the lease (keepingPlansBuilding), and deleted at the phase's
|
||||
// end; the bucket's age removes the key of a controller that stopped, so checks are held at most that long.
|
||||
//
|
||||
// The layout is documented in internal/link/holds.go.
|
||||
|
||||
// buildPhaseOf says whether a plan is in its build phase: open, not waiting for the operator's word, with a build asked and a
|
||||
// build of its tiers still to be built. A tier built and sent leaves the plan rolling while later tiers are still to be asked;
|
||||
// it is in the phase still, which is what keeps a check from taking a place between two tiers.
|
||||
func buildPhaseOf(p inventory.Plan) bool {
|
||||
if !p.Open() || p.Waiting() || p.Release != nil {
|
||||
return false
|
||||
}
|
||||
asked, unbuilt := false, false
|
||||
for _, tier := range p.Tiers {
|
||||
for _, m := range tier {
|
||||
st := p.Modules[m]
|
||||
if st != nil && st.State != "" {
|
||||
asked = true
|
||||
}
|
||||
if st == nil || (st.State != "built" && st.State != planDeleted) {
|
||||
unbuilt = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return asked && unbuilt
|
||||
}
|
||||
|
||||
// buildingSince is when the plan's first build was asked, or fallback when no module says.
|
||||
func buildingSince(p inventory.Plan, fallback time.Time) time.Time {
|
||||
since := time.Time{}
|
||||
for _, st := range p.Modules {
|
||||
if st != nil && st.AskedAt != nil && (since.IsZero() || st.AskedAt.Before(since)) {
|
||||
since = *st.AskedAt
|
||||
}
|
||||
}
|
||||
if since.IsZero() {
|
||||
return fallback
|
||||
}
|
||||
return since
|
||||
}
|
||||
|
||||
// buildingStore is the bucket of the plans building.
|
||||
type buildingStore interface {
|
||||
Put(ctx context.Context, plan string, value []byte) error
|
||||
Delete(ctx context.Context, plan string) error
|
||||
Keys(ctx context.Context) ([]string, error)
|
||||
}
|
||||
|
||||
// syncBuilding makes the bucket say what the plans say: a key for each plan in its build phase, put again, and none for any
|
||||
// other plan, deleted. It returns the faults, each once.
|
||||
func syncBuilding(ctx context.Context, store buildingStore, plans []inventory.Plan, now time.Time) []error {
|
||||
var faults []error
|
||||
live := map[string]bool{}
|
||||
for _, p := range plans {
|
||||
if !buildPhaseOf(p) {
|
||||
continue
|
||||
}
|
||||
live[p.ID] = true
|
||||
if err := putBuilding(ctx, store, p.ID, buildingSince(p, now), now); err != nil {
|
||||
faults = append(faults, err)
|
||||
}
|
||||
}
|
||||
keys, err := store.Keys(ctx)
|
||||
if err != nil {
|
||||
return append(faults, err)
|
||||
}
|
||||
for _, key := range keys {
|
||||
if live[key] {
|
||||
continue
|
||||
}
|
||||
if err := store.Delete(ctx, key); err != nil {
|
||||
faults = append(faults, fmt.Errorf("the key of %s could not be taken away: %w", key, err))
|
||||
}
|
||||
}
|
||||
return faults
|
||||
}
|
||||
|
||||
func putBuilding(ctx context.Context, store buildingStore, plan string, since, now time.Time) error {
|
||||
body, _ := json.Marshal(link.PlanBuilding{Plan: plan, Since: since.UTC(), Seen: now.UTC()})
|
||||
if err := store.Put(ctx, plan, body); err != nil {
|
||||
return fmt.Errorf("the key of %s could not be put: %w", plan, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// busBuilding is the store on the controller's bucket.
|
||||
type busBuilding struct {
|
||||
conn *nats.Conn
|
||||
mu sync.Mutex
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
|
||||
func (b *busBuilding) bucket(ctx context.Context) (jetstream.KeyValue, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.kv != nil {
|
||||
return b.kv, nil
|
||||
}
|
||||
api, err := jetstream.New(b.conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, broker.PlansBuildingBucket)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the plans building: %w", err)
|
||||
}
|
||||
b.kv = kv
|
||||
return kv, nil
|
||||
}
|
||||
|
||||
func (b *busBuilding) Put(ctx context.Context, plan string, value []byte) error {
|
||||
kv, err := b.bucket(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = kv.Put(ctx, plan, value)
|
||||
return err
|
||||
}
|
||||
|
||||
func (b *busBuilding) Delete(ctx context.Context, plan string) error {
|
||||
kv, err := b.bucket(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := kv.Delete(ctx, plan); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *busBuilding) Keys(ctx context.Context) ([]string, error) {
|
||||
kv, err := b.bucket(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keys, err := kv.Keys(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return keys, err
|
||||
}
|
||||
|
||||
// buildingKeys is the store of the serving controller; nil in a process that is not it, which says nothing.
|
||||
var buildingKeys atomic.Pointer[buildingStore]
|
||||
|
||||
func setBuildingStore(s buildingStore) {
|
||||
if s == nil {
|
||||
buildingKeys.Store(nil)
|
||||
return
|
||||
}
|
||||
buildingKeys.Store(&s)
|
||||
}
|
||||
|
||||
// sayBuilding puts a plan's key: its first build is being asked (or another of its builds). A key that could not be put is
|
||||
// said; the look that follows puts it again.
|
||||
func sayBuilding(ctx context.Context, plan string, since time.Time) {
|
||||
s := buildingKeys.Load()
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
if err := putBuilding(ctx, *s, plan, since, time.Now()); err != nil {
|
||||
fmt.Printf("plans: %v; checks may take a place of %s until the next look\n", err, plan)
|
||||
}
|
||||
}
|
||||
|
||||
// keepBuildingKey is told each plan this controller keeps: its key put again while it builds and taken away once it does not.
|
||||
func keepBuildingKey(ctx context.Context, p inventory.Plan) {
|
||||
s := buildingKeys.Load()
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
if buildPhaseOf(p) {
|
||||
sayBuilding(ctx, p.ID, buildingSince(p, time.Now()))
|
||||
return
|
||||
}
|
||||
if err := (*s).Delete(ctx, p.ID); err != nil {
|
||||
fmt.Printf("plans: the key of %s could not be taken away; it ages out: %v\n", p.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// keepingPlansBuilding looks at the plans every link.PlansBuildingRefresh for as long as this controller serves and holds the
|
||||
// lease, so a key is put again before its age and the key of a plan that left the phase unseen is taken away.
|
||||
func keepingPlansBuilding(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn) {
|
||||
store := &busBuilding{conn: conn}
|
||||
setBuildingStore(store)
|
||||
defer setBuildingStore(nil)
|
||||
tick := time.NewTicker(broker.PlansBuildingRefresh)
|
||||
defer tick.Stop()
|
||||
failing := ""
|
||||
for {
|
||||
if epoch, err := theLease.epoch(ctx); err == nil && epoch != 0 {
|
||||
why := ""
|
||||
if plans, err := inv.OpenPlans(ctx); err != nil {
|
||||
why = fmt.Sprintf("the plans cannot be read: %v", err)
|
||||
} else if faults := syncBuilding(ctx, store, plans, time.Now()); len(faults) > 0 {
|
||||
why = fmt.Sprint(faults)
|
||||
}
|
||||
if why != failing {
|
||||
if why != "" {
|
||||
fmt.Printf("plans: the keys of the plans building cannot be kept: %s\n", why)
|
||||
} else {
|
||||
fmt.Printf("plans: the keys of the plans building are kept again\n")
|
||||
}
|
||||
failing = why
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,175 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// Issue 1687 (ADR 0483 clause 2): a delivery plan's key lives from the ask of its first build until every build of its last
|
||||
// tier is built, or the plan fails, stops or waits on the operator.
|
||||
|
||||
type fakeBuildingStore struct {
|
||||
mu sync.Mutex
|
||||
keys map[string]link.PlanBuilding
|
||||
puts map[string]int
|
||||
}
|
||||
|
||||
func newFakeBuildingStore() *fakeBuildingStore {
|
||||
return &fakeBuildingStore{keys: map[string]link.PlanBuilding{}, puts: map[string]int{}}
|
||||
}
|
||||
|
||||
func (f *fakeBuildingStore) Put(_ context.Context, plan string, value []byte) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var v link.PlanBuilding
|
||||
if err := json.Unmarshal(value, &v); err != nil {
|
||||
return err
|
||||
}
|
||||
f.keys[plan] = v
|
||||
f.puts[plan]++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeBuildingStore) Delete(_ context.Context, plan string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
delete(f.keys, plan)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeBuildingStore) Keys(context.Context) ([]string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var out []string
|
||||
for k := range f.keys {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func aPlan(id, state string, tiers [][]string, states map[string]string) inventory.Plan {
|
||||
p := inventory.Plan{ID: id, State: state, Tiers: tiers, Modules: map[string]*inventory.PlanModule{}}
|
||||
now := time.Now().Add(-time.Minute)
|
||||
for m, st := range states {
|
||||
pm := &inventory.PlanModule{State: st}
|
||||
if st != "" {
|
||||
pm.AskedAt = &now
|
||||
}
|
||||
p.Modules[m] = pm
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func TestAPlanIsInItsBuildPhaseFromTheFirstAskToTheLastBuild(t *testing.T) {
|
||||
tiers := [][]string{{"a", "b"}, {"c"}}
|
||||
cases := []struct {
|
||||
name string
|
||||
plan inventory.Plan
|
||||
want bool
|
||||
}{
|
||||
{"no build asked yet", aPlan("p", inventory.PlanBuilding, tiers, nil), false},
|
||||
{"the first tier asked", aPlan("p", inventory.PlanBuilding, tiers, map[string]string{"a": "asked", "b": "asked"}), true},
|
||||
// Between two tiers: the first built and sent (rolling), the second not asked yet.
|
||||
{"between two tiers", aPlan("p", inventory.PlanRolling, tiers, map[string]string{"a": "built", "b": "built"}), true},
|
||||
{"the last tier asked", aPlan("p", inventory.PlanBuilding, tiers, map[string]string{"a": "built", "b": "built", "c": "asked"}), true},
|
||||
{"every build built", aPlan("p", inventory.PlanRolling, tiers, map[string]string{"a": "built", "b": "built", "c": "built"}), false},
|
||||
// A module the plan deletes is done as far as its tiers go (release_plan.go), so it holds no key through the rollout.
|
||||
{"a deleted module is done", aPlan("p", inventory.PlanRolling, tiers, map[string]string{"a": "built", "b": "built", "c": planDeleted}), false},
|
||||
{"failed", aPlan("p", inventory.PlanFailed, tiers, map[string]string{"a": "failed", "b": "asked"}), false},
|
||||
{"superseded", aPlan("p", inventory.PlanSuperseded, tiers, map[string]string{"a": "asked"}), false},
|
||||
{"a batch whose window is open", aPlan("p", inventory.PlanAssembling, tiers, map[string]string{"a": "asked"}), false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := buildPhaseOf(c.plan); got != c.want {
|
||||
t.Errorf("%s: in the build phase = %v, want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
waiting := aPlan("p", inventory.PlanBuilding, tiers, map[string]string{"a": "asked"})
|
||||
waiting.Delivery = &inventory.PlanDelivery{Awaits: "the operator's word"}
|
||||
if buildPhaseOf(waiting) {
|
||||
t.Error("a plan that waits on the operator is in its build phase")
|
||||
}
|
||||
}
|
||||
|
||||
// The key's life: put, refreshed, deleted at the phase's end; a plan that left the phase unseen loses its key at the next look.
|
||||
func TestAPlanKeyIsPutRefreshedAndDeletedAtThePhasesEnd(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := newFakeBuildingStore()
|
||||
tiers := [][]string{{"a"}, {"b"}}
|
||||
building := aPlan("plan-1", inventory.PlanBuilding, tiers, map[string]string{"a": "asked"})
|
||||
|
||||
if faults := syncBuilding(ctx, store, []inventory.Plan{building}, time.Now()); len(faults) > 0 {
|
||||
t.Fatal(faults)
|
||||
}
|
||||
if _, ok := store.keys["plan-1"]; !ok {
|
||||
t.Fatal("no key while the first build is asked")
|
||||
}
|
||||
first := store.keys["plan-1"]
|
||||
// Refreshed: put again, the phase's start kept.
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
syncBuilding(ctx, store, []inventory.Plan{building}, time.Now())
|
||||
if store.puts["plan-1"] != 2 || !store.keys["plan-1"].Seen.After(first.Seen) || !store.keys["plan-1"].Since.Equal(first.Since) {
|
||||
t.Errorf("the key was not refreshed with its start kept: %d puts, %+v then %+v", store.puts["plan-1"], first, store.keys["plan-1"])
|
||||
}
|
||||
// Between the tiers: still there.
|
||||
between := aPlan("plan-1", inventory.PlanRolling, tiers, map[string]string{"a": "built"})
|
||||
syncBuilding(ctx, store, []inventory.Plan{between}, time.Now())
|
||||
if _, ok := store.keys["plan-1"]; !ok {
|
||||
t.Error("the key was gone between two tiers")
|
||||
}
|
||||
// A key of another plan that is no longer in the phase.
|
||||
store.keys["plan-0"] = link.PlanBuilding{Plan: "plan-0"}
|
||||
done := aPlan("plan-1", inventory.PlanRolling, tiers, map[string]string{"a": "built", "b": "built"})
|
||||
syncBuilding(ctx, store, []inventory.Plan{done}, time.Now())
|
||||
if len(store.keys) != 0 {
|
||||
t.Errorf("keys remain after the last build was built: %v", store.keys)
|
||||
}
|
||||
}
|
||||
|
||||
// keepBuildingKey, told of each save, puts the key while the plan builds and deletes it when the plan fails.
|
||||
func TestASavedPlanPutsOrDeletesItsKey(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := newFakeBuildingStore()
|
||||
var s buildingStore = store
|
||||
buildingKeys.Store(&s)
|
||||
defer buildingKeys.Store(nil)
|
||||
|
||||
p := aPlan("plan-7", inventory.PlanBuilding, [][]string{{"a"}}, map[string]string{"a": "asked"})
|
||||
keepBuildingKey(ctx, p)
|
||||
if _, ok := store.keys["plan-7"]; !ok {
|
||||
t.Fatal("the key was not put when the plan was saved building")
|
||||
}
|
||||
p.State = inventory.PlanFailed
|
||||
keepBuildingKey(ctx, p)
|
||||
if _, ok := store.keys["plan-7"]; ok {
|
||||
t.Fatal("the key stayed after the plan failed")
|
||||
}
|
||||
}
|
||||
|
||||
// The plan's key is there before its first build is asked, so a check cannot take a place between the ask and the plan's save.
|
||||
func TestThePlansKeyIsPutBeforeItsFirstBuildIsAsked(t *testing.T) {
|
||||
store := newFakeBuildingStore()
|
||||
var s buildingStore = store
|
||||
buildingKeys.Store(&s)
|
||||
defer buildingKeys.Store(nil)
|
||||
p, byName := aTierOfThree()
|
||||
keyThere := false
|
||||
was := askABuild
|
||||
askABuild = func(context.Context, buildSource, string, string) (string, error) {
|
||||
_, keyThere = store.keys["plan-1"]
|
||||
return "build-x", nil
|
||||
}
|
||||
t.Cleanup(func() { askABuild = was })
|
||||
askModule(t.Context(), p, "a", byName)
|
||||
if !keyThere {
|
||||
t.Fatal("the plan's key was not there when its first build was asked")
|
||||
}
|
||||
}
|
||||
@@ -153,7 +153,7 @@ func planWaitCommand(ctx context.Context, inv *inventory.Inventory, id, untilAsk
|
||||
line := planLineSaid(p, now, pause, bounds.of(p.Repository), func(module string) []string {
|
||||
on, _ := inv.Running(ctx, module)
|
||||
return on
|
||||
}, waitsOf(ctx, inv, one)) + heldSaid(p, heldNow(ctx, inv))
|
||||
}, waitsOf(ctx, inv, one))
|
||||
doc := planWaitedDoc(p, planStatuses(one, now, pause, bounds)[0], until, came, now.Sub(began), line)
|
||||
return json.NewEncoder(os.Stdout).Encode(doc)
|
||||
}
|
||||
|
||||
@@ -1,130 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// processUser is the bus user of one process of a module on a node (novox/hq ADR 0436 §1).
|
||||
func processUser(node, module, process string) string {
|
||||
return broker.Principal{Kind: broker.KindProcess, Node: node, Module: module, Process: process}.Username()
|
||||
}
|
||||
|
||||
// processAccountWarning is the warning line for the own secret broker-<process id> of a process that declares bus
|
||||
// when nobody issued its account (novox/hq ADR 0436 §2, issue 1226): "" for any other secret and for an issued
|
||||
// account. The send goes on with the stand-in until the composition leaves the module off (issue 1753), so the
|
||||
// line is the only loud thing about it.
|
||||
func processAccountWarning(m catalogue.Manifest, node, secret string, minted []string) string {
|
||||
id, isProcess := strings.CutPrefix(secret, "broker-")
|
||||
if !isProcess {
|
||||
return ""
|
||||
}
|
||||
if _, declared := m.ProcessBuses()[id]; !declared {
|
||||
return ""
|
||||
}
|
||||
user := processUser(node, m.Module, id)
|
||||
for _, u := range minted {
|
||||
if u == user {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("%s/%s process %s: its bus account is not issued — mesh-controller.issue %s %s",
|
||||
node, m.Module, id, node, m.Module)
|
||||
}
|
||||
|
||||
// sayWarning writes a line where the compose's other warnings go: the act's answer when it carries a writer,
|
||||
// else the controller's journal.
|
||||
func sayWarning(ctx context.Context, line string) {
|
||||
var w io.Writer = os.Stderr
|
||||
if lw, ok := ctx.Value(leftOutKey{}).(io.Writer); ok {
|
||||
w = lw
|
||||
}
|
||||
fmt.Fprintln(w, line)
|
||||
}
|
||||
|
||||
// processAccountsMissing is each process of m that declares bus and whose account is not minted yet on node.
|
||||
func processAccountsMissing(m catalogue.Manifest, node string, minted []string) []broker.ProcessPrincipal {
|
||||
have := map[string]bool{}
|
||||
for _, u := range minted {
|
||||
have[u] = true
|
||||
}
|
||||
var out []broker.ProcessPrincipal
|
||||
for _, pr := range inventory.ProcessesOf(m) {
|
||||
if !have[processUser(node, m.Module, pr.Process)] {
|
||||
out = append(out, pr)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// issueMissingProcessAccounts issues the accounts of m's bus processes on node that are not minted, and no
|
||||
// other: an account already issued is rotated on purpose, never by a later delivery or assignment (issue 1226).
|
||||
// It says what it issued, one line, and nothing when nothing was missing.
|
||||
func issueMissingProcessAccounts(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node string) (string, error) {
|
||||
if len(m.ProcessBuses()) == 0 || mayIssue(m) != nil {
|
||||
return "", nil
|
||||
}
|
||||
minted, err := inv.BusMinted(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
missing := processAccountsMissing(m, node, minted)
|
||||
if len(missing) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
known, err := broker.FromEnvironment()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("cannot deliver a credential without knowing where the bus is: %w", err)
|
||||
}
|
||||
reachable, err := brokerReachableAt(ctx, inv, known, node)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
busAddress, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
busAddress = ""
|
||||
}
|
||||
var ids []string
|
||||
for _, pr := range missing {
|
||||
if err := issueProcess(ctx, inv, m, pr, node, busAddress, known, reachable); err != nil {
|
||||
return "", err
|
||||
}
|
||||
ids = append(ids, pr.Process)
|
||||
}
|
||||
return fmt.Sprintf("its process account(s) %s issued and sealed to %s, and arrive with the push",
|
||||
strings.Join(ids, ", "), node), nil
|
||||
}
|
||||
|
||||
// issueGainedProcessAccounts runs after a module's new definition is registered: on each machine assigned it, the
|
||||
// accounts of its bus processes not yet minted are issued, so a process a delivery adds is not pushed with a
|
||||
// random stand-in (issue 1226). Never fatal: the build worked and the module is registered, so a failure is said,
|
||||
// and the push refuses the module until `mesh-controller.issue` is run.
|
||||
func issueGainedProcessAccounts(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest) {
|
||||
if len(m.ProcessBuses()) == 0 {
|
||||
return
|
||||
}
|
||||
nodes, err := inv.Running(ctx, m.Module)
|
||||
if err != nil {
|
||||
fmt.Printf("%s: its machines are not read to issue its process accounts: %v\n", m.Module, err)
|
||||
return
|
||||
}
|
||||
for _, node := range nodes {
|
||||
line, err := issueMissingProcessAccounts(ctx, inv, m, node)
|
||||
if err != nil {
|
||||
fmt.Printf("%s on %s: its process accounts are not issued (%v): `mesh-controller.issue %s %s` first\n",
|
||||
m.Module, node, err, node, m.Module)
|
||||
continue
|
||||
}
|
||||
if line != "" {
|
||||
fmt.Printf("%s on %s: %s\n", m.Module, node, line)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,135 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
func gainedAndPlain(t *testing.T) (plain, gained catalogue.Manifest) {
|
||||
t.Helper()
|
||||
plain, err := catalogue.ParseManifest([]byte(`{"module":"mesh-mail","version":"1",
|
||||
"own-secrets":{"broker-send":{"path":"/var/lib/mesh-mail/broker-send","owner":"mesh-mail-send"}},
|
||||
"resources":[{"id":"account","type":"user","name":"mesh-mail-send"}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return plain, mailSenderWithProcess()
|
||||
}
|
||||
|
||||
// 1226/AC2: given an assigned module whose new definition adds a bus process, when the delivery records it
|
||||
// (takeIn, the function that records a build result), the process's account is minted; and 1226/AC3: the same
|
||||
// build heard again replaces nothing.
|
||||
func TestAGainedProcessIsIssuedWhenTheDeliveryRecordsIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
|
||||
dir := t.TempDir()
|
||||
if err := busCertificate([]string{"--into", dir}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv(broker.AddressVar, strings.TrimPrefix(testbus.URL(t), "nats://"))
|
||||
t.Setenv(broker.CertificateVar, filepath.Join(dir, "tls.crt"))
|
||||
|
||||
plain, gained := gainedAndPlain(t)
|
||||
register(t, open, plain)
|
||||
if _, err := assign(ctx, open, "laptop", "mesh-mail"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, minted, _ := inv.BusUserHash(ctx, "laptop.mesh-mail.send"); minted {
|
||||
t.Fatal("the account was minted before the module gained the process")
|
||||
}
|
||||
|
||||
manifest, _ := json.Marshal(gained)
|
||||
build := func(id, commit string) {
|
||||
t.Helper()
|
||||
if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, link.BuildResult{
|
||||
ID: id, Repository: "http://forge.internal:20000/novox/mesh-mail.git", Path: "modules/mesh-mail",
|
||||
Ref: "main", On: "anchor", Commit: commit, Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-mail"},
|
||||
})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
build("b-1", "aaaaaaa1111")
|
||||
hash, held, err := inv.BusUserHash(ctx, "laptop.mesh-mail.send")
|
||||
if err != nil || !held {
|
||||
t.Fatalf("1226/AC2: recording the gained definition did not mint the process's account: held %v, %v", held, err)
|
||||
}
|
||||
// And the next compose carries the account, with no warning (1226/AC2).
|
||||
var said bytes.Buffer
|
||||
if _, err := prepareSend(leftOutSaidBy(ctx, &said), open, "laptop"); err != nil {
|
||||
t.Fatalf("1226/AC2: the next compose failed: %v", err)
|
||||
}
|
||||
if strings.Contains(said.String(), "bus account is not issued") {
|
||||
t.Fatalf("1226/AC2: the next compose warns of an account just minted: %q", said.String())
|
||||
}
|
||||
// The row the compose reads is the account issueProcess accepted (origin accepted), never a value the mesh
|
||||
// made for it (a stand-in).
|
||||
in, err := inputFor(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var row *compose.OwnSecretRow
|
||||
for i, r := range in.OwnSecrets {
|
||||
if r.Node == "laptop" && r.Module == "mesh-mail" && r.Name == "broker-send" {
|
||||
row = &in.OwnSecrets[i]
|
||||
}
|
||||
}
|
||||
if row == nil {
|
||||
t.Fatal("1226/AC2: the next compose carries no row for the process's account")
|
||||
}
|
||||
if row.Origin != inventory.OriginAccepted {
|
||||
t.Fatalf("1226/AC2: the process's account is a stand-in the mesh made (origin %q), not the issued account", row.Origin)
|
||||
}
|
||||
build("b-2", "bbbbbbb2222")
|
||||
again, _, err := inv.BusUserHash(ctx, "laptop.mesh-mail.send")
|
||||
if err != nil || again != hash {
|
||||
t.Fatalf("1226/AC3: recording again replaced the issued account (%v)", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1226/AC1 and AC4 against the store: given two modules on a machine, one with an unissued process account, when
|
||||
// the machine is composed, the rest of it is composed and sent, and only the warning in the push's answer names
|
||||
// the first, with `mesh-controller.issue <node> <module>`.
|
||||
func TestAnUnissuedProcessAccountIsWarnedOfAndTheRestIsComposed(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
_, gained := gainedAndPlain(t)
|
||||
register(t, open, gained)
|
||||
other := catalogue.Manifest{Module: "other", Version: "1",
|
||||
OwnSecrets: catalogue.OwnSecrets{"token": {Path: "/var/lib/other/token"}}}
|
||||
register(t, open, other)
|
||||
for _, m := range []string{"mesh-mail", "other"} {
|
||||
if _, err := assign(ctx, open, "laptop", m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var said bytes.Buffer
|
||||
if _, err := prepareSend(leftOutSaidBy(ctx, &said), open, "laptop"); err != nil {
|
||||
t.Fatalf("1226/AC4: an unissued process account held the machine: %v", err)
|
||||
}
|
||||
want := "laptop/mesh-mail process send: its bus account is not issued — mesh-controller.issue laptop mesh-mail"
|
||||
if !strings.Contains(said.String(), want) {
|
||||
t.Fatalf("1226/AC1: the push says %q, want %q", said.String(), want)
|
||||
}
|
||||
if strings.Contains(said.String(), "other") {
|
||||
t.Fatalf("1226/AC4: the other module is warned of: %q", said.String())
|
||||
}
|
||||
if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "other", "token"); err != nil || !held {
|
||||
t.Fatalf("1226/AC4: the other module's secret was not composed: held %v, %v", held, err)
|
||||
}
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
// 1226/AC1: given a module whose process declares bus and has no issued account, when a machine is composed, the
|
||||
// push's warning names issue, and the machine's other modules are composed (AC4: nothing is refused).
|
||||
func TestAProcessAccountNobodyIssuedIsWarnedOfNamingIssue(t *testing.T) {
|
||||
m := mailSenderWithProcess()
|
||||
got := processAccountWarning(m, "laptop", "broker-send", nil)
|
||||
want := "laptop/mesh-mail process send: its bus account is not issued — mesh-controller.issue laptop mesh-mail"
|
||||
if got != want {
|
||||
t.Fatalf("1226/AC1: the warning is %q, want %q", got, want)
|
||||
}
|
||||
// Other names are not this rule's.
|
||||
if processAccountWarning(m, "laptop", "broker", nil) != "" || processAccountWarning(m, "laptop", "broker-other", nil) != "" ||
|
||||
processAccountWarning(m, "laptop", "token", nil) != "" {
|
||||
t.Fatal("1226/AC1: a name that is no declared process's account was warned of")
|
||||
}
|
||||
if got := processAccountWarning(m, "laptop", "broker-send", []string{"laptop.mesh-mail.send"}); got != "" {
|
||||
t.Fatalf("1226/AC3: an issued account was warned of: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1226/AC3: an account already issued is not issued again by a later delivery or assignment.
|
||||
func TestAnIssuedProcessAccountIsNotIssuedAgain(t *testing.T) {
|
||||
m := mailSenderWithProcess()
|
||||
if got := processAccountsMissing(m, "laptop", nil); len(got) != 1 || got[0].Process != "send" {
|
||||
t.Fatalf("1226/AC3: an account not issued is not missing: %v", got)
|
||||
}
|
||||
if got := processAccountsMissing(m, "laptop", []string{"laptop.mesh-mail.send"}); len(got) != 0 {
|
||||
t.Fatalf("1226/AC3: an issued account would be issued again: %v", got)
|
||||
}
|
||||
if got := processAccountsMissing(m, "ace", []string{"laptop.mesh-mail.send"}); len(got) != 1 {
|
||||
t.Fatalf("1226/AC3: another machine's account stands in for this one's: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -1,211 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"reflect"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// What the controller tells mesh-config of an open pull request, and asks it back (novox/hq ADR 0502 clauses 2 and 4,
|
||||
// issue 1826): a config key a pull request introduces is not declared on the trunk, so a value for it can be judged
|
||||
// only against the pull request's own declaration, which the core reads from the forge and mesh-config never does
|
||||
// (ADR 0495 §1). The call is mesh-config's `config` verb in the controller's own name, actions `pull-declares` and
|
||||
// `held`; the shapes are those of the interface the two sides agreed.
|
||||
|
||||
// meshConfigWait is how long the controller waits for mesh-config before it goes on without: a check is never held up
|
||||
// by a module that does not answer.
|
||||
const meshConfigWait = 10 * time.Second
|
||||
|
||||
// meshConfigAsk is one call of mesh-config's `config` verb in the controller's own name, answering its result. A
|
||||
// variable so a test needs no bus.
|
||||
var meshConfigAsk = func(ctx context.Context, args map[string]any) ([]byte, error) {
|
||||
var result []byte
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
answer, err := link.AskMeshSeatToolAs(ctx, conn, broker.ControllerName, "mesh-config", "config", args, meshConfigWait)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return fmt.Errorf("%s", answer.Error)
|
||||
}
|
||||
result = answer.Result
|
||||
return nil
|
||||
})
|
||||
return result, err
|
||||
}
|
||||
|
||||
// pullAt is an open pull request at a head: the change to read, and its number.
|
||||
type pullAt struct {
|
||||
changeSource
|
||||
Number int
|
||||
}
|
||||
|
||||
// pullAtOf is a pull request announced, as the change it makes over what it merges into.
|
||||
func pullAtOf(p link.PullUpdated) pullAt {
|
||||
old := p.BaseSHA
|
||||
if old == "" {
|
||||
old = p.Base
|
||||
}
|
||||
return pullAt{Number: p.Number, changeSource: changeSource{Owner: p.Owner, Repo: p.Repo, New: p.Commit, Old: old,
|
||||
Paths: p.Paths, Truncated: p.PathsTruncated, Removed: p.Removed}}
|
||||
}
|
||||
|
||||
func pullKey(owner, repo string, number int) string {
|
||||
return fmt.Sprintf("%s/%s#%d", owner, repo, number)
|
||||
}
|
||||
|
||||
func (p pullAt) key() string { return pullKey(p.Owner, p.Repo, p.Number) }
|
||||
|
||||
// pullDeclarations are the config declarations a pull request's head adds or changes over what it merges into, by
|
||||
// module, in the shape of the declared-config mirror. The bool says the head could be read: false for a pull request whose
|
||||
// paths are cut (the caller cuts them to the manifests of the modules it moves) or whose base is unknown, which is then
|
||||
// told nothing, never an empty set.
|
||||
func pullDeclarations(ctx context.Context, p pullAt) ([]inventory.DeclaredConfigMirrored, bool, error) {
|
||||
var out []inventory.DeclaredConfigMirrored
|
||||
if p.Truncated || p.Old == "" {
|
||||
return nil, false, nil
|
||||
}
|
||||
for _, file := range p.Paths {
|
||||
if path.Base(file) != moduleManifestFile || slices.Contains(p.Removed, file) {
|
||||
continue
|
||||
}
|
||||
now, exists, err := manifestCached(ctx, p.Owner, p.Repo, file, p.New)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if !exists || now.Module == "" {
|
||||
continue
|
||||
}
|
||||
was, _, err := manifestCached(ctx, p.Owner, p.Repo, file, p.Old)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
have, err := inventory.DeclaredOf(now.Settings)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("%s at %.8s: %w", file, p.New, err)
|
||||
}
|
||||
had, err := inventory.DeclaredOf(was.Settings)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("%s at %.8s: %w", file, p.Old, err)
|
||||
}
|
||||
changed := map[string]inventory.DeclaredMirrored{}
|
||||
for key, d := range have {
|
||||
if before, there := had[key]; !there || !reflect.DeepEqual(before, d) {
|
||||
changed[key] = d
|
||||
}
|
||||
}
|
||||
if len(changed) > 0 {
|
||||
out = append(out, inventory.DeclaredConfigMirrored{Module: now.Module, Settings: changed})
|
||||
}
|
||||
}
|
||||
return out, true, nil
|
||||
}
|
||||
|
||||
// tellPullDeclares tells mesh-config, for each pull request at every announced head, what the head declares, an empty set
|
||||
// when it declares nothing new: mesh-config re-judges the values held at every head, so a head that reverts the change
|
||||
// drops them (ADR 0502 clause 6). A
|
||||
// failure is said to w and the check goes on: the held values are then none (ADR 0502 clause 2).
|
||||
func tellPullDeclares(ctx context.Context, w io.Writer, pulls []pullAt) {
|
||||
for _, p := range pulls {
|
||||
mods, readable, err := pullDeclarations(ctx, p)
|
||||
if err != nil {
|
||||
fmt.Fprintf(w, "%s: what its head declares cannot be read, so mesh-config is told nothing: %v\n", p.key(), err)
|
||||
continue
|
||||
}
|
||||
if !readable {
|
||||
continue
|
||||
}
|
||||
declares := map[string]map[string]inventory.DeclaredMirrored{}
|
||||
for _, m := range mods {
|
||||
declares[m.Module] = m.Settings
|
||||
}
|
||||
if _, err := meshConfigAsk(ctx, map[string]any{"action": "pull-declares", "pull": p.key(), "head": p.New,
|
||||
"declares": declares}); err != nil {
|
||||
fmt.Fprintf(w, "%s: mesh-config cannot be told what its head declares (%v); the check goes on without held values\n",
|
||||
p.key(), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// heldFor asks mesh-config for the approved held values of these pull requests at their heads (ADR 0502 clause 4). Only
|
||||
// those of the pull requests asked about are carried. A failure is said to w and answers none.
|
||||
func heldFor(ctx context.Context, w io.Writer, pulls []pullAt) []link.HeldConfig {
|
||||
if len(pulls) == 0 {
|
||||
return nil
|
||||
}
|
||||
type pullHead struct {
|
||||
Pull string `json:"pull"`
|
||||
Head string `json:"head"`
|
||||
}
|
||||
asked := make([]pullHead, 0, len(pulls))
|
||||
for _, p := range pulls {
|
||||
asked = append(asked, pullHead{p.key(), p.New})
|
||||
}
|
||||
raw, err := meshConfigAsk(ctx, map[string]any{"action": "held", "pulls": asked})
|
||||
if err != nil {
|
||||
fmt.Fprintf(w, "mesh-config cannot say what it holds for these pull requests (%v); the check goes on without held values\n", err)
|
||||
return nil // empty-on-error: said above; a check without held values fails for want of the value and says so, it never passes on a guess
|
||||
}
|
||||
var answer struct {
|
||||
Held []link.HeldConfig `json:"held"`
|
||||
Skipped []struct {
|
||||
Pull string `json:"pull"`
|
||||
Why string `json:"why"`
|
||||
} `json:"skipped"`
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := json.Unmarshal(raw, &answer); err != nil {
|
||||
fmt.Fprintf(w, "mesh-config's held values cannot be read (%v); the check goes on without them\n", err)
|
||||
return nil // empty-on-error: said above; a check without held values fails for want of the value and says so, it never passes on a guess
|
||||
}
|
||||
for _, s := range answer.Skipped {
|
||||
fmt.Fprintf(w, "%s: mesh-config holds a value it does not carry: %s\n", s.Pull, s.Why)
|
||||
}
|
||||
var out []link.HeldConfig
|
||||
for _, h := range answer.Held {
|
||||
for _, p := range pulls {
|
||||
if h.Pull == p.key() {
|
||||
out = append(out, h)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// withHeldConfig tells mesh-config what these pull requests declare and puts the approved held values of them on the
|
||||
// check request (ADR 0502 clause 4). With carry false, a member of a group whose composed check composes, only the
|
||||
// declarations are told.
|
||||
func withHeldConfig(ctx context.Context, w io.Writer, request *link.BuildRequest, pulls []pullAt, carry bool) {
|
||||
tellPullDeclares(ctx, w, pulls)
|
||||
if carry {
|
||||
carryHeld(ctx, w, request, pulls)
|
||||
}
|
||||
}
|
||||
|
||||
// carryHeld puts the approved held values of these pull requests on the check request.
|
||||
func carryHeld(ctx context.Context, w io.Writer, request *link.BuildRequest, pulls []pullAt) {
|
||||
if request.Check != nil {
|
||||
request.Check.HeldConfig = heldFor(ctx, w, pulls)
|
||||
}
|
||||
}
|
||||
|
||||
// holdsModules is whether a module of the mesh is built from this repository.
|
||||
func holdsModules(entries []inventory.Entry, owner, repo string) bool {
|
||||
return slices.ContainsFunc(entries, func(e inventory.Entry) bool {
|
||||
return sameRepository(e.Source.Repository, link.SourceMoved{Owner: owner, Repo: repo})
|
||||
})
|
||||
}
|
||||
@@ -1,209 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
const (
|
||||
headSHA = "2222222222222222222222222222222222222222"
|
||||
baseSHA = "1111111111111111111111111111111111111111"
|
||||
)
|
||||
|
||||
// fakeManifests serves manifests by ref, and counts what the forge was asked. Issue 1826/AC1 (unit level).
|
||||
func fakeManifests(t *testing.T, at map[string]*catalogue.Manifest) {
|
||||
t.Helper()
|
||||
was := manifestAt
|
||||
t.Cleanup(func() { manifestAt = was })
|
||||
manifestsReadMu.Lock()
|
||||
clear(manifestsRead)
|
||||
manifestsReadMu.Unlock()
|
||||
manifestAt = func(_ context.Context, _, _, _, ref string) (catalogue.Manifest, bool, error) {
|
||||
if m := at[ref]; m != nil {
|
||||
return *m, true, nil
|
||||
}
|
||||
return catalogue.Manifest{}, false, nil
|
||||
}
|
||||
}
|
||||
|
||||
// fakeConfig records the calls to mesh-config and answers each action from answers.
|
||||
func fakeConfig(t *testing.T, answers map[string]string) *[]map[string]any {
|
||||
t.Helper()
|
||||
var calls []map[string]any
|
||||
was := meshConfigAsk
|
||||
t.Cleanup(func() { meshConfigAsk = was })
|
||||
meshConfigAsk = func(_ context.Context, args map[string]any) ([]byte, error) {
|
||||
calls = append(calls, args)
|
||||
return []byte(answers[args["action"].(string)]), nil
|
||||
}
|
||||
return &calls
|
||||
}
|
||||
|
||||
func manifest(module string, keys map[string]catalogue.SettingDeclaration) *catalogue.Manifest {
|
||||
return &catalogue.Manifest{Module: module, Settings: keys}
|
||||
}
|
||||
|
||||
func thePull() pullAt {
|
||||
return pullAt{Number: 434, changeSource: changeSource{Owner: "novox", Repo: "mesh-catalog", New: headSHA, Old: baseSHA,
|
||||
Paths: []string{"modules/n8n/module.json", "modules/n8n/README.md"}}}
|
||||
}
|
||||
|
||||
// 1826/AC1: a key the head adds is sent with its declaration, under the pull request and the head.
|
||||
func TestAKeyThePullRequestAddsIsToldToMeshConfig(t *testing.T) {
|
||||
fakeManifests(t, map[string]*catalogue.Manifest{
|
||||
baseSHA: manifest("n8n", map[string]catalogue.SettingDeclaration{"port": {Type: "number"}}),
|
||||
headSHA: manifest("n8n", map[string]catalogue.SettingDeclaration{"port": {Type: "number"},
|
||||
"media-gid": {Type: "number", Required: true}}),
|
||||
})
|
||||
calls := fakeConfig(t, map[string]string{"pull-declares": `{"kept":1}`})
|
||||
tellPullDeclares(context.Background(), io.Discard, []pullAt{thePull()})
|
||||
if len(*calls) != 1 {
|
||||
t.Fatalf("mesh-config was asked %d times: %v", len(*calls), *calls)
|
||||
}
|
||||
c := (*calls)[0]
|
||||
if c["action"] != "pull-declares" || c["pull"] != "novox/mesh-catalog#434" || c["head"] != headSHA {
|
||||
t.Fatalf("the call is %v", c)
|
||||
}
|
||||
body, _ := json.Marshal(c["declares"])
|
||||
var declares map[string]map[string]inventory.DeclaredMirrored
|
||||
if err := json.Unmarshal(body, &declares); err != nil || len(declares) != 1 || declares["n8n"] == nil {
|
||||
t.Fatalf("declares %s: %v", body, err)
|
||||
}
|
||||
mods := []inventory.DeclaredConfigMirrored{{Module: "n8n", Settings: declares["n8n"]}}
|
||||
if _, has := mods[0].Settings["port"]; has || len(mods[0].Settings) != 1 {
|
||||
t.Fatalf("only the added key is told, got %v", mods[0].Settings)
|
||||
}
|
||||
d := mods[0].Settings["media-gid"]
|
||||
if d.Type != "number" || !d.Required {
|
||||
t.Fatalf("the declaration is %+v", d)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC1: a key whose declaration the head changes is sent; one it leaves as it was is not.
|
||||
func TestAnUnchangedKeyIsNotToldAndAChangedOneIs(t *testing.T) {
|
||||
fakeManifests(t, map[string]*catalogue.Manifest{
|
||||
baseSHA: manifest("n8n", map[string]catalogue.SettingDeclaration{"port": {Type: "number"}, "x": {Type: "string"}}),
|
||||
headSHA: manifest("n8n", map[string]catalogue.SettingDeclaration{"port": {Type: "number"}, "x": {Type: "string", Pattern: "a+"}}),
|
||||
})
|
||||
mods, touched, err := pullDeclarations(context.Background(), thePull())
|
||||
if err != nil || !touched || len(mods) != 1 || len(mods[0].Settings) != 1 || mods[0].Settings["x"].Pattern != "a+" {
|
||||
t.Fatalf("%+v %v %v", mods, touched, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC1: a head that changes no module.json, or removes it, is still told, with declares {}: mesh-config re-judges at every
|
||||
// head, so a head that reverts the change drops the value held for it (review round 1, finding 1).
|
||||
func TestAHeadThatDeclaresNothingIsToldAsEmpty(t *testing.T) {
|
||||
fakeManifests(t, map[string]*catalogue.Manifest{baseSHA: manifest("n8n", nil), headSHA: manifest("n8n", nil)})
|
||||
calls := fakeConfig(t, nil)
|
||||
p := thePull()
|
||||
p.Paths = []string{"modules/n8n/README.md"}
|
||||
tellPullDeclares(context.Background(), io.Discard, []pullAt{p})
|
||||
p = thePull()
|
||||
p.Removed = p.Paths
|
||||
tellPullDeclares(context.Background(), io.Discard, []pullAt{p})
|
||||
if len(*calls) != 2 {
|
||||
t.Fatalf("mesh-config was told %v", *calls)
|
||||
}
|
||||
for _, c := range *calls {
|
||||
if d, _ := json.Marshal(c["declares"]); string(d) != "{}" || c["head"] != headSHA {
|
||||
t.Fatalf("told %v", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC1: a head whose paths are cut, or whose base is unknown, cannot be read: nothing is told, never an empty set.
|
||||
func TestAHeadThatCannotBeReadIsNotToldAsEmpty(t *testing.T) {
|
||||
calls := fakeConfig(t, nil)
|
||||
p := thePull()
|
||||
p.Truncated = true
|
||||
tellPullDeclares(context.Background(), io.Discard, []pullAt{p})
|
||||
if len(*calls) != 0 {
|
||||
t.Fatalf("told %v", *calls)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC1: a module.json changed without a declaration changing is told as an empty set, so mesh-config re-judges what it holds.
|
||||
func TestAChangedManifestWithNoNewDeclarationIsToldAsEmpty(t *testing.T) {
|
||||
m := manifest("n8n", map[string]catalogue.SettingDeclaration{"port": {Type: "number"}})
|
||||
fakeManifests(t, map[string]*catalogue.Manifest{baseSHA: m, headSHA: m})
|
||||
calls := fakeConfig(t, nil)
|
||||
tellPullDeclares(context.Background(), io.Discard, []pullAt{thePull()})
|
||||
if len(*calls) != 1 {
|
||||
t.Fatalf("calls %v", *calls)
|
||||
}
|
||||
if d, _ := json.Marshal((*calls)[0]["declares"]); string(d) != "{}" {
|
||||
t.Fatalf("declares %s", d)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC1: mesh-config not answering is said and goes on; the check is not stopped.
|
||||
func TestMeshConfigNotAnsweringIsSaidAndGoneOn(t *testing.T) {
|
||||
fakeManifests(t, map[string]*catalogue.Manifest{baseSHA: manifest("n8n", nil),
|
||||
headSHA: manifest("n8n", map[string]catalogue.SettingDeclaration{"k": {Type: "string"}})})
|
||||
was := meshConfigAsk
|
||||
t.Cleanup(func() { meshConfigAsk = was })
|
||||
meshConfigAsk = func(context.Context, map[string]any) ([]byte, error) { return nil, io.ErrUnexpectedEOF }
|
||||
var said strings.Builder
|
||||
tellPullDeclares(context.Background(), &said, []pullAt{thePull()})
|
||||
got := heldFor(context.Background(), &said, []pullAt{thePull()})
|
||||
if len(got) != 0 || !strings.Contains(said.String(), "mesh-config") {
|
||||
t.Fatalf("held %v, said %q", got, said.String())
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC2: the held values mesh-config answers for the check's pull requests are what the check request carries.
|
||||
func TestTheHeldValuesOfTheChecksPullRequestsAreAsked(t *testing.T) {
|
||||
calls := fakeConfig(t, map[string]string{"held": `{"held":[{"pull":"novox/mesh-catalog#434","head":"` + headSHA +
|
||||
`","node":"ace","module":"n8n","key":"media-gid","value":2000,"approved":"the operator"}]}`})
|
||||
other := pullAt{Number: 7, changeSource: changeSource{Owner: "novox", Repo: "mesh-controller", New: baseSHA}}
|
||||
got := heldFor(context.Background(), io.Discard, []pullAt{thePull(), other})
|
||||
if len(got) != 1 || got[0].Node != "ace" || got[0].Key != "media-gid" || string(got[0].Value) != "2000" {
|
||||
t.Fatalf("got %+v", got)
|
||||
}
|
||||
c := (*calls)[0]
|
||||
pulls, _ := json.Marshal(c["pulls"])
|
||||
if c["action"] != "held" || string(pulls) != `[{"pull":"novox/mesh-catalog#434","head":"`+headSHA+
|
||||
`"},{"pull":"novox/mesh-controller#7","head":"`+baseSHA+`"}]` {
|
||||
t.Fatalf("asked %v %s", c, pulls)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC2: a value held for a pull request the check is not of is never carried.
|
||||
func TestAHeldValueOfAnotherPullRequestIsNotCarried(t *testing.T) {
|
||||
fakeConfig(t, map[string]string{"held": `{"held":[{"pull":"novox/mesh-catalog#999","head":"x","node":"ace","module":"n8n","key":"k","value":1}]}`})
|
||||
if got := heldFor(context.Background(), io.Discard, []pullAt{thePull()}); len(got) != 0 {
|
||||
t.Fatalf("carried %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC2: the check request carries what mesh-config answered; a member of a composed group asks none.
|
||||
func TestTheCheckRequestCarriesTheHeldValues(t *testing.T) {
|
||||
fakeManifests(t, map[string]*catalogue.Manifest{baseSHA: manifest("n8n", nil), headSHA: manifest("n8n", nil)})
|
||||
fakeConfig(t, map[string]string{"held": `{"held":[{"pull":"novox/mesh-catalog#434","node":"ace","module":"n8n","key":"media-gid","value":2000}]}`})
|
||||
request := link.BuildRequest{Check: &link.CheckRequest{}}
|
||||
withHeldConfig(context.Background(), io.Discard, &request, []pullAt{thePull()}, true)
|
||||
if len(request.Check.HeldConfig) != 1 || request.Check.HeldConfig[0].Key != "media-gid" {
|
||||
t.Fatalf("the request carries %+v", request.Check.HeldConfig)
|
||||
}
|
||||
request = link.BuildRequest{Check: &link.CheckRequest{}}
|
||||
withHeldConfig(context.Background(), io.Discard, &request, []pullAt{thePull()}, false)
|
||||
if len(request.Check.HeldConfig) != 0 {
|
||||
t.Fatalf("a composed member carries %+v", request.Check.HeldConfig)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC1: a repository that builds a module of the mesh holds modules; another does not.
|
||||
func TestARepositoryThatBuildsAModuleHoldsModules(t *testing.T) {
|
||||
entries := []inventory.Entry{{Manifest: catalogue.Manifest{Module: "a"}, Source: inventory.Source{Repository: "novox/mesh-catalog", Path: "modules/a"}}}
|
||||
if !holdsModules(entries, "novox", "mesh-catalog") || holdsModules(entries, "novox", "hq") {
|
||||
t.Fatal("wrong")
|
||||
}
|
||||
}
|
||||
@@ -281,11 +281,7 @@ func serve(ctx context.Context) (err error) {
|
||||
// The provisioner's plan of each handed module, as its machine was last sent it (design 57 P6), in the module's
|
||||
// assignment.
|
||||
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
|
||||
go mirroringRecords(ctx, open, inv, bus.Conn, bus)
|
||||
// And the bucket of the files that travel by digest, judged against what each machine's latest declaration names (ADR 0486 §3).
|
||||
go sweepingDeclarationFiles(ctx, inv, bus.Conn)
|
||||
// And the key of each plan in its build phase, which holds the build seat's checks while a delivery builds (ADR 0483 clause 2).
|
||||
go keepingPlansBuilding(ctx, inv, bus.Conn)
|
||||
go mirroringRecords(ctx, inv, bus.Conn, bus)
|
||||
// Every call carries the lease's epoch, and its record is written only under the lease (novox/hq
|
||||
// to-be 45 §6).
|
||||
link.Calls.UnderLease(func() (uint64, error) { return theLease.epoch(ctx) })
|
||||
@@ -983,18 +979,6 @@ func deliver(ctx context.Context, d delivery, holder string, sending []readyNode
|
||||
return err
|
||||
}
|
||||
digest, err := d.declare(ctx, s, body)
|
||||
if errors.Is(err, errFilesNotPut) && s.declared.fileReferences {
|
||||
// **The grant that lets the controller write the bucket travels in the very declaration of the machine holding the
|
||||
// bus** (novox/hq ADR 0486 §2): until the bus has it, a put is refused, and the declaration that would deliver the
|
||||
// grant would wait for it. So that send, and only that, goes with every content inline, as to a node that reads no
|
||||
// references, and says so; the next one tries the bucket again.
|
||||
fmt.Printf("%s: the files that travel by digest could not be put in the bucket (%v): sending them inline\n", s.node, err)
|
||||
s.declared.fileReferences = false
|
||||
if body, err = s.declared.Body(); err != nil {
|
||||
return err
|
||||
}
|
||||
digest, err = d.declare(ctx, s, body)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1056,35 +1040,6 @@ type overTheBus struct {
|
||||
indent string
|
||||
}
|
||||
|
||||
// putFiles puts the contents s names by digest in the bucket.
|
||||
func (b overTheBus) putFiles(ctx context.Context, s readyNode) error {
|
||||
objects, err := s.declared.objects()
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: the files that travel by digest cannot be read: %w", s.node, err)
|
||||
}
|
||||
if len(objects) == 0 {
|
||||
return nil
|
||||
}
|
||||
bus, ok := b.server.Bus().(link.OverNATS)
|
||||
if !ok {
|
||||
return fmt.Errorf("%s: the declaration names files by digest and this bus has no object bucket", s.node)
|
||||
}
|
||||
putting, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
store, err := openBucket(putting, bus.Conn, s.node)
|
||||
if err == nil {
|
||||
err = putObjects(putting, store, objects)
|
||||
}
|
||||
notePut(ctx, s.node, err)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s was not sent its declaration: %w: %w", s.node, errFilesNotPut, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// errFilesNotPut marks a send that stopped because the files it names by digest could not be put in the bucket.
|
||||
var errFilesNotPut = errors.New("the files it names by digest are not in the bucket")
|
||||
|
||||
func (b overTheBus) grant(ctx context.Context, sending []readyNode) error {
|
||||
// The bus's objects first, which every declaration implies (novox/hq issue 208): said and raised
|
||||
// when they cannot be, and never what holds the send back (assertOnSend).
|
||||
@@ -1097,20 +1052,9 @@ func (b overTheBus) grant(ctx context.Context, sending []readyNode) error {
|
||||
}
|
||||
|
||||
func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
|
||||
// The contents the declaration names by digest are in the bucket before it is sent (novox/hq ADR 0486 §1, issue 1696): a node
|
||||
// that applies it finds every one. A node reading no references has none.
|
||||
// **Under the node's lock** from the put to the publication, so the sweep of its bucket cannot remove an object between them.
|
||||
unlock := lockFilesOf(s.node)
|
||||
err := b.putFiles(ctx, s)
|
||||
if err != nil {
|
||||
unlock()
|
||||
if err := link.Declare(ctx, b.server.Bus(), b.signer, s.node, body, 15*time.Second); err != nil {
|
||||
return "", err
|
||||
}
|
||||
declared := link.Declare(ctx, b.server.Bus(), b.signer, s.node, body, 15*time.Second)
|
||||
unlock()
|
||||
if declared != nil {
|
||||
return "", declared
|
||||
}
|
||||
// After it is away, not before. A digest recorded for something that failed to send would make
|
||||
// the machine look current for a declaration it never received.
|
||||
digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds, s.declared.Epoch,
|
||||
|
||||
@@ -91,9 +91,7 @@ type queueAnswer struct {
|
||||
Dead int `json:"dead"`
|
||||
MaxDeliver int `json:"max-deliver"`
|
||||
Asks []queuedAnswer `json:"asks"`
|
||||
// HeldFor are the delivery plans building, while which the waiting checks are held (ADR 0483 clause 5); empty when none.
|
||||
HeldFor []string `json:"held-for,omitempty"`
|
||||
Text string `json:"text"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
// queuedAnswer is one ask as data. Never what the request carries beyond its source — not `held`, every
|
||||
@@ -122,9 +120,6 @@ type queuedAnswer struct {
|
||||
// Delivery is the how-manyth time an ask in flight was handed out; more than 1 is a restart
|
||||
// (novox/hq issue 594).
|
||||
Delivery int `json:"delivery,omitempty"`
|
||||
// LetThrough is a check that waited past 30 minutes and is let through while a delivery builds: the next one, or the one
|
||||
// that runs (ADR 0483 clause 5).
|
||||
LetThrough bool `json:"let-through,omitempty"`
|
||||
}
|
||||
|
||||
// aCommitRef is a ref that names a commit by its full hash (SHA-1 or SHA-256).
|
||||
@@ -135,8 +130,7 @@ var aCommitRef = regexp.MustCompile(`^(?:[0-9a-f]{40}|[0-9a-f]{64})$`)
|
||||
func queueAnswerOf(q link.Queue, now time.Time) queueAnswer {
|
||||
waiting, running, dead := q.Of(link.AskWaiting), q.Of(link.AskInFlight), q.Of(link.AskDead)
|
||||
out := queueAnswer{Seat: q.Seat, Waiting: len(waiting), InFlight: len(running), Dead: len(dead),
|
||||
MaxDeliver: q.MaxDeliver, Asks: []queuedAnswer{}, HeldFor: q.Hold.Plans, Text: queueText(q, now)}
|
||||
letNext, letRunning := q.LetThroughOf(now)
|
||||
MaxDeliver: q.MaxDeliver, Asks: []queuedAnswer{}, Text: queueText(q, now)}
|
||||
one := func(a link.QueuedAsk, position int) queuedAnswer {
|
||||
kind := link.KindOf(a.Request)
|
||||
since := a.AskedAt
|
||||
@@ -149,7 +143,7 @@ func queueAnswerOf(q link.Queue, now time.Time) queueAnswer {
|
||||
}
|
||||
return queuedAnswer{ID: a.ID, State: a.State, Kind: kind, Repository: link.WithoutPassword(a.Repository), Path: a.Path,
|
||||
Ref: a.Ref, Commit: commit, Machine: a.On, WasOn: a.Was, AskedAt: a.AskedAt, Since: since,
|
||||
Position: position, Seq: a.Seq, Delivery: a.Delivery, LetThrough: a.ID != "" && (a.ID == letNext || a.ID == letRunning)}
|
||||
Position: position, Seq: a.Seq, Delivery: a.Delivery}
|
||||
}
|
||||
for _, a := range running {
|
||||
out.Asks = append(out.Asks, one(a, 0))
|
||||
@@ -185,11 +179,6 @@ func queueText(q link.Queue, now time.Time) string {
|
||||
}
|
||||
return s
|
||||
}
|
||||
letNext, letRunning := q.LetThroughOf(now)
|
||||
const passed = "waited past 30 minutes, let through"
|
||||
if q.HoldFault != "" {
|
||||
fmt.Fprintf(&b, "why checks wait cannot be read, so none is said held: %s\n", q.HoldFault)
|
||||
}
|
||||
if len(running) > 0 {
|
||||
fmt.Fprintln(&b, "\nin flight:")
|
||||
for _, a := range running {
|
||||
@@ -204,27 +193,14 @@ func queueText(q link.Queue, now time.Time) string {
|
||||
case a.Was != "":
|
||||
where = fmt.Sprintf("handed back by %s, to be handed out again", a.Was)
|
||||
}
|
||||
note := ""
|
||||
if a.ID != "" && a.ID == letRunning {
|
||||
note = " — " + passed
|
||||
}
|
||||
fmt.Fprintf(&b, " %-26s %-8s %s — %s, %s (seq %d)%s\n", a.ID, link.KindOf(a.Request), what(a), where,
|
||||
ago(a.AskedAt), a.Seq, note)
|
||||
fmt.Fprintf(&b, " %-26s %-8s %s — %s, %s (seq %d)\n", a.ID, link.KindOf(a.Request), what(a), where,
|
||||
ago(a.AskedAt), a.Seq)
|
||||
}
|
||||
}
|
||||
if len(waiting) > 0 {
|
||||
fmt.Fprintln(&b, "\nwaiting, in the order they are handed out:")
|
||||
// **Why the checks wait** (ADR 0483 clause 5): above them, the plans they are held for.
|
||||
if len(q.Hold.Plans) > 0 && hasCheck(waiting) {
|
||||
fmt.Fprintf(&b, " checks are held while delivery %s builds; the oldest check that waited 30 minutes goes first, one at a time\n",
|
||||
strings.Join(q.Hold.Plans, ", "))
|
||||
}
|
||||
for _, a := range waiting {
|
||||
note := ""
|
||||
if a.ID != "" && a.ID == letNext {
|
||||
note = " — " + passed + " next"
|
||||
}
|
||||
fmt.Fprintf(&b, " %-26s %-8s %s — %s (seq %d)%s\n", a.ID, link.KindOf(a.Request), what(a), ago(a.AskedAt), a.Seq, note)
|
||||
fmt.Fprintf(&b, " %-26s %-8s %s — %s (seq %d)\n", a.ID, link.KindOf(a.Request), what(a), ago(a.AskedAt), a.Seq)
|
||||
}
|
||||
}
|
||||
if len(dead) > 0 {
|
||||
@@ -242,16 +218,6 @@ func queueText(q link.Queue, now time.Time) string {
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// hasCheck says a check is among the asks.
|
||||
func hasCheck(asks []link.QueuedAsk) bool {
|
||||
for _, a := range asks {
|
||||
if a.Request.Check != nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// cancelCommand drops one waiting or dead ask.
|
||||
func cancelCommand(ctx context.Context, args []string) error {
|
||||
if len(args) != 1 {
|
||||
|
||||
@@ -1,77 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// Issue 1688 / 1652 AC5 (ADR 0483 clause 5): the queue names, above the waiting checks, the plan they are held for, and says of a
|
||||
// check let through that it waited past 30 minutes.
|
||||
|
||||
func aHeldQueue(now time.Time) link.Queue {
|
||||
check := func(id string, ago time.Duration, n int) link.QueuedAsk {
|
||||
return link.QueuedAsk{Seq: uint64(n), ID: id, Repository: "ssh://forge/novox/r", AskedAt: now.Add(-ago), State: link.AskWaiting,
|
||||
Request: link.BuildRequest{ID: id, Check: &link.CheckRequest{Owner: "novox", Repo: "r", Number: n}}}
|
||||
}
|
||||
return link.Queue{Seat: link.TheBuildMachine, Asks: []link.QueuedAsk{
|
||||
check("build-young", 5*time.Minute, 1),
|
||||
check("build-old", 45*time.Minute, 2),
|
||||
check("build-older", 50*time.Minute, 3),
|
||||
}, Hold: link.CheckHold{Plans: []string{"plan-42"}}}
|
||||
}
|
||||
|
||||
func TestTheQueueNamesThePlanTheChecksAreHeldFor(t *testing.T) {
|
||||
now := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)
|
||||
text := queueText(aHeldQueue(now), now)
|
||||
heldAt := strings.Index(text, "held while delivery plan-42 builds")
|
||||
firstAt := strings.Index(text, "build-young")
|
||||
if heldAt < 0 || firstAt < 0 || heldAt > firstAt {
|
||||
t.Fatalf("the plan the checks are held for is not named above the waiting checks:\n%s", text)
|
||||
}
|
||||
// Nothing building: nothing held.
|
||||
q := aHeldQueue(now)
|
||||
q.Hold = link.CheckHold{}
|
||||
if strings.Contains(queueText(q, now), "held while") {
|
||||
t.Fatal("checks are said held with no delivery building")
|
||||
}
|
||||
}
|
||||
|
||||
func TestACheckLetThroughIsSaidToHaveWaitedPastThirtyMinutes(t *testing.T) {
|
||||
now := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)
|
||||
q := aHeldQueue(now)
|
||||
text := queueText(q, now)
|
||||
var saidOf []string
|
||||
for _, l := range strings.Split(text, "\n") {
|
||||
if strings.Contains(l, "waited past 30 minutes, let through") {
|
||||
saidOf = append(saidOf, strings.Fields(l)[0])
|
||||
}
|
||||
}
|
||||
if len(saidOf) != 1 || saidOf[0] != "build-older" {
|
||||
t.Fatalf("the check let through next (the oldest) is not the one said: %v\n%s", saidOf, text)
|
||||
}
|
||||
answer := queueAnswerOf(q, now)
|
||||
if len(answer.HeldFor) != 1 || answer.HeldFor[0] != "plan-42" {
|
||||
t.Errorf("held-for = %v", answer.HeldFor)
|
||||
}
|
||||
for _, a := range answer.Asks {
|
||||
if a.LetThrough != (a.ID == "build-older") {
|
||||
t.Errorf("%s let-through = %v", a.ID, a.LetThrough)
|
||||
}
|
||||
}
|
||||
// A check in flight under the let-through key is said too.
|
||||
q.Asks = append(q.Asks, link.QueuedAsk{Seq: 9, ID: "build-running", State: link.AskInFlight, On: "ace", AskedAt: now.Add(-time.Hour),
|
||||
Started: now.Add(-time.Minute), Request: link.BuildRequest{ID: "build-running", Check: &link.CheckRequest{Owner: "novox", Repo: "r", Number: 9}}})
|
||||
q.Hold.LetThrough = &link.LetThrough{Builder: "ace"}
|
||||
if !strings.Contains(queueText(q, now), "build-running") || !strings.Contains(queueText(q, now), "waited past 30 minutes, let through\n") {
|
||||
t.Errorf("the check in flight let through is not said:\n%s", queueText(q, now))
|
||||
}
|
||||
// Younger than the bound: none is said.
|
||||
young := aHeldQueue(now)
|
||||
young.Asks = young.Asks[:1]
|
||||
if strings.Contains(queueText(young, now), "waited past") {
|
||||
t.Error("a check of 5 minutes is said to have waited past 30")
|
||||
}
|
||||
}
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
@@ -30,7 +29,7 @@ import (
|
||||
const mirrorEvery = 10 * time.Second
|
||||
|
||||
// mirroringRecords keeps the mirrors current for as long as this controller serves: ctx ends when it stops.
|
||||
func mirroringRecords(ctx context.Context, open *stores, inv *inventory.Inventory, conn *nats.Conn, announce recordmirror.Announcer) {
|
||||
func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn, announce recordmirror.Announcer) {
|
||||
api, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
fmt.Printf("the record mirrors are not kept: %v\n", err)
|
||||
@@ -45,9 +44,8 @@ func mirroringRecords(ctx context.Context, open *stores, inv *inventory.Inventor
|
||||
return e
|
||||
}
|
||||
sources := map[string]func(context.Context) (map[string][]byte, error){
|
||||
recordmirror.DeclaredConfig.Name: inv.DeclaredConfigMirror,
|
||||
recordmirror.Predefined.Name: inv.PredefinedMirror,
|
||||
recordmirror.DeclaredPackages.Name: declaredPackagesMirror(open),
|
||||
recordmirror.DeclaredConfig.Name: inv.DeclaredConfigMirror,
|
||||
recordmirror.Predefined.Name: inv.PredefinedMirror,
|
||||
}
|
||||
writers := map[string]*recordmirror.Writer{}
|
||||
failing := map[string]string{}
|
||||
@@ -178,36 +176,3 @@ func keepingAssignments(ctx context.Context, inv *inventory.Inventory, announce
|
||||
}
|
||||
events.known = now
|
||||
}
|
||||
|
||||
// declaredPackagesMirror is the source of the declared-packages mirror (ADR 0507 §5): each node's composition, asked as a
|
||||
// question that makes nothing, and the packages it declares with the modules that declare them. A node whose own set does
|
||||
// not compose has no key, and any other failure to read is the pass's error, so a key is never purged for a store that
|
||||
// could not be read.
|
||||
func declaredPackagesMirror(open *stores) func(context.Context) (map[string][]byte, error) {
|
||||
return func(ctx context.Context) (map[string][]byte, error) {
|
||||
in, err := inputFor(ctx, open)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Which module owns each resource is carried only for a node-engine that reads it, so every node is asked as one that does.
|
||||
asked := in.Input
|
||||
asked.Nodes = append([]compose.Node(nil), in.Nodes...)
|
||||
for i := range asked.Nodes {
|
||||
asked.Nodes[i].ReadsModules = true
|
||||
}
|
||||
out := map[string][]byte{}
|
||||
for _, n := range in.sortedNodes() {
|
||||
d, _, err := compose.ComposeIn(asked, n.Name, compose.Foreseeing)
|
||||
if compose.IsUnresolvable(err) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the packages %s declares cannot be read: %w", n.Name, err)
|
||||
}
|
||||
if out[n.Name], err = inventory.DeclaredPackagesValue(n.Name, d.Resources, d.ModuleOf); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -181,24 +181,15 @@ func planModuleNames(p inventory.Plan) map[string]string {
|
||||
return out
|
||||
}
|
||||
|
||||
// heldMove is one move of a recorded module that waits for a person on one machine (ADR 0377 §6).
|
||||
type heldMove struct {
|
||||
Module string
|
||||
From, To string
|
||||
// Why is the walk's own word for it: "waits: " and the recreate words, the data change, or the hold.
|
||||
Why string
|
||||
// Chosen is whether a person's own choice holds it — a module they set to record, or the bus's step — which the
|
||||
// operator is not asked about again: they chose that hold, and the bus has its own condition (S17).
|
||||
Chosen bool
|
||||
}
|
||||
|
||||
// heldMovesOn is, on one machine, every move of a recorded module that waits for a person, in module order.
|
||||
func heldMovesOn(ctx context.Context, inv *inventory.Inventory, f moveFacts, node string) ([]heldMove, error) {
|
||||
// recordedWaiting is, on one machine, every move of a recorded module that waits for a person, each said with why
|
||||
// (ADR 0377 §6): what `status` names the machine behind for, with `push <node>` as the remedy. A move under a
|
||||
// module's own `record` that recreates nothing and moves no data is not among them: it goes with its walk.
|
||||
func recordedWaiting(ctx context.Context, inv *inventory.Inventory, f moveFacts, node string) ([]string, error) {
|
||||
sent, known, err := inv.SentBuilds(ctx, node)
|
||||
if err != nil || !known {
|
||||
return nil, err
|
||||
}
|
||||
var out []heldMove
|
||||
var out []string
|
||||
for _, m := range sortedKeysOf(sent) {
|
||||
was := sent[m]
|
||||
now, held := f.current[m]
|
||||
@@ -206,25 +197,12 @@ func heldMovesOn(ctx context.Context, inv *inventory.Inventory, f moveFacts, nod
|
||||
continue
|
||||
}
|
||||
if v := f.recordedMove(m, was); !v.rolls {
|
||||
out = append(out, heldMove{Module: m, From: was, To: now.Commit, Why: v.said,
|
||||
Chosen: now.From == catalogue.FromPerson || now.From == catalogue.FromBus})
|
||||
out = append(out, fmt.Sprintf("%s %s → %s %s", m, short(was), short(now.Commit), v.said))
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// recordedWaiting is, on one machine, every move of a recorded module that waits for a person, each said with why
|
||||
// (ADR 0377 §6): what `status` names the machine behind for, with `push <node>` as the remedy. A move under a
|
||||
// module's own `record` that recreates nothing and moves no data is not among them: it goes with its walk.
|
||||
func recordedWaiting(ctx context.Context, inv *inventory.Inventory, f moveFacts, node string) ([]string, error) {
|
||||
moves, err := heldMovesOn(ctx, inv, f, node)
|
||||
var out []string
|
||||
for _, h := range moves {
|
||||
out = append(out, fmt.Sprintf("%s %s → %s %s", h.Module, short(h.From), short(h.To), h.Why))
|
||||
}
|
||||
return out, err
|
||||
}
|
||||
|
||||
// stringsKeys is a map of lists as a map of its keys, for sortedKeysOf.
|
||||
func stringsKeys(m map[string][]string) map[string]string {
|
||||
out := map[string]string{}
|
||||
|
||||
@@ -1371,10 +1371,6 @@ func backlogObservation() []conditions.Observation {
|
||||
"them, `upgrade release-backlog --why …` releases them", n, strings.Join(machines, ", "), backlogNow.held)}
|
||||
w := releaseHeldWords(modules, machines)
|
||||
o.Headline, o.Explanation, o.Resolved, o.Needs = w.Headline, w.Explanation, w.Resolved, w.Needs
|
||||
// Asked of the operator, who releases them with an Approve (novox/hq issue 1732): the controller's own
|
||||
// `upgrade release-backlog`, which opens the release plan that walks them through the machines, each judged.
|
||||
o.Actions = []conditions.Action{{Label: "Release", Verb: "mesh-controller.upgrade", Level: conditions.LevelApprove,
|
||||
Arguments: map[string]string{"release-backlog": "true", "why": "", "cause": conditions.CauseOperatorAnswer}}}
|
||||
return []conditions.Observation{o}
|
||||
}
|
||||
|
||||
|
||||
@@ -577,9 +577,6 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
|
||||
}
|
||||
// **The walk takes the check's build of the same key** (novox/hq ADR 0350 §2): its ask says so, with the pull
|
||||
// requests it answers, whose checks name the part of a differing key.
|
||||
// **The plan's key is there before its first build is** (ADR 0483 clause 2): a check taken between the ask and the plan's save
|
||||
// would hold a place for its whole run.
|
||||
sayBuilding(ctx, p.ID, now)
|
||||
id, err := askABuild(withReuse(ctx, reuseWithVerdicts(ctx, p.ID, reuseAskOf(p), mergedHeadsOf(p))), source,
|
||||
e.Source.Path, ref)
|
||||
if err != nil {
|
||||
@@ -628,124 +625,16 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string,
|
||||
planBuiltSaying(ctx, open, module, commit, failed, asked, id, "")
|
||||
}
|
||||
|
||||
// holdPlansForOutcome is how an outcome taken on the event loop holds the plans: **without waiting for them**
|
||||
// (novox/hq issue 1708). The loop takes one message at a time, and the plans' holder may keep them for minutes while
|
||||
// it sends machines their declarations; an outcome that waited its turn (up to inventory.HoldWaitFor, two minutes)
|
||||
// held every report, build and merge behind it for as long, and the watchdog said the controller stopped listening
|
||||
// (2026-10-21 14:20:45 to 14:22:45). Busy, the outcome is left to the build records, where it already is, and the
|
||||
// look settles the plan from them (leaveToRecords).
|
||||
func holdPlansForOutcome(ctx context.Context, hold func(context.Context, bool) (func(), error)) (func(), error) {
|
||||
return hold(ctx, false)
|
||||
}
|
||||
|
||||
// leaveToRecords says an outcome was not taken into the plans, which settle it from the build records (novox/hq issue
|
||||
// 214). When another act holds the plans it asks for the look, which waits for them off the event loop and settles
|
||||
// the plan from the records once they are let go (issue 1708); what the build did is kept for that settle by the
|
||||
// build's id, so the walk's module says it as it would have (ADR 0350 §4).
|
||||
func leaveToRecords(module, id, made, failed string, err error) {
|
||||
fmt.Printf("plans: %s's outcome is left to the build records: %v\n", module, err)
|
||||
if !errors.Is(err, inventory.ErrPlansBusy) {
|
||||
return
|
||||
}
|
||||
keepMadeFor(id, made, time.Now())
|
||||
// A refused outcome (failed, its build recorded as worked) is kept as refused, or the records settle it built.
|
||||
keepRefusalFor(id, failed, time.Now())
|
||||
askToLook()
|
||||
}
|
||||
|
||||
// refusalsKeptFor is the refusals kept in this process for these builds (keepRefusalFor), by id.
|
||||
func refusalsKeptFor(ids []string) map[string]string {
|
||||
out := map[string]string{}
|
||||
refusedLeftMu.Lock()
|
||||
defer refusedLeftMu.Unlock()
|
||||
for _, id := range ids {
|
||||
if k, ok := refusedLeft[id]; ok {
|
||||
out[id] = k.made
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// refusedLeft is why each outcome left to the records was refused, by the build's id: kept in the process beside the
|
||||
// store's record (RecordRefused), so a store that did not take it still fails the plan while this controller serves.
|
||||
var (
|
||||
refusedLeftMu sync.Mutex
|
||||
refusedLeft = map[string]madeKept{}
|
||||
)
|
||||
|
||||
// keepRefusalFor keeps why an outcome left to the records was refused, forgetting refusals older than madeLeftFor.
|
||||
func keepRefusalFor(id, why string, now time.Time) {
|
||||
if id == "" || why == "" {
|
||||
return
|
||||
}
|
||||
refusedLeftMu.Lock()
|
||||
defer refusedLeftMu.Unlock()
|
||||
for k, v := range refusedLeft {
|
||||
if now.Sub(v.at) > madeLeftFor {
|
||||
delete(refusedLeft, k)
|
||||
}
|
||||
}
|
||||
refusedLeft[id] = madeKept{made: why, at: now}
|
||||
}
|
||||
|
||||
// madeLeft is what each build left to the records did, by the build's id, until a settle takes it: kept in the
|
||||
// process, so a controller replaced before the settle loses only these words, never the outcome.
|
||||
var (
|
||||
madeLeftMu sync.Mutex
|
||||
madeLeft = map[string]madeKept{}
|
||||
)
|
||||
|
||||
type madeKept struct {
|
||||
made string
|
||||
at time.Time
|
||||
}
|
||||
|
||||
// madeLeftFor is how long words nobody settled are kept: a plan that ended or moved on never asks for them.
|
||||
const madeLeftFor = time.Hour
|
||||
|
||||
// keepMadeFor keeps what a build left to the records did for the settle that takes it, forgetting words older than
|
||||
// madeLeftFor.
|
||||
func keepMadeFor(id, made string, now time.Time) {
|
||||
if id == "" || made == "" {
|
||||
return
|
||||
}
|
||||
madeLeftMu.Lock()
|
||||
defer madeLeftMu.Unlock()
|
||||
for k, v := range madeLeft {
|
||||
if now.Sub(v.at) > madeLeftFor {
|
||||
delete(madeLeft, k)
|
||||
}
|
||||
}
|
||||
madeLeft[id] = madeKept{made: made, at: now}
|
||||
}
|
||||
|
||||
// takeMadeOf is what the build of this id did, if its outcome was left to the records with words; taken once.
|
||||
func takeMadeOf(id string) string {
|
||||
madeLeftMu.Lock()
|
||||
defer madeLeftMu.Unlock()
|
||||
k, ok := madeLeft[id]
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
delete(madeLeft, id)
|
||||
return k.made
|
||||
}
|
||||
|
||||
// planBuiltSaying is planBuilt keeping what the build did — "reused from <check build>" or "built: <why>" (novox/hq
|
||||
// ADR 0350 §4) — on the module of the walk it answers.
|
||||
func planBuiltSaying(ctx context.Context, open *stores, module, commit, failed string, asked time.Time, id, made string) {
|
||||
inv := open.inventory
|
||||
// One controller works the plans at a time (novox/hq issue 213). Not taken, the outcome is still in the build
|
||||
// records, which the holder settles the plan from (issue 214).
|
||||
release, err := holdPlansForOutcome(ctx, inv.HoldPlans)
|
||||
// One controller works the plans at a time (novox/hq issue 213); an outcome waits its turn rather
|
||||
// than write over what the holder is about to save. Not taken, it is still in the build records,
|
||||
// which the holder settles the plan from (issue 214).
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
if failed != "" && id != "" && errors.Is(err, inventory.ErrPlansBusy) {
|
||||
// In the store too, so a controller replaced before the look settles still fails the plan (issue 1708).
|
||||
if err := inv.RecordRefused(ctx, id, module, failed); err != nil {
|
||||
fmt.Printf("plans: %s's refusal could not be kept in the store, only in this controller: %v\n", module, err)
|
||||
}
|
||||
}
|
||||
leaveToRecords(module, id, made, failed, err)
|
||||
fmt.Printf("plans: %s's outcome is left to the build records: %v\n", module, err)
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
@@ -807,19 +696,7 @@ func planBuiltSaying(ctx context.Context, open *stores, module, commit, failed s
|
||||
fmt.Printf("%s: %s; the tiers after it are not asked\n", p.ID, p.Note)
|
||||
}
|
||||
}
|
||||
advanceAfterOutcome(func() { advanceHeld(ctx, open) })
|
||||
}
|
||||
|
||||
// advanceAfterOutcome moves the plans after an outcome was taken into them: **in the serving controller, by asking
|
||||
// the look** (novox/hq issue 1708). The outcome is taken on the event loop, and moving the plans may send machines
|
||||
// their declarations, a release's send retried for minutes among them; done here it held every report, build and
|
||||
// merge behind it (2026-10-21 14:28:40 to 14:32:10). The look runs off the loop, holding the plans itself. Where no
|
||||
// look runs (a command, which has no event loop) the plans are moved at once, as before.
|
||||
func advanceAfterOutcome(advance func()) {
|
||||
if askedToLook() {
|
||||
return
|
||||
}
|
||||
advance()
|
||||
advanceHeld(ctx, open)
|
||||
}
|
||||
|
||||
// startedBeside is the id of a started walk open beside this one that it is related to — a merge's walk past
|
||||
@@ -854,23 +731,6 @@ func startedBeside(ctx context.Context, inv *inventory.Inventory, p inventory.Pl
|
||||
return behind, why, nil
|
||||
}
|
||||
|
||||
// refusalNotePrefix and refusalNoteSuffix frame the note of a step the walk was refused: "tier N: <refusal> — tried
|
||||
// again". Only a note so framed is a hold (novox/hq issue 920); a tier's summary left in the note is not.
|
||||
const (
|
||||
refusalNotePrefix = "tier "
|
||||
refusalNoteSuffix = " — tried again"
|
||||
)
|
||||
|
||||
// refusalNote is the note of a plan whose step at tier was refused with err.
|
||||
func refusalNote(tier int, err error) string {
|
||||
return refusalNotePrefix + fmt.Sprint(tier) + ": " + err.Error() + refusalNoteSuffix
|
||||
}
|
||||
|
||||
// refused says a plan's note is a refusal the walk wrote.
|
||||
func refused(p inventory.Plan) bool {
|
||||
return strings.HasPrefix(p.Note, refusalNotePrefix) && strings.HasSuffix(p.Note, refusalNoteSuffix)
|
||||
}
|
||||
|
||||
// deferredNote begins the note of a walk deferred behind another.
|
||||
const deferredNote = "let go; starts once "
|
||||
|
||||
@@ -883,20 +743,6 @@ func deferred(p inventory.Plan) bool {
|
||||
p.Delivery.Awaits != "" && p.Delivery.Go != nil && strings.HasPrefix(p.Note, deferredNote)
|
||||
}
|
||||
|
||||
// currentTierBuilt says every module of a plan's current tier is built (or deleted), so no build is what it waits on.
|
||||
func currentTierBuilt(p inventory.Plan) bool {
|
||||
if p.Tier < 0 || p.Tier >= len(p.Tiers) || len(p.Tiers[p.Tier]) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
s := p.Modules[m]
|
||||
if s == nil || (s.State != "built" && s.State != planDeleted) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// askedAny says a plan asked any module.
|
||||
func askedAny(p inventory.Plan) bool {
|
||||
for _, s := range p.Modules {
|
||||
@@ -961,7 +807,7 @@ func advanceHeld(ctx context.Context, open *stores) (failed bool) {
|
||||
// Kept in the plan, so `plans` says why it has not moved rather than the log alone;
|
||||
// the state is left as it was and the step is tried again by the next look. Said and
|
||||
// kept when it is new: the same refusal on every tick is one fact, not one per tick.
|
||||
p.Note = refusalNote(p.Tier, err)
|
||||
p.Note = "tier " + fmt.Sprint(p.Tier) + ": " + err.Error() + " — tried again"
|
||||
// A refusal for the bus's planned step is a person's to end: S17 says it from its first moment.
|
||||
if refusedForTheBus(err) {
|
||||
busRefusedFirst.mark(p.ID, time.Now())
|
||||
@@ -1075,17 +921,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
before, was := statesOf(p), p.State
|
||||
if settleFromRecords(p, tier, recorded, byID) {
|
||||
// As taking the outcome directly does (planBuiltSaying): a module deleted at its source is forgotten, and a
|
||||
// failed plan says what it built and never sent (issue 1708).
|
||||
deleted, failed := settledEffects(before, p, was)
|
||||
for _, m := range deleted {
|
||||
forgetDeleted(ctx, inv, m, p)
|
||||
}
|
||||
if failed {
|
||||
sayUnsent(p, rollsOut)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
// Asked: wait for every build.
|
||||
@@ -2059,19 +1895,6 @@ func planLineSaid(p inventory.Plan, now time.Time, pause pauseView, bound time.D
|
||||
if p.State == inventory.PlanRolling {
|
||||
what = p.Note
|
||||
}
|
||||
if p.State == inventory.PlanBuilding && refused(p) && currentTierBuilt(p) {
|
||||
// Every build of the tier is in, so the plan waits on something else, and its note says what: a send
|
||||
// refused ("a build waits there for its gate: <machine> (...)") is kept there. "building" with LATE
|
||||
// hid that reason from `plans`, `status` and the stalled condition (novox/hq issue 920).
|
||||
what = p.Note
|
||||
// And when the builds are held after a failed release, they wait for a person, not for a build, and the
|
||||
// operator is asked to release them (novox/hq issue 1732): said so, and not late.
|
||||
if strings.Contains(p.Note, errUngated.Error()) && backlogNow.held != "" {
|
||||
what += " — held after a failed delivery until a person releases it (the operator is asked)"
|
||||
late = ""
|
||||
}
|
||||
return fmt.Sprintf("%s · %s, held: %s, for %s%s", name, where, what, since, late)
|
||||
}
|
||||
return fmt.Sprintf("%s · %s, %s for %s%s", name, where, what, since, late)
|
||||
}
|
||||
|
||||
@@ -2277,7 +2100,7 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
bounds.of(p.Repository), func(module string) []string {
|
||||
on, _ := inv.Running(ctx, module)
|
||||
return on
|
||||
}, waitsOf(ctx, inv, []inventory.Plan{p}))+heldSaid(p, heldNow(ctx, inv)))
|
||||
}, waitsOf(ctx, inv, []inventory.Plan{p})))
|
||||
if r := p.Release; r != nil {
|
||||
// A release plan's walk (ADR 0236): machines done, the one judged, those to come.
|
||||
fmt.Printf(" machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "),
|
||||
@@ -2472,12 +2295,11 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
return on
|
||||
}
|
||||
waits := waitsOf(ctx, inv, plans)
|
||||
held := heldNow(ctx, inv)
|
||||
for _, p := range plans {
|
||||
if p.Batch() {
|
||||
continue
|
||||
}
|
||||
fmt.Printf("%-28s %s%s\n", p.ID, planLineSaid(p, now, pause, bounds.of(p.Repository), running, waits), heldSaid(p, held))
|
||||
fmt.Printf("%-28s %s\n", p.ID, planLineSaid(p, now, pause, bounds.of(p.Repository), running, waits))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -2657,81 +2479,9 @@ func recordsOfAsked(ctx context.Context, inv *inventory.Inventory, p *inventory.
|
||||
}
|
||||
}
|
||||
}
|
||||
refused, err := refusalsOf(ctx, inv, recorded, byID)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
overlayRefusals(recorded, byID, refused)
|
||||
return recorded, byID, nil
|
||||
}
|
||||
|
||||
// overlayRefusals marks failed every build record whose registration the mesh refused (novox/hq issue 1708): the record
|
||||
// keeps the builder's words, which worked, and settled from it as it is the refused build would be built and sent.
|
||||
func overlayRefusals(recorded map[string][]inventory.Build, byID map[string]inventory.Build, refused map[string]string) {
|
||||
if len(refused) == 0 {
|
||||
return
|
||||
}
|
||||
fail := func(b *inventory.Build) {
|
||||
if why, ok := refused[b.ID]; ok && b.Worked() {
|
||||
b.Failed = why
|
||||
}
|
||||
}
|
||||
for m := range recorded {
|
||||
for i := range recorded[m] {
|
||||
fail(&recorded[m][i])
|
||||
}
|
||||
}
|
||||
for id, b := range byID {
|
||||
fail(&b)
|
||||
byID[id] = b
|
||||
}
|
||||
}
|
||||
|
||||
// refusalsOf is every refusal kept for the builds these records name, in the store and in this process.
|
||||
func refusalsOf(ctx context.Context, inv *inventory.Inventory, recorded map[string][]inventory.Build,
|
||||
byID map[string]inventory.Build) (map[string]string, error) {
|
||||
var ids []string
|
||||
for _, bs := range recorded {
|
||||
for _, b := range bs {
|
||||
ids = append(ids, b.ID)
|
||||
}
|
||||
}
|
||||
for id := range byID {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
refused, err := inv.RefusedOf(ctx, ids)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the refused registrations cannot be read: %w", err)
|
||||
}
|
||||
for id, why := range refusalsKeptFor(ids) {
|
||||
refused[id] = why
|
||||
}
|
||||
return refused, nil
|
||||
}
|
||||
|
||||
// settledEffects is what settling from the records did that needs more than the plan: the modules it found deleted
|
||||
// at their source, and whether it failed the plan.
|
||||
func settledEffects(before map[string]string, p *inventory.Plan, was string) (deleted []string, failed bool) {
|
||||
for m, s := range p.Modules {
|
||||
if s != nil && s.State == planDeleted && before[m] != planDeleted {
|
||||
deleted = append(deleted, m)
|
||||
}
|
||||
}
|
||||
sort.Strings(deleted)
|
||||
return deleted, p.State == inventory.PlanFailed && was != inventory.PlanFailed
|
||||
}
|
||||
|
||||
// statesOf is each module's state in a plan, for settledEffects.
|
||||
func statesOf(p *inventory.Plan) map[string]string {
|
||||
out := map[string]string{}
|
||||
for m, s := range p.Modules {
|
||||
if s != nil {
|
||||
out[m] = s.State
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// settleFromRecords marks every module of the tier still `asked` built — or failed — from a build
|
||||
// recorded after it was asked, and says whether it changed anything (novox/hq 04-ISSUES/214).
|
||||
// Newest first, as Builds answers: the first record after the ask is the outcome of that ask.
|
||||
@@ -2781,10 +2531,6 @@ func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]i
|
||||
s.State = "built"
|
||||
s.BuiltAt = &at
|
||||
s.Commit = outcome.Commit
|
||||
// What the build did, when its outcome was left to the records with words (issue 1708).
|
||||
if made := takeMadeOf(outcome.ID); made != "" {
|
||||
s.Made = made
|
||||
}
|
||||
} else {
|
||||
s.State = "failed"
|
||||
s.Why = outcome.Failed
|
||||
|
||||
@@ -46,11 +46,6 @@ func replay867(t *testing.T, recheck bool) {
|
||||
if _, err := exec.LookPath("go"); err != nil {
|
||||
t.Skip("no Go toolchain on this machine to build a judge with")
|
||||
}
|
||||
// The check's floor of free space (novox/hq issue 1834) reads the disk the test runs on, which is none of this
|
||||
// replay's business: a build node under the floor failed every mesh-controller check here (issue 1834, note 12).
|
||||
wasFree := builder.FreeBytes
|
||||
builder.FreeBytes = func(string) (uint64, error) { return 500 << 30, nil }
|
||||
t.Cleanup(func() { builder.FreeBytes = wasFree })
|
||||
r := theGroupOfIssue867(t)
|
||||
for _, c := range []struct {
|
||||
what, repo string
|
||||
|
||||
@@ -152,7 +152,7 @@ func answersRightly(q dnsmessage.Question) *dnsmessage.Message {
|
||||
func quickResolvers(t *testing.T, port string) {
|
||||
t.Helper()
|
||||
before, within, pause := resolverPort, resolverWithin, resolverPause
|
||||
resolverPort, resolverWithin, resolverPause = port, 60*time.Millisecond, 5*time.Millisecond
|
||||
resolverPort, resolverWithin, resolverPause = port, 150*time.Millisecond, 10*time.Millisecond
|
||||
t.Cleanup(func() { resolverPort, resolverWithin, resolverPause = before, within, pause })
|
||||
}
|
||||
|
||||
|
||||
@@ -93,27 +93,3 @@ func TestTheSeatsListingShowsEachSeatsPaths(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1718/AC2: a manifest claiming node-files that serves upload and download passes the controller's module
|
||||
// check, judged by the mesh-sdk it vendors; a verb outside the contract is still refused.
|
||||
func TestAFilesHolderServingUploadAndDownloadCanHoldTheSeat(t *testing.T) {
|
||||
seat, ok := catalogue.SeatNamed(catalogue.FilesSeat)
|
||||
if !ok {
|
||||
t.Fatal("node-files is not a seat")
|
||||
}
|
||||
serves := []string{"upload", "download"}
|
||||
for _, v := range seat.Serves {
|
||||
if v.Name != "upload" && v.Name != "download" {
|
||||
serves = append(serves, v.Name)
|
||||
}
|
||||
}
|
||||
holder := catalogue.Manifest{Module: "mesh-files", Version: "1", Claims: []catalogue.Claim{{
|
||||
Name: catalogue.FilesSeat, Scope: catalogue.ScopeNode, Serves: serves}}}
|
||||
if err := catalogue.CanHold(holder, seat); err != nil { // 1718/AC2
|
||||
t.Fatalf("a holder serving upload and download is refused: %v", err)
|
||||
}
|
||||
holder.Claims[0].Serves = append(append([]string{}, serves...), "teleport")
|
||||
if err := catalogue.CanHold(holder, seat); err == nil || !strings.Contains(err.Error(), "does not promise") {
|
||||
t.Fatalf("a verb outside the contract must be refused with 'does not promise': %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,10 +39,6 @@ type sendable struct {
|
||||
// ReadsModules says the machine's node-engine reads a module per resource and a build per module (novox/hq issue
|
||||
// 1145): only then are they sent, because an older node-engine refuses a key it does not know, whole.
|
||||
ReadsModules bool
|
||||
// fileReferences says the machine's node-engine reports the capability of reading a file given as a digest and a size
|
||||
// (declaration.FileReferencesCapability, novox/hq ADR 0486 §5): then every file over 64 KiB that may travel so does (Body,
|
||||
// objects). Not on the wire as a field; the bodies differ.
|
||||
fileReferences bool
|
||||
// moduleOf is the composer's owner of each resource id, and buildIDs the build each module carries (its id):
|
||||
// kept beside the resources, and on the wire only to a machine that reads them.
|
||||
moduleOf map[string]string
|
||||
@@ -119,16 +115,12 @@ type adoptionEnvelope struct {
|
||||
|
||||
// Body is the declaration's bytes, as sent and as digested.
|
||||
func (s sendable) Body() ([]byte, error) {
|
||||
resources, _, err := s.byReference()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
envelope := map[string]any{"declaration": 1, "resources": resources}
|
||||
envelope := map[string]any{"declaration": 1, "resources": s.Resources}
|
||||
if s.ReadsModules {
|
||||
// The module of each resource the composer owns (what the mesh adds of its own has none), and the build of each
|
||||
// module (novox/hq issue 1145): to a machine that said it reads them, and no other.
|
||||
named := make([]map[string]any, len(resources))
|
||||
for n, r := range resources {
|
||||
named := make([]map[string]any, len(s.Resources))
|
||||
for n, r := range s.Resources {
|
||||
module, owned := s.moduleOf[fmt.Sprint(r["id"])]
|
||||
if !owned {
|
||||
named[n] = r
|
||||
@@ -173,19 +165,3 @@ func (s sendable) Body() ([]byte, error) {
|
||||
}
|
||||
return json.Marshal(envelope)
|
||||
}
|
||||
|
||||
// byReference is the resources as the machine is sent them: with the contents of its large files replaced by their digests when
|
||||
// it reports it reads them (issue 1696), and the contents it replaced, by the name of the object that holds each.
|
||||
func (s sendable) byReference() ([]map[string]any, map[string][]byte, error) {
|
||||
if !s.fileReferences {
|
||||
return s.Resources, nil, nil
|
||||
}
|
||||
return referencing(s.Resources)
|
||||
}
|
||||
|
||||
// objects are the contents the body names by digest, by the name of each one's object: what is put in the bucket before the
|
||||
// body is sent. Empty for a machine that is sent every content inline.
|
||||
func (s sendable) objects() (map[string][]byte, error) {
|
||||
_, objects, err := s.byReference()
|
||||
return objects, err
|
||||
}
|
||||
|
||||
@@ -244,22 +244,6 @@ var signalsTable = []signalRow{
|
||||
newest: func(f *signalFacts) time.Time {
|
||||
return newestOf(f.failed, func(p failedPlanFacts) time.Time { return p.since })
|
||||
}},
|
||||
{Row: "S24", Signal: "a move of a recorded module that waits for a person's push is asked of the operator",
|
||||
Emitter: "controller's plan", Trigger: "each machine with a move of a recorded module held for a person (novox/hq ADR 0377, issue 1732)",
|
||||
Bound: "none: raised on the first look that finds the machine holding such a move, naming the modules and the " +
|
||||
"machine, with Release (the machine's push) as the answer; ended when the machine no longer waits, whoever " +
|
||||
"pushed it. A hold a person chose (a module set to record with a why, the bus) is not asked.",
|
||||
Kind: kindMovesHeld, Severity: conditions.Warning, Phase: 3,
|
||||
needs: func(f *signalFacts) error { return f.heldErr }, watch: watchMovesHeld,
|
||||
newest: func(f *signalFacts) time.Time { return time.Time{} }},
|
||||
{Row: "S25", Signal: "a module left out of a send because the artifact store lacks an image it names",
|
||||
Emitter: "controller's send", Trigger: "each send whose declaration names an image of the store that the store does not hold (novox/hq issue 1742)",
|
||||
Bound: "none: raised on the first look at a registered build that names such an image, naming the module, each " +
|
||||
"machine that runs it and the image; every send leaves that module out; the store is asked again at each " +
|
||||
"look and the condition ends when it holds the image",
|
||||
Kind: kindImageMissing, Severity: conditions.Warning, Phase: 3,
|
||||
needs: func(f *signalFacts) error { return f.imagesErr }, watch: watchMissingImages,
|
||||
newest: func(f *signalFacts) time.Time { return time.Time{} }},
|
||||
{Row: "S20", Signal: "a send refused for the assignment generation it was composed from",
|
||||
Emitter: "node-engine", Trigger: "each refusal (novox/hq issue 234)",
|
||||
Bound: "none: raised at the first refusal, naming its sender, the generation it came from and the one the " +
|
||||
|
||||
@@ -171,19 +171,6 @@ var suppressions = map[string]suppression{
|
||||
since: f.now.Add(-time.Second)}}
|
||||
},
|
||||
},
|
||||
// A machine holding a move of a recorded module for a person is asked at its first look (novox/hq issue 1732).
|
||||
// Inside: nothing waits there.
|
||||
"S24": {
|
||||
inside: func(f *signalFacts) { f.held = nil },
|
||||
past: func(f *signalFacts) { f.held = []heldFacts{aHeldMailu()} },
|
||||
},
|
||||
// A registered build naming an image the artifact store lacks is raised at the first look (novox/hq issue 1742).
|
||||
"S25": {
|
||||
inside: func(f *signalFacts) { f.images = nil },
|
||||
past: func(f *signalFacts) {
|
||||
f.images = []imageFacts{{missingImage{Node: "anchor", Module: "automx", Image: automxImage}}}
|
||||
},
|
||||
},
|
||||
// A batch queued behind an open walk past that walk's bound, named.
|
||||
"S19": {
|
||||
inside: func(f *signalFacts) {
|
||||
|
||||
@@ -415,59 +415,7 @@ func askNewModules(ctx context.Context, inv *inventory.Inventory, m link.SourceM
|
||||
if len(added) == 0 || len(from) == 0 {
|
||||
return nil
|
||||
}
|
||||
source := buildSource{Repository: from[0].Source.Repository, Seat: from[0].Source.Seat}
|
||||
// Asked once (novox/hq issue 1617): a module is asked when its merge is heard, and again here when the batch is cut.
|
||||
var requests []inventory.RequestOutcome
|
||||
for _, dir := range added {
|
||||
if got, err := inv.RequestsNamed(ctx, inventory.BuildRequest{Repository: source.Repository, Path: dir}.Name()); err == nil {
|
||||
requests = append(requests, got...)
|
||||
}
|
||||
}
|
||||
return askBuilds(ctx, inv, m, source, unaskedDirs(added, requests, source, time.Now()))
|
||||
}
|
||||
|
||||
// askedInFlightFor is how long a merge's build request stands as the build coming for its module: past it, a lost one
|
||||
// is asked again.
|
||||
const askedInFlightFor = time.Hour
|
||||
|
||||
// unaskedDirs are the directories of a new module no build of a merge is on its way for: a kept request of the same
|
||||
// repository and directory, asked for a merge and handed over, whose outcome has not come and is not given up on,
|
||||
// is the build coming (novox/hq issue 1617).
|
||||
func unaskedDirs(added []string, requests []inventory.RequestOutcome, source buildSource, now time.Time) []string {
|
||||
var out []string
|
||||
for _, dir := range added {
|
||||
path := dir
|
||||
if path == "." {
|
||||
path = ""
|
||||
}
|
||||
coming := false
|
||||
for _, r := range requests {
|
||||
if r.For == "merge" && r.Repository == source.Repository && strings.Trim(r.Path, "/") == path &&
|
||||
r.NotAsked == "" && r.OutcomeUnknown == "" && !r.Heard && now.Sub(r.At) < askedInFlightFor {
|
||||
coming = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !coming {
|
||||
out = append(out, dir)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// askNewModulesAtHearing asks the first build of the modules a merge adds to a repository the mesh builds modules from,
|
||||
// when the merge is heard, not when its batch is cut (novox/hq issue 1617): a batch waits out its window and the walk
|
||||
// before it, and a module added by a merge delivered meanwhile could not be assigned. Outside the batch and its plan.
|
||||
func askNewModulesAtHearing(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, entries []inventory.Entry,
|
||||
read map[string][]inventory.ReadRepository, from []inventory.Entry) {
|
||||
if len(from) == 0 {
|
||||
return
|
||||
}
|
||||
_, added, _ := touchedBy(from, entries, m, read)
|
||||
if asked := askNewModules(ctx, inv, m, from, added); len(asked) > 0 {
|
||||
fmt.Printf("%s/%s merged into %s (%.8s) adds %s: the first build is asked now, before its batch is cut\n",
|
||||
m.Owner, m.Repo, m.Base, m.Commit, strings.Join(asked, ", "))
|
||||
}
|
||||
return askBuilds(ctx, inv, m, buildSource{Repository: from[0].Source.Repository, Seat: from[0].Source.Seat}, added)
|
||||
}
|
||||
|
||||
// askBuilds asks the first build of each directory of a merge's repository, as askNewModules does, from the source given.
|
||||
|
||||
@@ -1,163 +0,0 @@
|
||||
package main
|
||||
|
||||
// The machines whose declaration a walk's merges change (novox/hq ADR 0505 §2): beside what the walk sends, a merge
|
||||
// can change what another machine is to run — an assignment, a provision or a requirement added or removed — and a
|
||||
// problem raised there as the walk runs, or as it settles, is the walk's as much as one on a machine it sends to. The
|
||||
// router holds what a walk touches (ADR 0308 §1); these are touches of their own, marked `declared`.
|
||||
//
|
||||
// **Compared as the mesh compares whether a machine is up to date.** Each machine's declaration is composed as a push
|
||||
// would compose it, with the number, epoch and generation it was last sent, and its digest held against the one it was
|
||||
// last sent (wouldSend, inventory.Waiting): byte for byte what it was sent when nothing changed, so a secret, a sequence
|
||||
// or a lease's epoch never makes a machine differ by itself. A machine whose declaration cannot be worked out now and
|
||||
// was sent one before is taken as changed: not known is changed, never "unchanged" (a merge that leaves a machine
|
||||
// unable to compose is the case this exists for, the 11:03 message of ADR 0505's context).
|
||||
//
|
||||
// **Said again, not waited for.** Composing every machine takes seconds; the walk's start and end are said at once as
|
||||
// before, and the serving controller, which hears every `plan-moved` — its own and a verb's — says a start or an end
|
||||
// again with the declared touches added, marked `declared_read`, which it never reads twice. The router merges a walk
|
||||
// said again (messenger walks.go).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// walkDeclaredRead marks a walk's start or end said again with its declared touches: read once.
|
||||
const walkDeclaredRead = "declared_read"
|
||||
|
||||
// declaredWithin bounds the comparison of every machine for one walk's start or end.
|
||||
const declaredWithin = time.Minute
|
||||
|
||||
// changedDeclarations is every machine whose declaration differs from what it was last sent, and every machine sent
|
||||
// one before whose declaration cannot be worked out now. A seam a test replaces.
|
||||
var changedDeclarations = func(ctx context.Context, open *stores) ([]string, error) {
|
||||
nodes, err := open.inventory.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx = withInputCache(ctx)
|
||||
would, err := wouldSend(ctx, open, nodes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
behind, err := open.inventory.Waiting(ctx, would)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sent := map[string]bool{}
|
||||
for _, n := range nodes {
|
||||
if s, known, err := open.inventory.SentTo(ctx, n.Name); err == nil && known && s.Digest != "" {
|
||||
sent[n.Name] = true
|
||||
}
|
||||
}
|
||||
return changedOf(nodes, would, behind, sent), nil
|
||||
}
|
||||
|
||||
// changedOf is changedDeclarations over what was read: the machines behind, and those sent one before that could not
|
||||
// be worked out. Pure; sorted.
|
||||
func changedOf(nodes []inventory.Node, would map[string]string, behind []inventory.Machine, sent map[string]bool) []string {
|
||||
set := map[string]bool{}
|
||||
for _, m := range behind {
|
||||
set[m.Node] = true
|
||||
}
|
||||
for _, n := range nodes {
|
||||
if _, worked := would[n.Name]; !worked && sent[n.Name] {
|
||||
set[n.Name] = true
|
||||
}
|
||||
}
|
||||
out := make([]string, 0, len(set))
|
||||
for n := range set {
|
||||
out = append(out, n)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// withDeclared is a walk's start or end said again with the machines whose declaration changed added as declared
|
||||
// touches: false when the body says no start or end, was said again already, or names no machine it does not touch.
|
||||
func withDeclared(body []byte, changed []string) ([]byte, bool, error) {
|
||||
said, wanted, err := toSayAgain(body)
|
||||
if err != nil || !wanted {
|
||||
return nil, false, err
|
||||
}
|
||||
var touches []walkTouch
|
||||
if raw := said["touches"]; raw != nil && string(raw) != "null" {
|
||||
if err := json.Unmarshal(raw, &touches); err != nil {
|
||||
return nil, false, fmt.Errorf("its touches are not a list of {machine, modules}: %w", err)
|
||||
}
|
||||
}
|
||||
added := declaredTouches(touches, changed)
|
||||
if len(added) == 0 {
|
||||
return nil, false, nil
|
||||
}
|
||||
touches = append(touches, added...)
|
||||
sort.Slice(touches, func(i, j int) bool { return touches[i].Machine < touches[j].Machine })
|
||||
if said["touches"], err = json.Marshal(touches); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
said[walkDeclaredRead] = json.RawMessage(`true`)
|
||||
out, err := json.Marshal(said)
|
||||
return out, err == nil, err
|
||||
}
|
||||
|
||||
// toSayAgain reads a plan-moved: whether it says a walk's start or end not said again yet.
|
||||
func toSayAgain(body []byte) (map[string]json.RawMessage, bool, error) {
|
||||
var said map[string]json.RawMessage
|
||||
if err := json.Unmarshal(body, &said); err != nil {
|
||||
return nil, false, fmt.Errorf("a plan-moved that is not an object: %w", err)
|
||||
}
|
||||
var phase string
|
||||
_ = json.Unmarshal(said["phase"], &phase)
|
||||
return said, (phase == inventory.WalkStarted || phase == inventory.WalkEnded) && said[walkDeclaredRead] == nil, nil
|
||||
}
|
||||
|
||||
// declaredTouches are the machines changed that a walk's touches do not name, each a declared touch. Pure; sorted.
|
||||
func declaredTouches(touches []walkTouch, changed []string) []walkTouch {
|
||||
named := map[string]bool{}
|
||||
for _, t := range touches {
|
||||
named[t.Machine] = true
|
||||
}
|
||||
var out []walkTouch
|
||||
for _, n := range changed {
|
||||
if !named[n] {
|
||||
out = append(out, walkTouch{Machine: n, Modules: []string{}, Declared: true})
|
||||
named[n] = true
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Machine < out[j].Machine })
|
||||
return out
|
||||
}
|
||||
|
||||
// sayDeclared says a walk's start or end again with its declared touches, when it has any. What cannot be read is
|
||||
// said on the controller's log: the walk is then held as before, never less.
|
||||
func sayDeclared(ctx context.Context, bus link.Bus, open *stores, body []byte) {
|
||||
if _, wanted, _ := toSayAgain(body); !wanted {
|
||||
return // no start or end, or said again already: nothing to compare
|
||||
}
|
||||
comparing, cancel := context.WithTimeout(ctx, declaredWithin)
|
||||
defer cancel()
|
||||
changed, err := changedDeclarations(comparing, open)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "plan-moved: which machines' declarations the walk changes cannot be read: %v\n", err)
|
||||
return
|
||||
}
|
||||
again, ok, err := withDeclared(body, changed)
|
||||
if err != nil || !ok {
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "plan-moved: the walk cannot be said again with its declared machines: %v\n", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
stating, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer stop()
|
||||
if err := bus.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeyPlanMoved, again); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "plan-moved: the walk could not be said again with its declared machines: %v\n", err)
|
||||
}
|
||||
}
|
||||
@@ -1,61 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0505 §2: the machines whose declaration a walk's merges change are said as declared touches.
|
||||
|
||||
// The machines behind what they were last sent, and those sent one before that cannot be worked out now, are changed;
|
||||
// one never sent and not worked out is not.
|
||||
func TestAMachineBehindOrNoLongerComposableIsChanged(t *testing.T) {
|
||||
nodes := []inventory.Node{{Name: "ace"}, {Name: "g14"}, {Name: "novox"}, {Name: "shanks"}, {Name: "new"}}
|
||||
would := map[string]string{"ace": "d1", "g14": "d2", "novox": "d3"}
|
||||
behind := []inventory.Machine{{Node: "ace"}}
|
||||
sent := map[string]bool{"ace": true, "g14": true, "novox": true, "shanks": true}
|
||||
if got := changedOf(nodes, would, behind, sent); !reflect.DeepEqual(got, []string{"ace", "shanks"}) {
|
||||
t.Fatalf("changed: %v, want ace (behind) and shanks (no longer composes)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A walk's start said by plan-moved (walk_said.go's body), said again with the changed machines it does not touch as
|
||||
// declared touches, once.
|
||||
func TestAWalksStartIsSaidAgainWithTheMachinesItsMergesChange(t *testing.T) {
|
||||
body, err := json.Marshal(planMovedBody{Plan: inventory.Plan{ID: "plan-1", State: inventory.PlanBuilding},
|
||||
walkSaid: &walkSaid{Walk: "plan-1", Phase: inventory.WalkStarted, BudgetClass: "leaf", BoundSeconds: 600,
|
||||
Touches: []walkTouch{{Machine: "novox", Modules: []string{"gitea"}}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again, ok, err := withDeclared(body, []string{"novox", "shanks"})
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("not said again: %v %v", ok, err)
|
||||
}
|
||||
var said struct {
|
||||
ID string `json:"id"`
|
||||
Phase string `json:"phase"`
|
||||
Touches []walkTouch `json:"touches"`
|
||||
DeclaredRead bool `json:"declared_read"`
|
||||
}
|
||||
if err := json.Unmarshal(again, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []walkTouch{{Machine: "novox", Modules: []string{"gitea"}}, {Machine: "shanks", Modules: []string{}, Declared: true}}
|
||||
if said.ID != "plan-1" || said.Phase != inventory.WalkStarted || !said.DeclaredRead || !reflect.DeepEqual(said.Touches, want) {
|
||||
t.Fatalf("said again: %+v", said)
|
||||
}
|
||||
if _, ok, _ := withDeclared(again, []string{"ace"}); ok {
|
||||
t.Fatalf("a start said again was said a third time")
|
||||
}
|
||||
if _, ok, _ := withDeclared(body, []string{"novox"}); ok {
|
||||
t.Fatalf("said again with nothing it does not touch")
|
||||
}
|
||||
moved, _ := json.Marshal(inventory.Plan{ID: "plan-2", State: inventory.PlanBuilding})
|
||||
if _, ok, _ := withDeclared(moved, []string{"shanks"}); ok {
|
||||
t.Fatalf("a move inside a walk was said as its start")
|
||||
}
|
||||
}
|
||||
@@ -59,9 +59,6 @@ type walkTouch struct {
|
||||
// Carried names those of Modules that are there only because a moved tool runner or node-engine carries
|
||||
// them: what restarts with it, not what the walk builds.
|
||||
Carried []string `json:"carried,omitempty"`
|
||||
// Declared marks a machine the walk sends nothing to whose declaration its merges change (novox/hq ADR 0505 §2):
|
||||
// said again by the serving controller once it has compared every machine (walk_declared.go).
|
||||
Declared bool `json:"declared,omitempty"`
|
||||
}
|
||||
|
||||
// planMovedBody is `plan-moved`'s body: the walk whole, as before, and on its start and end what it touches.
|
||||
|
||||
@@ -264,46 +264,3 @@ func TestAWalkStoppedByHandAndRetriedEndsFailedWhenItFailsAgain(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0505 §2: a walk's start is said again, over the real bus and against the real store, with a machine it
|
||||
// sends nothing to whose declaration differs from what it was last sent (here: assigned a module and never pushed),
|
||||
// marked declared.
|
||||
func TestAWalksStartIsSaidAgainWithAMachineWhoseDeclarationChanged(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
next := planMovedHeard(t)
|
||||
register(t, open, catalogue.Manifest{Module: "app", Version: "1"})
|
||||
if _, err := assign(ctx, open, "anchor", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
core, err := js.Conn().JetStream()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := json.Marshal(planMovedBody{Plan: inventory.Plan{ID: "plan-505", State: inventory.PlanBuilding},
|
||||
walkSaid: &walkSaid{Walk: "plan-505", Phase: inventory.WalkStarted, BudgetClass: inventory.ClassLeaf,
|
||||
BoundSeconds: 360, Touches: []walkTouch{}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sayDeclared(ctx, link.OverNATS{Conn: js.Conn(), JS: core}, open, body)
|
||||
again := next(inventory.WalkStarted)
|
||||
raw, _ := json.Marshal(again["touches"])
|
||||
var touches []walkTouch
|
||||
_ = json.Unmarshal(raw, &touches)
|
||||
anchor := false
|
||||
for _, tc := range touches {
|
||||
anchor = anchor || (tc.Machine == "anchor" && tc.Declared && len(tc.Modules) == 0)
|
||||
if !tc.Declared {
|
||||
t.Fatalf("a machine the walk sends nothing to was said sent: %s", raw)
|
||||
}
|
||||
}
|
||||
if again["walk"] != "plan-505" || again[walkDeclaredRead] != true || !anchor {
|
||||
t.Fatalf("said again: %v", again)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,12 +58,6 @@ type signalFacts struct {
|
||||
// failed are the delivery plans that failed and are asked of the operator (S23, novox/hq ADR 0449).
|
||||
failed []failedPlanFacts
|
||||
failedErr error
|
||||
// held are the machines with moves of recorded modules waiting for a person's push (S24, novox/hq issue 1732).
|
||||
held []heldFacts
|
||||
heldErr error
|
||||
// images are the modules left out of a send because the artifact store lacks an image they name (S25, issue 1742).
|
||||
images []imageFacts
|
||||
imagesErr error
|
||||
// refusedSends are the sends a machine refused within the hour for the generation they were composed
|
||||
// from, each naming its sender (S20, novox/hq issue 234).
|
||||
refusedSends []inventory.Send
|
||||
@@ -363,8 +357,6 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
|
||||
f.batches, f.plansErr = gatherBatches(ctx, inv, now)
|
||||
}
|
||||
f.failed, f.failedErr = gatherFailedPlans(ctx, inv, now)
|
||||
f.held, f.heldErr = gatherHeldMoves(ctx, inv)
|
||||
f.images, f.imagesErr = gatherMissingImagesHere(ctx, inv)
|
||||
f.loop, f.loopErr = w.gatherLoop()
|
||||
f.asks, f.asksErr = w.gatherAsks(ctx, inv)
|
||||
if open, err := w.keeper.Open(ctx); err != nil {
|
||||
|
||||
@@ -4,14 +4,14 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// What the mesh issued this proxy, read on the bus (novox/hq ADR 0160, ADR 0167).
|
||||
@@ -21,6 +21,16 @@ import (
|
||||
// network, which is who may be served an internal name. Read once at connect and followed, so a
|
||||
// route added or a machine joining reaches a running proxy without a restart.
|
||||
|
||||
// credential is the bus account the mesh delivered as this module's own secret named broker.
|
||||
type credential struct {
|
||||
URL string `json:"url"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
User string `json:"user"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
|
||||
// followMembership connects with the credential in path and applies every membership the mesh
|
||||
// issues this proxy. It retries the first connection for as long as it takes: a proxy that started
|
||||
// before the bus keeps serving the file, and takes the bus when it answers.
|
||||
@@ -36,23 +46,60 @@ func followMembership(path string, held *table, fromBus *atomic.Bool) {
|
||||
}
|
||||
|
||||
func followOnce(path string, held *table, fromBus *atomic.Bool) error {
|
||||
account, err := process.ReadAccount(path)
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
conn, err := process.Connect(account, nats.Name(account.Node+"."+account.Module))
|
||||
if err != nil {
|
||||
return fmt.Errorf("connecting to the bus at %s: %w", membership.BareAddress(account.URL), err)
|
||||
var cred credential
|
||||
if err := json.Unmarshal(raw, &cred); err != nil {
|
||||
return fmt.Errorf("the broker credential is not one: %w", err)
|
||||
}
|
||||
subject := membership.Subject(account.Node, account.Module)
|
||||
err = conn.FollowMembership(func(issued membership.Membership) {
|
||||
if cred.Node == "" || cred.Module == "" {
|
||||
return fmt.Errorf("the broker credential names no node or module, so it has no membership")
|
||||
}
|
||||
|
||||
opts := []nats.Option{
|
||||
nats.Name(cred.Node + "." + cred.Module),
|
||||
nats.UserInfo(cred.User, cred.Password),
|
||||
// Its own inbox, and nothing wider: every principal is granted `_INBOX.<its user>.>` alone.
|
||||
nats.CustomInboxPrefix("_INBOX." + cred.User),
|
||||
// The bus being restarted is an upgrade, not a reason to stop following.
|
||||
nats.MaxReconnects(-1),
|
||||
}
|
||||
if strings.TrimSpace(cred.Fingerprint) != "" {
|
||||
opts = append(opts, nats.Secure(broker.PinnedToFingerprint(cred.Fingerprint)))
|
||||
}
|
||||
conn, err := nats.Connect(cred.URL, opts...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("connecting to the bus at %s: %w", broker.BareAddress(cred.URL), err)
|
||||
}
|
||||
|
||||
subject := broker.MembershipSubject(cred.Node, cred.Module)
|
||||
apply := func(body []byte) {
|
||||
var issued broker.Membership
|
||||
if err := json.Unmarshal(body, &issued); err != nil {
|
||||
log.Printf("a membership arrived that is not one: %v", err)
|
||||
return
|
||||
}
|
||||
if took := applyMembership(issued, held); took && !fromBus.Swap(true) {
|
||||
log.Printf("routes now come from this proxy's membership on %s", subject)
|
||||
}
|
||||
}, func(err error) { log.Print(err) })
|
||||
if err != nil {
|
||||
}
|
||||
|
||||
// Followed first, read second: an issue landing between the two is applied, not missed.
|
||||
if _, err := conn.Subscribe(subject, func(m *nats.Msg) { apply(m.Data) }); err != nil {
|
||||
conn.Close()
|
||||
return err
|
||||
return fmt.Errorf("cannot follow %s: %w", subject, err)
|
||||
}
|
||||
// The subject-addressed direct get: the one request this account may make of the stream.
|
||||
got, err := conn.Request("$JS.API.DIRECT.GET."+broker.AssignmentsStream+"."+subject, nil, 5*time.Second)
|
||||
switch {
|
||||
case err != nil:
|
||||
log.Printf("cannot read the membership issued on %s yet (%v); following it", subject, err)
|
||||
case got.Header.Get("Status") != "" || len(got.Data) == 0:
|
||||
log.Printf("no membership issued on %s yet; serving the file until one is", subject)
|
||||
default:
|
||||
apply(got.Data)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -61,7 +108,7 @@ func followOnce(path string, held *table, fromBus *atomic.Bool) error {
|
||||
//
|
||||
// A membership with no routes in it is one from a controller older than ADR 0167, and the file stays
|
||||
// the source rather than every route being withdrawn because a field was absent.
|
||||
func applyMembership(issued membership.Membership, held *table) bool {
|
||||
func applyMembership(issued broker.Membership, held *table) bool {
|
||||
raw, carries := issued.Receives["route"]
|
||||
if !carries {
|
||||
return false
|
||||
+4
-4
@@ -12,7 +12,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// behind is a workload the proxy can send to, and a table routing one public name and one
|
||||
@@ -169,7 +169,7 @@ func TestTheMeshIsReadAsAddressesAndRanges(t *testing.T) {
|
||||
// machines it names (novox/hq ADR 0167).
|
||||
func TestAMembershipIsServedAsIssued(t *testing.T) {
|
||||
held := newTable()
|
||||
took := applyMembership(membership.Membership{
|
||||
took := applyMembership(broker.Membership{
|
||||
Receives: map[string]json.RawMessage{"route": json.RawMessage(`[
|
||||
{"from":"admin","node":"anchor","at":"anchor.internal",
|
||||
"values":{"internal-name":"admin.anchor.internal","port":8080}}]`)},
|
||||
@@ -191,13 +191,13 @@ func TestAMembershipIsServedAsIssued(t *testing.T) {
|
||||
func TestAMembershipWithoutRoutesLeavesTheFileServing(t *testing.T) {
|
||||
held := behind(t, "10.10.0.7")
|
||||
before := held.names()
|
||||
if applyMembership(membership.Membership{Mesh: []string{"10.10.0.7"}}, held) {
|
||||
if applyMembership(broker.Membership{Mesh: []string{"10.10.0.7"}}, held) {
|
||||
t.Error("a membership without routes was taken as the source of routes")
|
||||
}
|
||||
if got := held.names(); strings.Join(got, ",") != strings.Join(before, ",") {
|
||||
t.Errorf("a membership without routes changed what is served: %v, was %v", got, before)
|
||||
}
|
||||
if applyMembership(membership.Membership{
|
||||
if applyMembership(broker.Membership{
|
||||
Receives: map[string]json.RawMessage{"route": json.RawMessage(`[]`)},
|
||||
Mesh: []string{"not-an-address"},
|
||||
}, held) {
|
||||
@@ -3,7 +3,7 @@ module github.com/novox/mesh-controller
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161324-8892a1176800
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021105408-c8e35097e861
|
||||
github.com/jackc/pgx/v5 v5.10.0
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5 h1:E5stfIeCN6uYgA9TCv+4K8JgaURg3zQppvUgJLaDMP4=
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5/go.mod h1:4wyoCr6IIIIarEUuROThWDj7DAvJvIidqMjWcpSnOwE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161324-8892a1176800 h1:Q29osS06HHcXH1WXURDyITbOLnJG7XJYgCzmZD++Nws=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161324-8892a1176800/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021105408-c8e35097e861 h1:BsqolZMMsC5jLP8X88BwShqFssLgtREcgBIy6OfPSI8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021105408-c8e35097e861/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
|
||||
@@ -46,7 +46,6 @@ func CheckStreams() []Stream {
|
||||
Retention: RetentionWorkQueue,
|
||||
MaxAge: 7 * 24 * 60 * 60,
|
||||
MaxMsgsPerSubject: 1,
|
||||
Direct: true,
|
||||
Why: "the checks of pull requests marked ready, taken after a delivery build and before any other check " +
|
||||
"(novox/hq ADR 0406 §3); one subject a pull request, the newest request kept, so a newer one replaces a " +
|
||||
"request still waiting",
|
||||
@@ -57,7 +56,6 @@ func CheckStreams() []Stream {
|
||||
Retention: RetentionWorkQueue,
|
||||
MaxAge: 7 * 24 * 60 * 60,
|
||||
MaxMsgsPerSubject: 1,
|
||||
Direct: true,
|
||||
Why: "every other check, taken last and in the order asked (novox/hq ADR 0406 §3); one subject a pull " +
|
||||
"request, the newest request kept; one left behind for a head already judged is dropped when taken",
|
||||
},
|
||||
@@ -100,13 +98,6 @@ func holderCheckGrants() []string {
|
||||
"$KV."+CheckRunningBucket+".>",
|
||||
"$JS.API.STREAM.INFO.KV_"+CheckRunningBucket,
|
||||
"$JS.API.DIRECT.GET.KV_"+CheckRunningBucket+".>")
|
||||
// And, to hold checks while a delivery builds (ADR 0483, issues 1687 and 1688): the bucket of the plans building, **read and
|
||||
// never written** (the controller's alone, so no `$KV.` subject of it is granted), and the oldest waiting ask of each check
|
||||
// queue, read by a direct get that takes nothing from the queue and so counts against no delivery of the ask.
|
||||
pub = append(pub, readGrants(PlansBuildingBucket, "")...)
|
||||
for _, w := range CheckWorkers() {
|
||||
pub = append(pub, "$JS.API.DIRECT.GET."+w.Stream, "$JS.API.DIRECT.GET."+w.Stream+".>")
|
||||
}
|
||||
return pub
|
||||
}
|
||||
|
||||
|
||||
@@ -40,37 +40,3 @@ func TestTheBuildAgentReadsAndWatchesThePullRequestsKeys(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1687 (ADR 0483 clause 2): a holder of the build seat reads the bucket of the plans building and writes none of it, and
|
||||
// reads the oldest waiting ask of each check queue by a direct get, which takes nothing from the queue.
|
||||
func TestTheBuildSeatsHoldersReadThePlansBuildingAndWriteNone(t *testing.T) {
|
||||
seat := Seat{Name: TheBuildSeat, Scope: "node", Accepts: []string{"build"}, Emits: []string{"built", "started"}}
|
||||
p := Principal{Kind: KindModule, Node: "one", Module: "build-agent", Holds: []Seat{seat}, PasswordHash: "x"}
|
||||
perms, err := PermissionsFor(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stream := "KV_" + PlansBuildingBucket
|
||||
for _, s := range []string{
|
||||
"$JS.API.STREAM.INFO." + stream,
|
||||
"$JS.API.DIRECT.GET." + stream + ".>",
|
||||
"$JS.API.CONSUMER.CREATE." + stream + ".>",
|
||||
"$JS.API.DIRECT.GET.CHECKS_READY",
|
||||
"$JS.API.DIRECT.GET.CHECKS_OTHER",
|
||||
} {
|
||||
has(t, perms.Publish, s)
|
||||
}
|
||||
for _, s := range perms.Publish {
|
||||
if SubjectsOverlap(s, "$KV."+PlansBuildingBucket+".plan-1") {
|
||||
t.Errorf("a holder of the build seat may write %s through %q", PlansBuildingBucket, s)
|
||||
}
|
||||
}
|
||||
// And the controller alone writes it, as the writers table says.
|
||||
for _, row := range WritersTable {
|
||||
for _, s := range row.Subjects {
|
||||
if s == "$KV."+PlansBuildingBucket+".>" && row.Writes(p, "$KV."+PlansBuildingBucket+".plan-1") {
|
||||
t.Errorf("%q lets the build seat's holder write the plans building", row.State)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -63,26 +63,6 @@ var (
|
||||
// repository: the controller reads the ask's warrant from the router and accepts only the operator's Approve of
|
||||
// those heads. Written by the controller alone, from the announcement, when it holds the lease.
|
||||
DeliveryMergesBucket = BucketName(ControllerSeat, "delivery-merges")
|
||||
// PlansBuildingBucket holds one key per delivery plan in its build phase (novox/hq ADR 0483 clause 2, issue 1687): the key
|
||||
// is `<plan id>` (`plan-<digits>`), its value the plan, the time its build phase began and the time of the last refresh.
|
||||
// The controller (lease holder) puts it when the plan's first build is asked, puts it again about every thirty seconds
|
||||
// while the plan builds, and deletes it when the phase ends (every build of the last tier built, or the plan fails, stops
|
||||
// or waits on the operator); the bucket's age, PlansBuildingKeptFor, is how long a key outlives a controller that
|
||||
// stopped. **Read by the build seat's holders and the checker, written by the controller alone**: a holder takes no new
|
||||
// check while any key is there, but for one that waited past ADR 0483's bound. A reader lists the keys and reads none of
|
||||
// their values to decide that (`link.BuildingPlans`).
|
||||
PlansBuildingBucket = BucketName(ControllerSeat, PlansBuilding)
|
||||
)
|
||||
|
||||
// PlansBuilding is the local name of the bucket of the plans building, as a reader names it in `reads`
|
||||
// (`mesh-controller.plans-building`).
|
||||
const PlansBuilding = "plans-building"
|
||||
|
||||
// PlansBuildingKeptFor is the age of a plan-building key: the controller puts it again at PlansBuildingRefresh, so a
|
||||
// controller that stopped leaves checks held at most this long (ADR 0483 clause 2).
|
||||
const (
|
||||
PlansBuildingKeptFor = 2 * time.Minute
|
||||
PlansBuildingRefresh = 30 * time.Second
|
||||
)
|
||||
|
||||
// DeliveryMergesKeptFor is how long a merge mesh-delivery announced is kept: a week, as the events are.
|
||||
@@ -104,9 +84,6 @@ var (
|
||||
AssignmentsBucket = BucketName(ControllerSeat, MirrorAssignments)
|
||||
DeclaredConfigBucket = BucketName(ControllerSeat, MirrorDeclaredConfig)
|
||||
PredefinedBucket = BucketName(ControllerSeat, MirrorPredefined)
|
||||
// DeclaredPackagesBucket holds, keyed by the node, each package the modules assigned to it declare, present or absent,
|
||||
// with the declaring modules: what the pacman module's verbs read to say a package is declared (ADR 0507 §5).
|
||||
DeclaredPackagesBucket = BucketName(ControllerSeat, MirrorDeclaredPackages)
|
||||
)
|
||||
|
||||
// The mirrors' local names, as a reader names them in `reads` after `mesh-controller.`.
|
||||
@@ -114,8 +91,6 @@ const (
|
||||
MirrorAssignments = "assignments"
|
||||
MirrorDeclaredConfig = "declared-config"
|
||||
MirrorPredefined = "predefined"
|
||||
// MirrorDeclaredPackages is the local name of the packages each node's modules declare (ADR 0507 §5, issue 1840/S3).
|
||||
MirrorDeclaredPackages = "declared-packages"
|
||||
)
|
||||
|
||||
// The chain's buckets (novox/hq ADR 0408, issue 1089): builds, rollouts and assignments are one chain kept on the bus
|
||||
@@ -158,7 +133,7 @@ func ChainBuckets() []string { return []string{BuildsBucket, RolloutsBucket, Ass
|
||||
// ControllerMirrors are the local names of the controller's buckets a module may read: the record mirrors and the
|
||||
// chain's buckets.
|
||||
func ControllerMirrors() []string {
|
||||
return []string{MirrorAssignments, MirrorDeclaredConfig, MirrorPredefined, ChainBuilds, ChainRollouts, ChecksRunning, PlansBuilding, MirrorDeclaredPackages}
|
||||
return []string{MirrorAssignments, MirrorDeclaredConfig, MirrorPredefined, ChainBuilds, ChainRollouts, ChecksRunning}
|
||||
}
|
||||
|
||||
// MirrorsWrittenByAnother are the controller's mirrors whose writer is not the controller, each with its reason: the rule
|
||||
@@ -227,17 +202,16 @@ func IsControllerBucket(bucket string) bool {
|
||||
return bucket == CallsBucket || bucket == HandActsBucket || bucket == ConditionsBucket ||
|
||||
bucket == ConditionHistoryBucket || bucket == LeaseBucket || bucket == AskedBucket ||
|
||||
bucket == AssignmentsBucket || bucket == DeclaredConfigBucket || bucket == PredefinedBucket ||
|
||||
bucket == DeclaredPackagesBucket ||
|
||||
bucket == MergeWindowBucket || bucket == RolloutSilentBucket || bucket == CheckRunningBucket ||
|
||||
bucket == DeliveryMergesBucket || bucket == BuildsBucket || bucket == RolloutsBucket ||
|
||||
bucket == PlansBuildingBucket || bucket == delivery.HaveBucket
|
||||
bucket == delivery.HaveBucket
|
||||
}
|
||||
|
||||
// ControllerBuckets are the controller's own buckets, in the order they are asserted.
|
||||
func ControllerBuckets() []string {
|
||||
return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket, AskedBucket,
|
||||
AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket, DeclaredPackagesBucket, MergeWindowBucket, RolloutSilentBucket, CheckRunningBucket,
|
||||
DeliveryMergesBucket, PlansBuildingBucket, BuildsBucket, RolloutsBucket, delivery.HaveBucket}
|
||||
AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket, MergeWindowBucket, RolloutSilentBucket, CheckRunningBucket,
|
||||
DeliveryMergesBucket, BuildsBucket, RolloutsBucket, delivery.HaveBucket}
|
||||
}
|
||||
|
||||
// ControllerBucketsAsserter is what raising the controller's buckets needs of a connection.
|
||||
@@ -380,22 +354,6 @@ func (j *JetStream) EnsureControllerBuckets() error {
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", CheckRunningBucket, err)
|
||||
}
|
||||
// **The plans building: keys that age** (ADR 0483 clause 2, issue 1687). One message a key, each refresh a new one, so the
|
||||
// age of the bucket is how long a key outlives a controller that stopped refreshing it.
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: PlansBuildingBucket,
|
||||
Description: "the delivery plans in their build phase (novox/hq ADR 0483): `<plan id>` holds the plan and when its " +
|
||||
"build phase began; put when the plan's first build is asked and about every thirty seconds, deleted when the " +
|
||||
"phase ends, gone within two minutes of a controller that stopped; read by the build seat's holders, which " +
|
||||
"take no new check while a key is there; written by the controller alone",
|
||||
History: 1,
|
||||
TTL: PlansBuildingKeptFor,
|
||||
MaxValueSize: 4 << 10,
|
||||
MaxBytes: 8 << 20,
|
||||
Storage: jetstream.FileStorage,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", PlansBuildingBucket, err)
|
||||
}
|
||||
// **The merges mesh-delivery announced: keys that age** (ADR 0420 §7, issue 1025). One message a key; a week.
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: DeliveryMergesBucket,
|
||||
@@ -452,9 +410,6 @@ func (j *JetStream) EnsureControllerBuckets() error {
|
||||
{PredefinedBucket, "the list of predefined keys the mesh published, the newest trunk build's of the " +
|
||||
"catalogue, one key `list` (novox/hq ADR 0348 §7, design 57 K3): written by the controller alone from " +
|
||||
"its store when a newer list is kept, read through `reads`"},
|
||||
{DeclaredPackagesBucket, "each package the modules assigned to a node declare, present or absent, with the " +
|
||||
"declaring modules, keyed by the node (novox/hq ADR 0507 §5): written by the controller alone when it " +
|
||||
"composes the node, only on a change, read through `reads`"},
|
||||
} {
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: m.bucket,
|
||||
|
||||
@@ -1,123 +0,0 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// The object buckets a declaration's large files travel in (novox/hq ADR 0486, issues 1696 and 1697): **one bucket for each
|
||||
// node**, `declaration-files-<node>`.
|
||||
//
|
||||
// **The controller alone writes them, and a node-engine reads an object by its name from its own bucket** (ADR 0486 §2): its
|
||||
// account may neither list the bucket nor write it, no account may touch another node's, and no other account has any access.
|
||||
// Why a bucket each: a subject pattern cannot tell a name from a wildcard. A grant of `...M.*` matches a publish to the literal
|
||||
// subject `...M.*` too (probed on nats-server 2.11.17 by the review of this issue: a node created a consumer filtered on
|
||||
// `$O.<bucket>.C.*` and read every object of the bucket), so in one shared bucket a node could read every other node's files.
|
||||
// With a bucket each, the same trick reads only the node's own files, which are the files of its own declaration.
|
||||
//
|
||||
// A bucket of the bus's object store is a stream, OBJ_<bucket>, whose messages are the objects' metadata
|
||||
// (`$O.<bucket>.M.<base64 of the name>`) and chunks (`$O.<bucket>.C.<nuid>`).
|
||||
|
||||
// DeclarationFilesBucket is the prefix of the buckets' names, which mesh-sdk fixes for the node-engine and the controller alike.
|
||||
const DeclarationFilesBucket = declaration.FileBucketPrefix
|
||||
|
||||
// DeclarationFilesBucketOf is the bucket a node's files are in: mesh-sdk's FileBucketFor, the one place the name is composed. A
|
||||
// node name it refuses (one that could carry a subject wildcard or separator into a grant) is an error, never a bucket.
|
||||
func DeclarationFilesBucketOf(node string) (string, error) { return declaration.FileBucketFor(node) }
|
||||
|
||||
// DeclarationFilesStreamOf is the stream under a node's bucket: mesh-sdk's FileStreamFor.
|
||||
func DeclarationFilesStreamOf(node string) (string, error) { return declaration.FileStreamFor(node) }
|
||||
|
||||
// DeclarationFilesMaxBytes is a bucket's size: the objects are immutable and regenerable, and one node's hold well under a hundred
|
||||
// megabytes.
|
||||
const DeclarationFilesMaxBytes = 1 << 30
|
||||
|
||||
// DeclarationFilesKeptFor is how long an object stays after no node's latest declaration names it (ADR 0486 §3).
|
||||
const DeclarationFilesKeptFor = 24 * time.Hour
|
||||
|
||||
// DeclarationFilesWrites is the one pattern the controller may publish to write a node's bucket: every object's metadata and
|
||||
// chunks. The server's API for the stream is the controller's `$JS.API.>`.
|
||||
func DeclarationFilesWrites(node string) (string, error) {
|
||||
bucket, err := DeclarationFilesBucketOf(node)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "$O." + bucket + ".>", nil
|
||||
}
|
||||
|
||||
// DeclarationFilesReads is what a node-engine's account may publish to read an object of its own bucket by its name, and nothing
|
||||
// more. The set is the one node-engine's own call (mesh-host internal/link FileObject) makes, for B the node's bucket and S its
|
||||
// stream (issue 1697, notes 4 and 5):
|
||||
//
|
||||
// - the object's metadata, by a direct get of the last message of the one subject that names it. **Only the suffixed
|
||||
// form**: the bare `$JS.API.DIRECT.GET.S` takes a sequence number and would read any message of the stream.
|
||||
// - the ordered consumer on that object's chunks: `$JS.API.CONSUMER.CREATE.S.<name>.$O.B.C.<nuid>`. **Not `S.>`**:
|
||||
// a consumer filtered on `$O.B.M.>` would read every object's name.
|
||||
// - the flow-control replies of that consumer, without which a large object stalls at the flow-control window.
|
||||
// - the consumer's deletion, best effort: the consumer's inactivity threshold removes it if this is refused.
|
||||
//
|
||||
// **Never granted:** STREAM.INFO (a `subjects_filter` of `>` lists every name), STREAM.MSG.GET (any message by its sequence), a
|
||||
// publish to `$O.B.>` (a write), and any subject of another node's stream. A literal `*` in a subject matches the grant's `*`: it
|
||||
// reaches the objects of this node's own bucket and no other (above).
|
||||
func DeclarationFilesReads(node string) ([]string, error) {
|
||||
bucket, err := DeclarationFilesBucketOf(node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stream, err := DeclarationFilesStreamOf(node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []string{
|
||||
"$JS.API.DIRECT.GET." + stream + ".$O." + bucket + ".M.*",
|
||||
"$JS.API.CONSUMER.CREATE." + stream + ".*.$O." + bucket + ".C.*",
|
||||
"$JS.API.CONSUMER.DELETE." + stream + ".*",
|
||||
"$JS.FC." + stream + ".>",
|
||||
}, nil
|
||||
}
|
||||
|
||||
// EnsureDeclarationFiles creates the buckets of these nodes if absent and brings their options to match. A bucket exists whenever the
|
||||
// bus does, so a node that is starting finds it (ADR 0486 §4). An update, never a delete.
|
||||
func (j *JetStream) EnsureDeclarationFiles(nodes []string) error {
|
||||
js, err := jetstream.New(j.conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
return EnsureDeclarationFiles(ctx, js, nodes...)
|
||||
}
|
||||
|
||||
// EnsureDeclarationFiles is the same on a JetStream the caller holds.
|
||||
func EnsureDeclarationFiles(ctx context.Context, js jetstream.JetStream, nodes ...string) error {
|
||||
for _, node := range nodes {
|
||||
cfg, err := DeclarationFilesConfig(node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := js.CreateOrUpdateObjectStore(ctx, cfg); err != nil {
|
||||
return fmt.Errorf("asserting the object bucket %s: %w", cfg.Bucket, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeclarationFilesConfig is a node's bucket as the controller asserts it.
|
||||
func DeclarationFilesConfig(node string) (jetstream.ObjectStoreConfig, error) {
|
||||
bucket, err := DeclarationFilesBucketOf(node)
|
||||
if err != nil {
|
||||
return jetstream.ObjectStoreConfig{}, err
|
||||
}
|
||||
return jetstream.ObjectStoreConfig{
|
||||
Bucket: bucket,
|
||||
Description: "the contents of the files over 64 KiB of the declaration of " + node + ", each under the hex of its sha256 " +
|
||||
"digest (novox/hq ADR 0486): written by the controller alone, read by the node-engine of that node by name, removed " +
|
||||
"a day after the node's latest declaration no longer names it",
|
||||
MaxBytes: DeclarationFilesMaxBytes,
|
||||
Storage: jetstream.FileStorage,
|
||||
}, nil
|
||||
}
|
||||
@@ -1,316 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package broker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
)
|
||||
|
||||
// nodeEnginesFileObject is the node-engine's own call (mesh-host internal/link files.go, OverNATS.FileObject), as the
|
||||
// engine makes it: the object's metadata by a direct get of one subject, then its chunks through an ordered consumer
|
||||
// bound to the stream. It is copied here so the grants are judged by the call that has to work, and its subjects are the
|
||||
// ones the grant names (issue 1697, notes 4 and 5); the node-engine's own integration test holds the other copy.
|
||||
func nodeEnginesFileObject(ctx context.Context, nc *nats.Conn, node, name string) ([]byte, error) {
|
||||
js, err := nc.JetStream(nats.MaxWait(10 * time.Second))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stream, err := DeclarationFilesStreamOf(node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bucket, err := DeclarationFilesBucketOf(node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
meta, err := js.GetLastMsg(stream, "$O."+bucket+".M."+base64.URLEncoding.EncodeToString([]byte(name)),
|
||||
nats.DirectGet(), nats.Context(ctx))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var info struct {
|
||||
NUID string `json:"nuid"`
|
||||
Size uint64 `json:"size"`
|
||||
}
|
||||
if err := json.Unmarshal(meta.Data, &info); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sub, err := js.SubscribeSync("$O."+bucket+".C."+info.NUID, nats.OrderedConsumer(), nats.BindStream(stream))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer sub.Unsubscribe()
|
||||
body := make([]byte, 0, info.Size)
|
||||
for uint64(len(body)) < info.Size {
|
||||
msg, err := sub.NextMsgWithContext(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body = append(body, msg.Data...)
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
// **The buckets' grants on a real server, judged by the node-engine's own call** (novox/hq ADR 0486 §2, issue 1697, 1697/AC2): the
|
||||
// controller's account writes an object into each node's bucket, a node-engine's account reads one of its own bucket by its name,
|
||||
// a large object included, and is refused a list of the bucket (a stream info asking for every subject), a message get by
|
||||
// sequence, a direct get by sequence, a consumer on the names, and a write; it is refused every one of those and the read by name on
|
||||
// another node's bucket, **the literal-`*` forms included** (a published subject `...M.*` or `...C.*` matches the grant's `*`: on
|
||||
// its own bucket that reaches its own files, on another's it is refused); another module's account is refused all of them.
|
||||
func TestNatsANodeEngineReadsAnObjectOfItsOwnBucketByNameAndNothingElse(t *testing.T) { // 1697/AC2
|
||||
hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
users, err := Users(Records{Nodes: []string{"alpha", "beta"}, Assigned: map[string][]Declared{
|
||||
"alpha": {{Module: RuntimeModule}, {Module: "other-module"}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := range users {
|
||||
users[i].PasswordHash = string(hash)
|
||||
}
|
||||
accounts, err := ComposeAccounts(users)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conf := filepath.Join(t.TempDir(), "bus.conf")
|
||||
if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+
|
||||
`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts, err := server.ProcessConfigFile(conf)
|
||||
if err != nil {
|
||||
t.Fatalf("the composed accounts do not parse: %v", err)
|
||||
}
|
||||
opts.NoLog, opts.NoSigs = true, true
|
||||
s, err := server.NewServer(opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go s.Start()
|
||||
if !s.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(s.Shutdown)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
defer cancel()
|
||||
connect := func(user string) (*nats.Conn, chan error) {
|
||||
t.Helper()
|
||||
refused := make(chan error, 16)
|
||||
nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, "pw"), nats.CustomInboxPrefix("_INBOX."+user),
|
||||
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refused <- err }))
|
||||
if err != nil {
|
||||
t.Fatalf("%s cannot connect: %v", user, err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
return nc, refused
|
||||
}
|
||||
|
||||
// The controller makes the buckets (as it does at its start) and writes into each: for alpha a small object and one of
|
||||
// several chunks, which needs the consumer's flow control; for beta one that holds a marker alpha must never read.
|
||||
controller, _ := connect(ControllerName)
|
||||
cjs, err := jetstream.New(controller)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := EnsureDeclarationFiles(ctx, cjs, "alpha", "beta"); err != nil {
|
||||
t.Fatalf("the controller cannot make the buckets: %v", err)
|
||||
}
|
||||
if err := EnsureDeclarationFiles(ctx, cjs, "alpha", "beta"); err != nil {
|
||||
t.Fatalf("asserting them again: %v", err)
|
||||
}
|
||||
put := func(node string, content []byte) string {
|
||||
t.Helper()
|
||||
store, err := cjs.ObjectStore(ctx, mustBucket(t, node))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sum := sha256.Sum256(content)
|
||||
name := hex.EncodeToString(sum[:])
|
||||
if _, err := store.PutBytes(ctx, name, content); err != nil {
|
||||
t.Fatalf("the controller cannot write the object %s of %s: %v", name[:8], node, err)
|
||||
}
|
||||
return name
|
||||
}
|
||||
small := bytes.Repeat([]byte("small file\n"), 100)
|
||||
large := make([]byte, 900<<10)
|
||||
for i := range large {
|
||||
large[i] = byte(i*7 + i>>8)
|
||||
}
|
||||
secret := bytes.Repeat([]byte("BETA-SECRET "), 50)
|
||||
smallName, largeName, betaName := put("alpha", small), put("alpha", large), put("beta", secret)
|
||||
|
||||
alpha, alphaRefused := connect("node.alpha")
|
||||
for name, want := range map[string][]byte{smallName: small, largeName: large} {
|
||||
got, err := nodeEnginesFileObject(ctx, alpha, "alpha", name)
|
||||
if err != nil {
|
||||
t.Fatalf("a node-engine cannot read the object %s of its own bucket by its name: %v", name[:8], err)
|
||||
}
|
||||
if !bytes.Equal(got, want) {
|
||||
t.Fatalf("the node-engine read %d bytes of the object %s, want %d", len(got), name[:8], len(want))
|
||||
}
|
||||
}
|
||||
beta, _ := connect("node.beta")
|
||||
if got, err := nodeEnginesFileObject(ctx, beta, "beta", betaName); err != nil || !bytes.Equal(got, secret) {
|
||||
t.Fatalf("beta's node-engine reads its own object: %v", err)
|
||||
}
|
||||
select {
|
||||
case err := <-alphaRefused:
|
||||
t.Fatalf("a node-engine's own read was refused something: %v", err)
|
||||
default:
|
||||
}
|
||||
|
||||
// What each account is refused, judged by the server.
|
||||
alphaStream, alphaBucket := mustStream(t, "alpha"), mustBucket(t, "alpha")
|
||||
betaStream, betaBucket := mustStream(t, "beta"), mustBucket(t, "beta")
|
||||
listing := []byte(`{"subjects_filter":">"}`)
|
||||
bySeq := []byte(`{"seq":1}`)
|
||||
type refusal struct {
|
||||
what string
|
||||
do func(nc *nats.Conn)
|
||||
}
|
||||
onBucket := func(stream, bucket, which string) []refusal {
|
||||
return []refusal{
|
||||
{which + ": a list of the bucket (stream info asking for every subject)", func(nc *nats.Conn) {
|
||||
_, _ = nc.Request("$JS.API.STREAM.INFO."+stream, listing, 300*time.Millisecond)
|
||||
}},
|
||||
{which + ": a message get by sequence", func(nc *nats.Conn) {
|
||||
_, _ = nc.Request("$JS.API.STREAM.MSG.GET."+stream, bySeq, 300*time.Millisecond)
|
||||
}},
|
||||
{which + ": a direct get by sequence", func(nc *nats.Conn) {
|
||||
_, _ = nc.Request("$JS.API.DIRECT.GET."+stream, bySeq, 300*time.Millisecond)
|
||||
}},
|
||||
{which + ": a consumer on every object's name", func(nc *nats.Conn) {
|
||||
js, _ := nc.JetStream(nats.MaxWait(500 * time.Millisecond))
|
||||
if sub, err := js.SubscribeSync("$O."+bucket+".M.>", nats.OrderedConsumer(), nats.BindStream(stream)); err == nil {
|
||||
sub.Unsubscribe()
|
||||
}
|
||||
}},
|
||||
{which + ": a write of an object's metadata", func(nc *nats.Conn) {
|
||||
_ = nc.Publish("$O."+bucket+".M.abc", []byte(`{}`))
|
||||
_ = nc.Flush()
|
||||
}},
|
||||
{which + ": a write of a chunk", func(nc *nats.Conn) {
|
||||
_ = nc.Publish("$O."+bucket+".C.abc", []byte("x"))
|
||||
_ = nc.Flush()
|
||||
}},
|
||||
}
|
||||
}
|
||||
refusedWith := func(who string, nc *nats.Conn, ch chan error, r refusal) {
|
||||
t.Helper()
|
||||
for len(ch) > 0 {
|
||||
<-ch
|
||||
}
|
||||
r.do(nc)
|
||||
_ = nc.Flush()
|
||||
deadline := time.After(3 * time.Second)
|
||||
for {
|
||||
select {
|
||||
case err := <-ch:
|
||||
if errors.Is(err, nats.ErrPermissionViolation) {
|
||||
return
|
||||
}
|
||||
case <-deadline:
|
||||
t.Errorf("the server did not refuse %s %s", who, r.what)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
// Alpha's account on its own bucket (list, sequence reads, names, writes) and on beta's (all of that, and the read by name,
|
||||
// and the literal-`*` probes of the review).
|
||||
reachBeta := []refusal{
|
||||
{"beta's bucket: a read of one object by its name", func(nc *nats.Conn) {
|
||||
_, _ = nc.Request("$JS.API.DIRECT.GET."+betaStream+".$O."+betaBucket+".M."+
|
||||
base64.URLEncoding.EncodeToString([]byte(betaName)), nil, 300*time.Millisecond)
|
||||
}},
|
||||
{"beta's bucket: a direct get with a literal `*` for the name", func(nc *nats.Conn) {
|
||||
_, _ = nc.Request("$JS.API.DIRECT.GET."+betaStream+".$O."+betaBucket+".M.*", nil, 300*time.Millisecond)
|
||||
}},
|
||||
{"beta's bucket: a consumer with a literal `*` for the chunks", func(nc *nats.Conn) {
|
||||
js, _ := nc.JetStream(nats.MaxWait(500 * time.Millisecond))
|
||||
if sub, err := js.SubscribeSync("$O."+betaBucket+".C.*", nats.OrderedConsumer(), nats.BindStream(betaStream)); err == nil {
|
||||
sub.Unsubscribe()
|
||||
}
|
||||
}},
|
||||
}
|
||||
all := append(append(onBucket(alphaStream, alphaBucket, "its own bucket"), onBucket(betaStream, betaBucket, "beta's bucket")...), reachBeta...)
|
||||
for _, r := range all {
|
||||
nodeNC, nodeCh := connect("node.alpha")
|
||||
refusedWith("a node-engine", nodeNC, nodeCh, r)
|
||||
}
|
||||
// Another module is refused all of that on both buckets, and the reads by name on both.
|
||||
for _, r := range append(append(onBucket(alphaStream, alphaBucket, "alpha's bucket"), reachBeta...),
|
||||
refusal{"alpha's bucket: a read of one object by its name", func(nc *nats.Conn) {
|
||||
_, _ = nc.Request("$JS.API.DIRECT.GET."+alphaStream+".$O."+alphaBucket+".M."+
|
||||
base64.URLEncoding.EncodeToString([]byte(smallName)), nil, 300*time.Millisecond)
|
||||
}}) {
|
||||
otherNC, otherCh := connect("alpha.other-module")
|
||||
refusedWith("another module", otherNC, otherCh, r)
|
||||
}
|
||||
|
||||
// The literal `*` on alpha's own bucket reaches alpha's own files and never beta's: the chunks it is given hold no marker of beta's.
|
||||
js, err := alpha.JetStream(nats.MaxWait(2 * time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sub, err := js.SubscribeSync("$O."+alphaBucket+".C.*", nats.OrderedConsumer(), nats.BindStream(alphaStream)); err == nil {
|
||||
for i := 0; i < 20; i++ {
|
||||
msg, err := sub.NextMsg(300 * time.Millisecond)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if bytes.Contains(msg.Data, []byte("BETA-SECRET")) {
|
||||
t.Fatal("a literal `*` on alpha's bucket gave it a chunk of beta's file")
|
||||
}
|
||||
}
|
||||
sub.Unsubscribe()
|
||||
}
|
||||
|
||||
// Nothing was written by the refused: each bucket holds what the controller wrote.
|
||||
for node, want := range map[string]int{"alpha": 2, "beta": 1} {
|
||||
store, err := cjs.ObjectStore(ctx, mustBucket(t, node))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
infos, err := store.List(ctx)
|
||||
if err != nil || len(infos) != want {
|
||||
t.Errorf("the bucket of %s holds %d objects (%v), want the %d the controller wrote", node, len(infos), err, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mustBucket(t *testing.T, node string) string {
|
||||
t.Helper()
|
||||
b, err := DeclarationFilesBucketOf(node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func mustStream(t *testing.T, node string) string {
|
||||
t.Helper()
|
||||
b, err := DeclarationFilesStreamOf(node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
@@ -1,131 +0,0 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 1697/AC1: the composed accounts give a node-engine exactly the four reads of one object by its name, on its own node's bucket and
|
||||
// no other's; the controller the write of every node's bucket; and no other principal anything of any bucket.
|
||||
func TestEachNodesBucketIsWrittenByTheControllerAndReadByNameByThatNodesEngine(t *testing.T) { // 1697/AC1
|
||||
node, err := PermissionsFor(Principal{Kind: KindNode, Node: "alpha", PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var mine []string
|
||||
for _, s := range node.Publish {
|
||||
if strings.Contains(s, "declaration-files") {
|
||||
mine = append(mine, s)
|
||||
}
|
||||
}
|
||||
want := []string{
|
||||
"$JS.API.CONSUMER.CREATE.OBJ_declaration-files-alpha.*.$O.declaration-files-alpha.C.*",
|
||||
"$JS.API.CONSUMER.DELETE.OBJ_declaration-files-alpha.*",
|
||||
"$JS.API.DIRECT.GET.OBJ_declaration-files-alpha.$O.declaration-files-alpha.M.*",
|
||||
"$JS.FC.OBJ_declaration-files-alpha.>",
|
||||
}
|
||||
if !slices.Equal(mine, want) {
|
||||
t.Fatalf("a node-engine may publish %q of the buckets, want exactly %q", mine, want)
|
||||
}
|
||||
// What would make a list, a write, or a read of another node's files: none of it, in any pattern a node holds. (The real
|
||||
// server's judgement of the wildcard forms is the live test's.)
|
||||
for _, refused := range []string{
|
||||
"$JS.API.STREAM.INFO.OBJ_declaration-files-alpha",
|
||||
"$JS.API.STREAM.MSG.GET.OBJ_declaration-files-alpha",
|
||||
"$JS.API.DIRECT.GET.OBJ_declaration-files-alpha",
|
||||
"$JS.API.DIRECT.GET.OBJ_declaration-files-alpha.$O.declaration-files-alpha.M.abc.def",
|
||||
"$JS.API.CONSUMER.CREATE.OBJ_declaration-files-alpha.x.$O.declaration-files-alpha.M.YWJj",
|
||||
"$JS.API.CONSUMER.CREATE.OBJ_declaration-files-alpha.x.$O.declaration-files-alpha.M.YWJj.more",
|
||||
"$JS.API.CONSUMER.CREATE.OBJ_declaration-files-alpha",
|
||||
"$O.declaration-files-alpha.M.abc",
|
||||
"$O.declaration-files-alpha.C.abc",
|
||||
// another node's bucket, however it is named
|
||||
"$JS.API.DIRECT.GET.OBJ_declaration-files-beta.$O.declaration-files-beta.M.YWJj",
|
||||
"$JS.API.CONSUMER.CREATE.OBJ_declaration-files-beta.h4sh.$O.declaration-files-beta.C.NUID",
|
||||
"$JS.API.CONSUMER.DELETE.OBJ_declaration-files-beta.h4sh",
|
||||
"$JS.FC.OBJ_declaration-files-beta.x",
|
||||
// and the old shared bucket
|
||||
"$JS.API.DIRECT.GET.OBJ_declaration-files.$O.declaration-files.M.YWJj",
|
||||
} {
|
||||
if MayPublish(node, refused) {
|
||||
t.Errorf("a node-engine may publish %s, which lists or writes a bucket or reads another node's", refused)
|
||||
}
|
||||
}
|
||||
for _, kept := range []string{
|
||||
"$JS.API.DIRECT.GET.OBJ_declaration-files-alpha.$O.declaration-files-alpha.M.YWJj",
|
||||
"$JS.API.CONSUMER.CREATE.OBJ_declaration-files-alpha.h4sh.$O.declaration-files-alpha.C.NUID",
|
||||
} {
|
||||
if !MayPublish(node, kept) {
|
||||
t.Errorf("a node-engine may not publish %s, which reads one object of its own bucket by its name", kept)
|
||||
}
|
||||
}
|
||||
|
||||
controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x", FilesFor: []string{"alpha", "beta"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, s := range []string{"$O.declaration-files-alpha.M.abc", "$O.declaration-files-beta.C.nuid"} {
|
||||
if !MayPublish(controller, s) {
|
||||
t.Errorf("the controller may not write %s", s)
|
||||
}
|
||||
}
|
||||
if MayPublish(controller, "$O.declaration-files-gamma.M.abc") {
|
||||
t.Error("the controller may write the bucket of a node it was not given")
|
||||
}
|
||||
|
||||
for _, p := range []Principal{
|
||||
{Kind: KindModule, Module: "mesh-vault", PasswordHash: "x"},
|
||||
{Kind: KindNodeTools, Node: "alpha", PasswordHash: "x"},
|
||||
{Kind: KindPerson, Module: "jochen", Invokes: []string{"*"}, PasswordHash: "x"},
|
||||
} {
|
||||
perms, err := PermissionsFor(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, s := range append(append([]string(nil), perms.Publish...), perms.Subscribe...) {
|
||||
if strings.Contains(s, "declaration-files") {
|
||||
t.Errorf("%s holds %s, and no account but the controller's and the node-engines' has any access", p.Username(), s)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The users the controller composes carry each node's bucket to the controller and to that node alone.
|
||||
func TestTheComposedUsersGiveEachNodeItsOwnBucketAndTheControllerAll(t *testing.T) {
|
||||
users, err := Users(Records{Nodes: []string{"alpha", "beta"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var controller Principal
|
||||
for _, u := range users {
|
||||
if u.Kind == KindController {
|
||||
controller = u
|
||||
}
|
||||
}
|
||||
if !slices.Equal(controller.FilesFor, []string{"alpha", "beta"}) {
|
||||
t.Fatalf("the controller writes the buckets of %v", controller.FilesFor)
|
||||
}
|
||||
}
|
||||
|
||||
// A node name that could carry a subject wildcard or separator into a grant is refused with an error, never composed into a bucket
|
||||
// (ADR 0488): the grants of such a name would reach another node's bucket.
|
||||
func TestANodeNameThatCarriesAWildcardGetsNoBucketAndNoGrant(t *testing.T) {
|
||||
for _, node := range []string{"", "*", ">", "a.b", "a b", "a*"} {
|
||||
if b, err := DeclarationFilesBucketOf(node); err == nil {
|
||||
t.Errorf("node %q got the bucket %q", node, b)
|
||||
}
|
||||
if _, err := DeclarationFilesReads(node); err == nil {
|
||||
t.Errorf("node %q got read grants", node)
|
||||
}
|
||||
if _, err := DeclarationFilesWrites(node); err == nil {
|
||||
t.Errorf("node %q got write grants", node)
|
||||
}
|
||||
if _, err := DeclarationFilesConfig(node); err == nil {
|
||||
t.Errorf("node %q got a bucket config", node)
|
||||
}
|
||||
}
|
||||
if b, err := DeclarationFilesBucketOf("g14"); err != nil || b != "declaration-files-g14" {
|
||||
t.Errorf("g14: %q %v", b, err)
|
||||
}
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Issue 1840/S3 (ADR 0507 §5), 1840/AC5: a module whose manifest reads mesh-controller.declared-packages may name it in
|
||||
// `reads`, and reads the bucket and writes none; the controller writes it, so it is no mirror written by another, and it is
|
||||
// one of the controller's own buckets, asserted at its start.
|
||||
func TestAModuleReadingDeclaredPackagesReadsAndWritesNone(t *testing.T) {
|
||||
if !slices.Contains(ControllerMirrors(), "declared-packages") ||
|
||||
BucketName(ControllerSeat, MirrorDeclaredPackages) != DeclaredPackagesBucket {
|
||||
t.Fatalf("a module cannot name %s in `reads`: %v", DeclaredPackagesBucket, ControllerMirrors())
|
||||
}
|
||||
if _, other := MirrorsWrittenByAnother[MirrorDeclaredPackages]; other {
|
||||
t.Error("the controller writes the declared packages itself")
|
||||
}
|
||||
if !IsControllerBucket(DeclaredPackagesBucket) || !slices.Contains(ControllerBuckets(), DeclaredPackagesBucket) {
|
||||
t.Errorf("%s is not among the controller's buckets", DeclaredPackagesBucket)
|
||||
}
|
||||
perms, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "pacman", PasswordHash: "x",
|
||||
Reads: []string{"mesh-controller.declared-packages"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stream := "KV_" + DeclaredPackagesBucket
|
||||
has(t, perms.Publish, "$JS.API.STREAM.INFO."+stream)
|
||||
has(t, perms.Publish, "$JS.API.DIRECT.GET."+stream+".>")
|
||||
has(t, perms.Publish, "$JS.API.CONSUMER.CREATE."+stream+".>")
|
||||
for _, p := range perms.Publish {
|
||||
if SubjectsOverlap(p, "$KV."+DeclaredPackagesBucket+".one") {
|
||||
t.Errorf("the reading module may write %s through %q", DeclaredPackagesBucket, p)
|
||||
}
|
||||
}
|
||||
controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
has(t, controller.Publish, "$KV."+DeclaredPackagesBucket+".>")
|
||||
}
|
||||
@@ -2,7 +2,10 @@ package broker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
@@ -109,6 +112,25 @@ func DialPinned(url, fingerprint string, opts ...nats.Option) (*JetStream, error
|
||||
return Dial(url, opts...)
|
||||
}
|
||||
|
||||
// PinnedToFingerprint accepts exactly the certificate with this SHA-256 and no other.
|
||||
func PinnedToFingerprint(want string) *tls.Config {
|
||||
return &tls.Config{
|
||||
InsecureSkipVerify: true, //nolint:gosec // replaced by the pin below, which is stricter
|
||||
MinVersion: tls.VersionTLS12,
|
||||
VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
|
||||
if len(rawCerts) == 0 {
|
||||
return errors.New("the bus presented no certificate")
|
||||
}
|
||||
sum := sha256.Sum256(rawCerts[0])
|
||||
got := "sha256:" + hex.EncodeToString(sum[:])
|
||||
if got != want {
|
||||
return fmt.Errorf("the bus presented a certificate this mesh does not know (%s…), expected %s…", got[:23], want[:23])
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Conn is the connection itself, for what the mesh keeps off JetStream on purpose — a heartbeat,
|
||||
// a tool call — where a lost message is answered by the next one or by a timeout the caller
|
||||
// already handles (design 25 §3).
|
||||
|
||||
@@ -98,6 +98,17 @@ func (l Log) Config() jetstream.StreamConfig {
|
||||
}
|
||||
}
|
||||
|
||||
// LogIssued is one log an assignment may reach, by the name its module uses for it (novox/hq ADR
|
||||
// 0297): its stream, the subject its entries go under, and whether it may append. Only the owner's
|
||||
// instances are issued a log, so Writes is always true today; it is said so the runtime need not
|
||||
// assume it.
|
||||
type LogIssued struct {
|
||||
Name string `json:"name"`
|
||||
Stream string `json:"stream"`
|
||||
Subject string `json:"subject"`
|
||||
Writes bool `json:"writes"`
|
||||
}
|
||||
|
||||
// logsIssuedFor is every log a module's code may reach, as its membership lists them: its own.
|
||||
func logsIssuedFor(d Declared) []LogIssued {
|
||||
var out []LogIssued
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user