Compare commits
7
Commits
main
..
9b00d2ddf7
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9b00d2ddf7 | ||
|
|
1d5a523a53 | ||
|
|
e003f0e37b | ||
|
|
87075fee68 | ||
|
|
8f76123cbe | ||
|
|
0694f17934 | ||
|
|
c30b79dd2a |
@@ -3,8 +3,3 @@
|
||||
# A built binary. The lab writes one here when pointed at the repository root by mistake;
|
||||
# built artifacts belong in build/, which is already ignored.
|
||||
/mesh-builder
|
||||
/mesh-controller
|
||||
/postgres-provisioner
|
||||
/redis-provisioner
|
||||
# And where `go build` in cmd/mesh-controller leaves it (novox/hq issue 233).
|
||||
/cmd/mesh-controller/mesh-controller
|
||||
|
||||
+1
-3
@@ -27,11 +27,9 @@ WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
COPY vendor/ vendor/
|
||||
|
||||
# `COPY . .` brings `.git` along, so Go is told not to stamp the commit: the version is said by -X, and a
|
||||
# rebuild of an unchanged program is the same bytes (novox/hq ADR 0267 rule 6, issue 363).
|
||||
COPY . .
|
||||
ARG VERSION=development
|
||||
RUN CGO_ENABLED=0 GOFLAGS=-mod=vendor GOPROXY=off go build -trimpath -buildvcs=false \
|
||||
RUN CGO_ENABLED=0 GOFLAGS=-mod=vendor GOPROXY=off go build -trimpath \
|
||||
-ldflags "-s -w -X main.version=${VERSION}" \
|
||||
-o /mesh-controller ./cmd/mesh-controller
|
||||
|
||||
|
||||
@@ -95,43 +95,31 @@ PROXY_IMAGE ?= mesh-route-proxy:$(VERSION)
|
||||
PROXY_DEV_TAG ?= mesh-route-proxy:development
|
||||
|
||||
proxy-image:
|
||||
docker build --build-arg GO_BASE=$(GO_BASE) -f modules/route-proxy/Dockerfile -t $(PROXY_IMAGE) -t $(PROXY_DEV_TAG) .
|
||||
docker build --build-arg GO_BASE=$(GO_BASE) -f examples/route-proxy/Dockerfile -t $(PROXY_IMAGE) -t $(PROXY_DEV_TAG) .
|
||||
@echo
|
||||
@docker image inspect $(PROXY_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||
|
||||
# The whole check. Raises a database, runs the repository's own check against it -- merge-check.sh,
|
||||
# the very script `mesh/repo-check` runs -- and takes the database down again, including when the
|
||||
# check fails, which is why the teardown is not conditional and the script's exit status is the
|
||||
# target's.
|
||||
# The whole gate. Raises a database, runs everything against it, and takes it down again --
|
||||
# including when the tests fail, which is why the teardown is not conditional.
|
||||
#
|
||||
# **It calls the script rather than restating it** (novox/hq issue 431): this target used to list its
|
||||
# own steps, its formatting step walked vendor/ where the script's does not, and it failed on
|
||||
# third-party code no change could fix -- so a developer's check and the gate disagreed, and the steps
|
||||
# after formatting never ran through it. The script is the one list of steps; this target only gives it
|
||||
# a database (MESH_TEST_POSTGRES, exported above).
|
||||
#
|
||||
# **Two levels** (issue 895): the unit level, `go test ./...` untagged, timed and held to a bound; and the integration
|
||||
# level, `-tags integration`, the tests that start a bus, a store or another process. Each is said with its time.
|
||||
#
|
||||
# **Packages in parallel, under the race detector, each test on a bus of its own** (internal/testbus),
|
||||
# as the script runs them: a bus per test, of the release the mesh runs, makes the suite the same in any
|
||||
# order, and the timeout bounds a hang to a failure with a stack, never a stalled gate.
|
||||
check: postgres
|
||||
@sh merge-check.sh ; status=$$? ; docker rm -f $(PG_CONTAINER) >/dev/null 2>&1 || true ; exit $$status
|
||||
# **Packages in parallel, under the race detector, each test on a bus of its own** (internal/testbus).
|
||||
# It was one package at a time against one shared bus, because the live tests assert, read and remove
|
||||
# the mesh's own objects by their fixed names, and two packages at once deleted what the other read; the
|
||||
# suite was red run as Go runs it and read as noise. A bus per test, of the release the mesh runs, made
|
||||
# it the same in any order. The timeout bounds a hang to a failure with a stack, never a stalled gate.
|
||||
check: fmt vet postgres
|
||||
@go test -race -timeout 15m ./... ; status=$$? ; $(MAKE) postgres-stop ; exit $$status
|
||||
|
||||
# The unit level alone (untagged: no bus, no store, no other process), which a developer runs continuously. The tests
|
||||
# that need a bus or the store carry `//go:build integration` and run in `check`, not here; without a database the
|
||||
# store's tests would skip rather than fail anyway.
|
||||
# Without a database the store's tests skip rather than fail, so this is the honest subset and not
|
||||
# the gate.
|
||||
test:
|
||||
go test -timeout 15m ./...
|
||||
|
||||
vet:
|
||||
go vet ./...
|
||||
|
||||
# Quick steps for a developer, not part of `check`. The directories gofmt reads must be the ones
|
||||
# merge-check.sh lists, never `.`, which walks vendor/ (novox/hq issue 431).
|
||||
fmt:
|
||||
@unformatted=$$(gofmt -l cmd internal examples) ; \
|
||||
@unformatted=$$(gofmt -l . 2>/dev/null) ; \
|
||||
if [ -n "$$unformatted" ] ; then echo "not gofmt'd:" ; echo "$$unformatted" ; exit 1 ; fi
|
||||
|
||||
postgres:
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
# Release 57: config and provisioning
|
||||
|
||||
This branch carries the work of hq design 57 (ADR 0348, ADR 0349) in this repository. Each package
|
||||
merges here through its own reviewed pull request; the branch reaches main as one pull request, in one
|
||||
delivery group with the branch of the same name in mesh-controller, mesh-host, mesh-sdk, mesh-catalog
|
||||
and hq, after a rehearsal in the lab. This file is deleted by the last commit before that pull request.
|
||||
@@ -1,29 +0,0 @@
|
||||
#!/bin/sh
|
||||
# The unit tests of every Go module of its own below this repository's root, run from the root, for merge-check.sh
|
||||
# and for the test that tries it on a tree (merge_check_nested_test.go). Its own file so that the test needs no
|
||||
# particular merge-check.sh: the check that judges a change to merge-check.sh is main's.
|
||||
set -eu
|
||||
cgo="${cgo:-0}"
|
||||
race="${race:-}"
|
||||
|
||||
# nested_modules vets and tests every Go module of its own below the root, `modules/<name>/go.mod` (novox/hq ADR 0420
|
||||
# §5, issue 1025): `./...` stops at a nested go.mod, so a module that lives whole in this repository would have its
|
||||
# unit tests run by nobody. None is a pass. Each runs in its own directory, outside the vendor directory this
|
||||
# repository's own packages are built from; a nested module vendors what the forge holds (its own `vendor/`, so the
|
||||
# check runs offline), as this repository's root module does (issue 1019). A failing module fails the check; the others are still run.
|
||||
nested_modules() {
|
||||
nested=0
|
||||
for gomod in modules/*/go.mod; do
|
||||
[ -e "$gomod" ] || continue
|
||||
dir=$(dirname "$gomod")
|
||||
echo "nested module $dir:"
|
||||
if ! (cd "$dir" && GOFLAGS= CGO_ENABLED=0 go vet ./... &&
|
||||
GOFLAGS= CGO_ENABLED="${cgo:-0}" go test ${race:-} -count=1 -timeout 30m ./...); then
|
||||
echo "nested module $dir FAILED"
|
||||
nested=1
|
||||
fi
|
||||
done
|
||||
return $nested
|
||||
}
|
||||
|
||||
nested_modules
|
||||
@@ -11,7 +11,7 @@ WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN CGO_ENABLED=0 go build -trimpath -buildvcs=false -ldflags '-s -w' -o /mesh-builder ./cmd/mesh-builder
|
||||
RUN CGO_ENABLED=0 go build -trimpath -ldflags '-s -w' -o /mesh-builder ./cmd/mesh-builder
|
||||
|
||||
FROM ${ALPINE_BASE}
|
||||
# git to clone what it is asked to build, and the docker client to build and push it. The daemon
|
||||
|
||||
@@ -1,146 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **The build agent says whether it is busy, and being asked drains it** (novox/hq ADR 0335 §1, §5). Its
|
||||
// container declares `busy` as `mesh-builder busy`: the node-engine runs it inside the container before it
|
||||
// replaces it. It exits 0 while a build is in hand and 1 when none is; and from the first ask the agent takes
|
||||
// no new build request, so its answer ends once the build in hand ends — a replaced builder finishes the check
|
||||
// it runs, instead of the build queue handing it out again from the start (issue 594).
|
||||
//
|
||||
// Two marks in the container's own temporary directory, as the ready mark (ready.go): a new container starts
|
||||
// without them, and a restarted one removes them before it takes work again.
|
||||
//
|
||||
// - in hand: written by the agent when it claims a request, before it reads whether it is drained; removed
|
||||
// when the build ends.
|
||||
// - drained: written by the ask, before it reads whether a build is in hand.
|
||||
//
|
||||
// Each side writes its own mark before it reads the other's, so an ask racing a fetch sees the build in hand,
|
||||
// or the agent sees the drain and hands the request back; never neither.
|
||||
|
||||
func inHandMark() string { return filepath.Join(os.TempDir(), "mesh-builder.in-hand") }
|
||||
func drainMark() string { return filepath.Join(os.TempDir(), "mesh-builder.drained") }
|
||||
|
||||
// The drain lapses when no ask has come for drainLapse: the node-engine asks every 20 seconds for as long as it
|
||||
// really waits, so an agent whose replacement was withdrawn — the declaration put back, the walk stopped —
|
||||
// takes work again rather than staying drained for ever (review of ADR 0335). And an agent drained for longer
|
||||
// than the node-engine's bound and the slack says it is not ready: the replacement it was drained for never
|
||||
// came, which is a fault to see.
|
||||
const (
|
||||
drainLapse = 2 * time.Minute
|
||||
drainTooLong = 15*time.Minute + 5*time.Minute
|
||||
)
|
||||
|
||||
// drainedAt is when the drain began and when it was last asked, from its mark; false when there is no mark.
|
||||
func drainedAt() (began, asked time.Time, ok bool) {
|
||||
info, err := os.Stat(drainMark())
|
||||
if err != nil {
|
||||
return time.Time{}, time.Time{}, false
|
||||
}
|
||||
asked = info.ModTime()
|
||||
began = asked
|
||||
if raw, err := os.ReadFile(drainMark()); err == nil {
|
||||
if t, err := time.Parse(time.RFC3339, strings.TrimSpace(string(raw))); err == nil && !t.After(asked) {
|
||||
began = t
|
||||
}
|
||||
}
|
||||
return began, asked, true
|
||||
}
|
||||
|
||||
// drained says the agent was asked whether it is busy within drainLapse: it takes no new build request.
|
||||
func drained() bool { return drainedSince(time.Now()) }
|
||||
|
||||
func drainedSince(now time.Time) bool {
|
||||
_, asked, ok := drainedAt()
|
||||
return ok && now.Sub(asked) < drainLapse
|
||||
}
|
||||
|
||||
// drainedTooLong is the health check's second half: nil unless the agent has been drained, asked all along, for
|
||||
// longer than its node-engine's bound and the slack.
|
||||
func drainedTooLong(now time.Time) error {
|
||||
began, asked, ok := drainedAt()
|
||||
if !ok || now.Sub(asked) >= drainLapse || now.Sub(began) <= drainTooLong {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("drained since %s and still asked: the replacement its node-engine waits for has not come "+
|
||||
"within %s (novox/hq ADR 0335)", began.UTC().Format(time.RFC3339), drainTooLong)
|
||||
}
|
||||
|
||||
// claim marks a request in hand, unless the agent is drained: then nothing is in hand, and the request goes
|
||||
// back to the queue.
|
||||
func claim() bool {
|
||||
// Made, never written over: one build at a time, and a mark already there is a build in hand.
|
||||
f, err := os.OpenFile(inHandMark(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
|
||||
if err != nil {
|
||||
// Unmarked, an ask would say idle while this build runs: the build is not taken.
|
||||
fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so it is handed back: %v\n", inHandMark(), err)
|
||||
return false
|
||||
}
|
||||
_, err = f.WriteString(time.Now().UTC().Format(time.RFC3339) + "\n")
|
||||
if closeErr := f.Close(); err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
if err != nil {
|
||||
_ = os.Remove(inHandMark())
|
||||
fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so it is handed back: %v\n", inHandMark(), err)
|
||||
return false
|
||||
}
|
||||
if drained() {
|
||||
_ = os.Remove(inHandMark())
|
||||
fmt.Fprintln(os.Stderr, "asked whether busy, so this agent takes no new build: handed back")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// holding says what is in hand, for the ask to print: written once the request is read.
|
||||
func holding(what string) {
|
||||
_ = os.WriteFile(inHandMark(), []byte(time.Now().UTC().Format(time.RFC3339)+" "+what+"\n"), 0o644)
|
||||
}
|
||||
|
||||
// released ends the build in hand.
|
||||
func released() {
|
||||
if err := os.Remove(inHandMark()); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
fmt.Fprintf(os.Stderr, "cannot remove the mark of the build in hand (%s): %v\n", inHandMark(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// undrained clears both marks at the start of a holder: a container started again is asked afresh.
|
||||
func undrained() error {
|
||||
for _, mark := range []string{drainMark(), inHandMark()} {
|
||||
if err := os.Remove(mark); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("cannot remove %s: %w", mark, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// busy is the ask: it drains the agent, then says whether a build is in hand. Nil — exit 0 — while one is.
|
||||
func busy() (string, error) {
|
||||
// The mark keeps when the drain began while it is asked within drainLapse, and is written again on every
|
||||
// ask: its time is the last ask.
|
||||
began := time.Now().UTC()
|
||||
if was, _, ok := drainedAt(); ok && drained() {
|
||||
began = was.UTC()
|
||||
}
|
||||
if err := os.WriteFile(drainMark(), []byte(began.Format(time.RFC3339)+"\n"), 0o644); err != nil {
|
||||
// Not drained, it could take the next build: said, and answered busy only for what is in hand now.
|
||||
fmt.Fprintf(os.Stderr, "cannot mark this agent drained (%s): %v\n", drainMark(), err)
|
||||
}
|
||||
held, err := os.ReadFile(inHandMark())
|
||||
if err != nil {
|
||||
return "", errors.New("idle: no build in hand, and this agent takes no new one")
|
||||
}
|
||||
since, what, _ := strings.Cut(strings.TrimSpace(string(held)), " ")
|
||||
if what == "" {
|
||||
what = "a build"
|
||||
}
|
||||
return fmt.Sprintf("busy: %s, in hand since %s; taking no new build", what, since), nil
|
||||
}
|
||||
@@ -1,162 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// `mesh-builder busy` answers 0 while a build is in hand and non-zero when idle, and after its first ask the
|
||||
// agent takes no new build request (novox/hq ADR 0335 §5).
|
||||
func TestBusyAnswersWhileABuildIsInHandAndDrainsTheAgent(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
if err := undrained(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if drained() {
|
||||
t.Fatal("an agent never asked is drained")
|
||||
}
|
||||
// A build is claimed and runs.
|
||||
if !claim() {
|
||||
t.Fatal("an agent never asked refused a build")
|
||||
}
|
||||
holding("novox/mesh-controller (check-280)")
|
||||
said, err := busy()
|
||||
if err != nil || !strings.Contains(said, "novox/mesh-controller (check-280)") {
|
||||
t.Fatalf("an agent with a build in hand did not answer busy with what it holds: %q %v", said, err)
|
||||
}
|
||||
// From that ask on it takes nothing new: the fetch is not made, and one fetched is handed back.
|
||||
if !drained() {
|
||||
t.Fatal("the ask did not drain the agent")
|
||||
}
|
||||
if claim() {
|
||||
t.Fatal("a drained agent took a new build")
|
||||
}
|
||||
// The build in hand is still in hand after the refused claim, and busy goes on saying so.
|
||||
if _, err := busy(); err != nil {
|
||||
t.Fatalf("a refused claim ended the build in hand: %v", err)
|
||||
}
|
||||
// It ends: idle, and still drained.
|
||||
released()
|
||||
if _, err := busy(); err == nil {
|
||||
t.Fatal("an agent with nothing in hand answered busy")
|
||||
}
|
||||
if claim() {
|
||||
t.Fatal("a drained agent took a build after its last one ended")
|
||||
}
|
||||
// Started again, it is asked afresh.
|
||||
if err := undrained(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if drained() || !claim() {
|
||||
t.Fatal("a holder started again is still drained")
|
||||
}
|
||||
released()
|
||||
}
|
||||
|
||||
// An ask racing a fetch sees the build in hand, or the agent sees the drain and hands the request back: never
|
||||
// neither, which would replace the container under a build just taken.
|
||||
func TestAnAskRacingAFetchNeverMissesTheBuild(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
for i := 0; i < 200; i++ {
|
||||
if err := undrained(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var claimed bool
|
||||
var askErr error
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() { defer wg.Done(); claimed = claim() }()
|
||||
go func() { defer wg.Done(); _, askErr = busy() }()
|
||||
wg.Wait()
|
||||
if claimed && askErr != nil {
|
||||
t.Fatalf("round %d: the agent took a build and the ask said idle", i)
|
||||
}
|
||||
released()
|
||||
}
|
||||
}
|
||||
|
||||
// A drain no longer asked lapses: an agent whose replacement was withdrawn after the first ask (a put-back, a
|
||||
// revert, a stopped walk) takes work again after drainLapse, rather than staying drained with its health green.
|
||||
func TestADrainNoLongerAskedLapses(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
if err := undrained(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := busy(); err == nil {
|
||||
t.Fatal("an idle agent answered busy")
|
||||
}
|
||||
if !drained() || claim() {
|
||||
t.Fatal("an agent just asked takes work")
|
||||
}
|
||||
// The last ask was three minutes ago.
|
||||
old := time.Now().Add(-3 * time.Minute)
|
||||
if err := os.Chtimes(drainMark(), old, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if drained() {
|
||||
t.Fatal("a drain no longer asked for three minutes still holds")
|
||||
}
|
||||
if !claim() {
|
||||
t.Fatal("an agent whose drain lapsed takes no work")
|
||||
}
|
||||
released()
|
||||
}
|
||||
|
||||
// An agent drained, and asked all along, for longer than its node-engine's bound and the slack is not ready: the
|
||||
// replacement it was drained for never came. One asked again after a lapse starts a new drain.
|
||||
func TestAnAgentDrainedTooLongIsNotReady(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
if err := undrained(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
taking("mesh-build-machine")
|
||||
if _, err := busy(); err == nil {
|
||||
t.Fatal("an idle agent answered busy")
|
||||
}
|
||||
now := time.Now()
|
||||
if err := drainedTooLong(now); err != nil {
|
||||
t.Fatalf("an agent drained a moment ago is not ready: %v", err)
|
||||
}
|
||||
// The drain began 25 minutes ago, and the node-engine asked a moment ago.
|
||||
began := now.Add(-25 * time.Minute).UTC().Format(time.RFC3339)
|
||||
if err := os.WriteFile(drainMark(), []byte(began+"\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err := drainedTooLong(time.Now())
|
||||
if err == nil || !strings.Contains(err.Error(), "has not come") {
|
||||
t.Fatalf("an agent drained for 25 minutes reads ready: %v", err)
|
||||
}
|
||||
// Each ask keeps when the drain began.
|
||||
if _, err := busy(); err == nil {
|
||||
t.Fatal("an idle agent answered busy")
|
||||
}
|
||||
if err := drainedTooLong(time.Now()); err == nil {
|
||||
t.Fatal("an ask started the drain again")
|
||||
}
|
||||
// The asks stopped: the drain lapsed, and so did the fault.
|
||||
old := time.Now().Add(-3 * time.Minute)
|
||||
if err := os.Chtimes(drainMark(), old, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := drainedTooLong(time.Now()); err != nil {
|
||||
t.Fatalf("a lapsed drain still reads not ready: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// An ask this build does not know exits non-zero: an older image asked `busy` reads idle, not busy for the
|
||||
// node-engine's whole bound.
|
||||
func TestAnUnknownCommandFails(t *testing.T) {
|
||||
was := os.Args
|
||||
t.Cleanup(func() { os.Args = was })
|
||||
os.Args = []string{"mesh-builder", "some-later-word"}
|
||||
if err := run(); err == nil {
|
||||
t.Fatal("an unknown command exited 0")
|
||||
}
|
||||
os.Args = []string{"mesh-builder", "help"}
|
||||
if err := run(); err != nil {
|
||||
t.Fatalf("help failed: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The build seat hands the check the judge cache the build agent declares (MESH_JUDGE_CACHE), never a path of its own.
|
||||
func TestTheBuildSeatHandsTheCheckTheJudgeCacheItDeclares(t *testing.T) {
|
||||
t.Setenv("MESH_JUDGE_CACHE", "/var/lib/mesh/build-agent/judge-cache")
|
||||
if got := judgeCache(); got != "/var/lib/mesh/build-agent/judge-cache" {
|
||||
t.Fatalf("the judge cache the manifest declares is read as %q", got)
|
||||
}
|
||||
t.Setenv("MESH_JUDGE_CACHE", "")
|
||||
if got := judgeCache(); got != "" {
|
||||
t.Fatalf("with no judge cache declared, %q is used", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The siblings a check is asked to read reach the build seat's spec and, as read, the outcome (novox/hq ADR 0382).
|
||||
func TestTheSiblingsAnAskNamesReachTheSpecAndTheOutcome(t *testing.T) {
|
||||
spec := checkSpecOf(link.BuildRequest{Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-host", Base: "main",
|
||||
Siblings: []link.CheckSibling{{Repository: "novox/mesh-controller", Number: 346, Head: "a", Base: "main", Dir: "mesh-controller"}},
|
||||
SiblingsUnknown: "why"}})
|
||||
if len(spec.Siblings) != 1 || spec.Siblings[0] != (builder.Sibling{Repository: "novox/mesh-controller", Number: 346,
|
||||
Head: "a", Base: "main", Dir: "mesh-controller"}) || spec.SiblingsUnknown != "why" {
|
||||
t.Fatalf("the spec reads %+v", spec)
|
||||
}
|
||||
out := siblingsOut([]builder.JudgedSibling{{Repository: "novox/mesh-controller", Number: 346, Head: "a", Base: "main", BaseCommit: "b"}})
|
||||
if len(out) != 1 || out[0].BaseCommit != "b" || out[0].Number != 346 {
|
||||
t.Fatalf("the outcome reads %+v", out)
|
||||
}
|
||||
if plain := checkSpecOf(link.BuildRequest{Check: &link.CheckRequest{Owner: "o", Repo: "r"}}); plain.Siblings != nil {
|
||||
t.Fatalf("a request that names none has siblings %+v", plain.Siblings)
|
||||
}
|
||||
}
|
||||
@@ -41,9 +41,6 @@ type holder struct {
|
||||
say func(link.HolderState) error
|
||||
// remove runs what a kill needs outside the build: the containers left behind.
|
||||
remove builder.Runner
|
||||
// scratch is where check builds are kept (novox/hq ADR 0350 §5): what the week's sweep lets go of. Nil where
|
||||
// this machine publishes to no artifact store.
|
||||
scratch builder.Scratch
|
||||
|
||||
mu sync.Mutex
|
||||
paused bool
|
||||
@@ -134,34 +131,6 @@ func (h *holder) stateWhilePaused(ctx context.Context, every time.Duration) {
|
||||
}
|
||||
}
|
||||
|
||||
// expireChecks lets go of every check build older than a week (novox/hq ADR 0350 §5), at start and every while
|
||||
// after. Every holder sweeps the one store; what one let go of, another finds gone.
|
||||
func (h *holder) expireChecks(ctx context.Context, every time.Duration, now func() time.Time) {
|
||||
sweep := func() {
|
||||
if h.scratch == nil {
|
||||
return
|
||||
}
|
||||
n, err := h.scratch.Expire(ctx, builder.CheckBuildsKept, now())
|
||||
switch {
|
||||
case err != nil:
|
||||
fmt.Fprintf(os.Stderr, "check builds older than %s: %d let go of, and not all: %v\n", builder.CheckBuildsKept, n, err)
|
||||
case n > 0:
|
||||
fmt.Fprintf(os.Stderr, "check builds older than %s: %d check(s)' builds let go of\n", builder.CheckBuildsKept, n)
|
||||
}
|
||||
}
|
||||
sweep()
|
||||
tick := time.NewTicker(every)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
sweep()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// begin records the build this machine took, with the cancel that ends it.
|
||||
func (h *holder) begin(request link.BuildRequest, cancel context.CancelFunc) *running {
|
||||
r := &running{request: request, started: time.Now(), cancel: cancel, done: make(chan struct{})}
|
||||
|
||||
+68
-382
@@ -18,16 +18,12 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -58,8 +54,6 @@ is dialled except the broker.
|
||||
MESH_BINDING a file the mesh wrote saying where the artifact store is
|
||||
MESH_PACKAGE_BINDING a file the mesh wrote saying where the package registry is
|
||||
MESH_NPM_TOKEN_FILE a file the mesh sealed holding the token for it
|
||||
MESH_GENESIS_PACKAGE_REGISTRY the document the installer sealed at genesis, read while no
|
||||
binding is present (novox/hq ADR 0336)
|
||||
MESH_NPM_REGISTRY a package registry URL, when the mesh has not said (a person, the bootstrap)
|
||||
MESH_NPM_TOKEN the token for it, likewise
|
||||
MESH_NPM_SCOPE the scope it answers for (default: @novox)
|
||||
@@ -72,17 +66,6 @@ broker to take work from or a registry to publish into:
|
||||
|
||||
Without --registry the artifacts stay in this machine's container runtime, named by the
|
||||
digest of their own configuration. The result is printed as JSON.
|
||||
|
||||
mesh-builder ready
|
||||
|
||||
exits 0 once the agent running in this container is bound to its build work and pulls it, and
|
||||
1 before: its container's health check.
|
||||
|
||||
mesh-builder busy
|
||||
|
||||
exits 0 while the agent running in this container has a build in hand, and 1 when it has none;
|
||||
from the first ask the agent takes no new build: its container's busy command, asked before the
|
||||
container is replaced (novox/hq ADR 0335).
|
||||
`
|
||||
|
||||
func run() error {
|
||||
@@ -93,42 +76,12 @@ func run() error {
|
||||
return nil
|
||||
case "build":
|
||||
return buildOnce(context.Background(), os.Args[2:])
|
||||
case "ready":
|
||||
// The container's health check (novox/hq issue 544): exits non-zero until the agent takes work, and
|
||||
// once it has been drained longer than its node-engine would wait to replace it (ADR 0335).
|
||||
if err := ready(); err != nil {
|
||||
return err
|
||||
}
|
||||
return drainedTooLong(time.Now())
|
||||
case "busy":
|
||||
// The container's busy command (novox/hq ADR 0335): drains the agent, and exits non-zero once
|
||||
// nothing is in hand.
|
||||
said, err := busy()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(said)
|
||||
return nil
|
||||
case "help", "-h", "--help":
|
||||
default:
|
||||
fmt.Print(usage)
|
||||
return nil
|
||||
default:
|
||||
// **An ask this build does not know fails** (novox/hq ADR 0335): an image older than `busy`, asked
|
||||
// it, must read as idle — exiting 0 would hold its container for the node-engine's whole bound.
|
||||
fmt.Fprint(os.Stderr, usage)
|
||||
return fmt.Errorf("%q is not a command this mesh-builder knows", os.Args[1])
|
||||
}
|
||||
}
|
||||
|
||||
if err := notReady(); err != nil {
|
||||
return err
|
||||
}
|
||||
// **The builder's own version is part of every reuse key** (novox/hq ADR 0350 §1): a builder that builds
|
||||
// differently never takes a build another one made.
|
||||
builder.Version = builder.SelfVersion(version)
|
||||
if err := undrained(); err != nil {
|
||||
return err
|
||||
}
|
||||
credential, err := brokerFrom()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -181,13 +134,7 @@ func run() error {
|
||||
// **What an earlier holder here left goes before anything is taken** (novox/hq issue 306): a holder
|
||||
// recreated mid-check left the check's store and bus running, and the ask went to another machine.
|
||||
h.removeLeftBehind()
|
||||
// Paused or drained, it takes nothing new (novox/hq ADR 0219, ADR 0335): a drained agent finishes the build
|
||||
// in hand and then says idle, and its container is replaced between builds.
|
||||
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{
|
||||
Paused: func() bool { return h.Paused() || drained() },
|
||||
Taking: func() { taking(seat) },
|
||||
Claim: claim,
|
||||
})
|
||||
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused})
|
||||
defer machine.Close()
|
||||
|
||||
// The seat's verbs, on this machine's subjects. Only the seat that declares them: the retired
|
||||
@@ -211,13 +158,8 @@ func run() error {
|
||||
|
||||
fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry)
|
||||
publisher := builder.Registry{Address: registry, Run: builder.Command}
|
||||
// Check builds are let go of after a week (novox/hq ADR 0350 §5).
|
||||
h.scratch = publisher
|
||||
go h.expireChecks(ctx, time.Hour, time.Now)
|
||||
|
||||
return machine.Take(ctx, func(ctx context.Context, work link.Build) {
|
||||
defer released()
|
||||
holding(work.Request().Repository + " (" + work.Request().ID + ")")
|
||||
answer(ctx, publisher, on, workspace, work, h)
|
||||
})
|
||||
}
|
||||
@@ -274,12 +216,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
// role's own events under the build's id (novox/hq ADR 0157) — so whoever asked, and anybody
|
||||
// watching, reads the same lines this container's log holds, live, and after the fact from the
|
||||
// stream. Said first, before anything runs, so a build that hangs is one that visibly started.
|
||||
// One line at a time: a check's builds for its merge say their lines while its tests say theirs (novox/hq
|
||||
// issue 716).
|
||||
var saying sync.Mutex
|
||||
say := func(step, message string) {
|
||||
saying.Lock()
|
||||
defer saying.Unlock()
|
||||
fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message)
|
||||
work.Say(step, message)
|
||||
if mine != nil && step != "run" && step != "output" {
|
||||
@@ -291,26 +228,6 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
if err := work.Began(ctx); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "cannot say a build started: %v\n", err)
|
||||
}
|
||||
// **A running check watches its pull request's key** (novox/hq ADR 0406 §4, issue 974): when the head or the trunk
|
||||
// commit in it moves, the check is for a state nobody wants any more, and ends itself — nothing is asked of the forge
|
||||
// (issue 726). A group's check, a try and a build watch nothing.
|
||||
var superseded atomic.Bool
|
||||
if w, ok := work.(interface {
|
||||
WatchPull(context.Context, func(link.PullKey)) (func(), error)
|
||||
}); ok && request.Check != nil {
|
||||
stopWatching, err := w.WatchPull(building, func(k link.PullKey) {
|
||||
superseded.Store(true)
|
||||
say("superseded", fmt.Sprintf("the pull request's key moved to %.12s on %.12s while this checked %.12s on %.12s: stopped",
|
||||
k.Head, k.Trunk, request.Ref, request.Check.Trunk))
|
||||
cancel()
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "the key of %s/%s#%d could not be watched, so its check runs on if the key moves: %v\n",
|
||||
request.Check.Owner, request.Check.Repo, request.Check.Number, err)
|
||||
} else {
|
||||
defer stopWatching()
|
||||
}
|
||||
}
|
||||
|
||||
result := link.BuildResult{
|
||||
ID: request.ID, Repository: request.Repository, Path: request.Path,
|
||||
@@ -325,11 +242,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
}
|
||||
say("build", what)
|
||||
|
||||
npmrc, err := packagesFrom(on)
|
||||
if err == nil {
|
||||
say("packages", saysRegistry(npmrc))
|
||||
}
|
||||
registryHere, isRegistry := publisher.(builder.Registry)
|
||||
npmrc, err := packagesFrom()
|
||||
var built builder.Result
|
||||
if request.Check != nil {
|
||||
// **A pull request's merge check, not a build** (novox/hq to-be 45 §9): nothing is built,
|
||||
@@ -339,37 +252,11 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
if r, ok := publisher.(builder.Registry); ok {
|
||||
registry = r.Address
|
||||
}
|
||||
spec := checkSpecOf(request)
|
||||
// The judge cache the build agent declares as its own data, never in the workspace: only judge builds and
|
||||
// mesh-lab's replays mount it, never a check's containers, so no pull request's script can write what a judge
|
||||
// is built with or kept as, or what the replays run (novox/hq ADR 0358 §6, issue 709).
|
||||
spec.JudgeCache = judgeCache()
|
||||
// What the merge would move, built by the check for the walk to take (novox/hq ADR 0350 §1): into the
|
||||
// scratch namespace. **Never with the package registry's credential**: this is code nobody has approved
|
||||
// yet, and a recipe could read the credential it is handed (issue 462). A module that resolves packages
|
||||
// at build time has no reuse key anyway, so nothing is lost by refusing it here.
|
||||
if isRegistry && len(spec.Builds) > 0 {
|
||||
spec.BuildWith = &builder.CheckBuilding{Publish: publisher, Scratch: registryHere, Held: request.Held,
|
||||
Running: request.Check.Running, Seats: request.Seats, Named: namedOf(request.Check.Named)}
|
||||
}
|
||||
var v builder.CheckVerdict
|
||||
v, err = builder.Check(building, builder.Command, spec, workspace, registry, forgeFrom(), say)
|
||||
v, err = builder.Check(building, builder.Command, checkSpecOf(request), workspace, registry, forgeFrom(), say)
|
||||
if err == nil {
|
||||
result.Check = &link.CheckOutcome{Verdict: v.Verdict, Summary: v.Summary, Report: v.Report,
|
||||
Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo),
|
||||
Head: v.Head, Trunk: v.Trunk, Beside: v.Beside, Built: checkBuiltOf(v.Built),
|
||||
Validator: v.Validator, Siblings: siblingsOut(v.Siblings), SiblingsUnknown: v.SiblingsUnknown, JudgeFrom: v.JudgeFrom}
|
||||
// What each of a group's heads was judged as (novox/hq ADR 0323 §6).
|
||||
for _, m := range v.Members {
|
||||
result.Check.Members = append(result.Check.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo,
|
||||
Commit: m.Head, Trunk: m.Trunk})
|
||||
}
|
||||
// A branch tried before its pull request reads its outcome from this log, whole (novox/hq issue 686).
|
||||
if request.Check.Try {
|
||||
if line, err := json.Marshal(triedOf(v)); err == nil {
|
||||
say(link.StepTried, string(line))
|
||||
}
|
||||
}
|
||||
Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo)}
|
||||
}
|
||||
} else if err == nil {
|
||||
// The package-registry credential is a build input, so it is resolved before the clone: a
|
||||
@@ -377,22 +264,9 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
// after a clone that then fails at npm ci.
|
||||
// Every container it starts is labelled with its id, so a kill finds what outlived the
|
||||
// docker client (ADR 0219).
|
||||
if request.Reuse != nil && isRegistry && !request.DryRun {
|
||||
// **A walk's build takes the check's build of the same key** (novox/hq ADR 0350 §2).
|
||||
reuse := builder.Reuse{Scratch: registryHere}
|
||||
for _, p := range request.Reuse.Pulls {
|
||||
reuse.Pulls = append(reuse.Pulls, builder.PullRef{Owner: p.Owner, Repo: p.Repo, Number: p.Number})
|
||||
}
|
||||
// Only what a verdict names is taken (ADR 0350 §2).
|
||||
reuse.Named = namedOf(request.Reuse.Builds)
|
||||
built, err = builder.BuildReusing(building, builder.Labelled(builder.Command, request.ID), publisher, reuse,
|
||||
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
||||
forgeFrom(), say, request.Seats)
|
||||
} else {
|
||||
built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher,
|
||||
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
||||
forgeFrom(), say, request.Seats)
|
||||
}
|
||||
built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher,
|
||||
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
||||
forgeFrom(), say, request.Seats)
|
||||
}
|
||||
// What it copied into the store, whatever became of the build (novox/hq ADR 0257).
|
||||
result.Mirrored = built.Mirrored
|
||||
@@ -402,11 +276,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
if mine != nil {
|
||||
killed = h.returned(mine) && err != nil
|
||||
}
|
||||
if superseded.Load() && err != nil {
|
||||
// Ended by its own key moving on (ADR 0406 §4): said as the check's failure, as a kill is.
|
||||
result.Failed = link.SupersededByKey
|
||||
say("failed", link.SupersededByKey)
|
||||
} else if killed {
|
||||
if killed {
|
||||
// **Killed by hand is the outcome, whatever the build was doing** (novox/hq ADR 0219): the
|
||||
// error it ended with is the kill's consequence, not a fault of the source.
|
||||
result.Failed = link.KilledByHand
|
||||
@@ -432,31 +302,10 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
}
|
||||
result.Against = built.Against
|
||||
result.SourceFingerprint = built.Source
|
||||
// What the walk did instead of building, or why it built (novox/hq ADR 0350 §3, §4).
|
||||
result.ReuseKey = built.ReuseKey
|
||||
if built.Reused != nil {
|
||||
f := built.Reused.From
|
||||
result.Reused = built.Reused.Words()
|
||||
result.ReusedFrom = &link.ReusedFrom{Pull: link.PullRef{Owner: f.Pull.Owner, Repo: f.Pull.Repo,
|
||||
Number: f.Pull.Number}, Head: f.Head, Trunk: f.Trunk, Check: f.Check, Key: f.Key}
|
||||
} else if request.Reuse != nil {
|
||||
result.BuiltWhy = built.Why
|
||||
if result.BuiltWhy == "" {
|
||||
result.BuiltWhy = "it has no artifact to build"
|
||||
}
|
||||
}
|
||||
result.Trunk, result.OnTrunk, result.Branches = built.Trunk, built.OnTrunk, built.Branches
|
||||
for _, r := range built.Read {
|
||||
result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref})
|
||||
}
|
||||
for _, s := range built.Sources {
|
||||
result.Sources = append(result.Sources, link.BuildSource{Repository: s.Repository, Ref: s.Ref, Paths: s.Paths})
|
||||
}
|
||||
// The list of predefined keys the manifest was judged against, for the controller to publish
|
||||
// (novox/hq ADR 0348 §7).
|
||||
if p := built.Predefined; p != nil {
|
||||
result.Predefined = &link.PredefinedList{Keys: p.Keys, Generation: p.Generation}
|
||||
}
|
||||
say("built", built.Manifest.Module+" from "+short(built.Commit))
|
||||
}
|
||||
}
|
||||
@@ -496,88 +345,17 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
|
||||
spec := builder.CheckSpec{ID: request.ID, Repository: request.Repository, Ref: request.Ref,
|
||||
Owner: c.Owner, Repo: c.Repo, Number: c.Number, Paths: c.Paths, Beside: map[string]builder.Beside{},
|
||||
Modules: c.Modules, New: c.New, Manifests: c.Manifests, Judge: c.Judge, Base: c.Base,
|
||||
Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held),
|
||||
Tests: c.Tests, StoreOnly: c.StoreOnly, SiblingsUnknown: c.SiblingsUnknown}
|
||||
for _, s := range c.Siblings {
|
||||
spec.Siblings = append(spec.Siblings, builder.Sibling{Repository: s.Repository, Number: s.Number, Head: s.Head,
|
||||
Base: s.Base, Dir: s.Dir})
|
||||
}
|
||||
Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held)}
|
||||
for dir, b := range c.Beside {
|
||||
spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref}
|
||||
}
|
||||
for _, b := range c.Builds {
|
||||
spec.Builds = append(spec.Builds, builder.CheckBuildSpec{Module: b.Module, Repository: b.Repository, Path: b.Path,
|
||||
Ref: b.Ref, Merged: b.Merged, Tier: b.Tier, Skip: b.Skip})
|
||||
}
|
||||
for _, m := range c.Members {
|
||||
spec.Group = append(spec.Group, builder.GroupHead{Owner: m.Owner, Repo: m.Repo, Repository: m.Repository,
|
||||
Ref: m.Ref, Paths: m.Paths, Base: m.Base})
|
||||
Ref: m.Ref, Paths: m.Paths})
|
||||
}
|
||||
return spec
|
||||
}
|
||||
|
||||
// siblingsOut is the siblings a check read as the outcome carries them (novox/hq ADR 0382 §4).
|
||||
func siblingsOut(read []builder.JudgedSibling) []link.CheckSibling {
|
||||
var out []link.CheckSibling
|
||||
for _, s := range read {
|
||||
out = append(out, link.CheckSibling{Repository: s.Repository, Number: s.Number, Head: s.Head, Base: s.Base,
|
||||
BaseCommit: s.BaseCommit})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// judgeCache is the judge cache the build agent's manifest declares (MESH_JUDGE_CACHE, novox/hq ADR 0358 §6): a
|
||||
// directory mounted at the same path on the host, so a judge build's mounts name what the agent sees. Empty when the
|
||||
// agent declares none, and every judge is then built cold under its check's root.
|
||||
func judgeCache() string { return os.Getenv("MESH_JUDGE_CACHE") }
|
||||
|
||||
// triedOf is a try's outcome as its log line says it (novox/hq issue 686): the repository's own check's verdict,
|
||||
// or the check's where that did not run — a head that does not merge — and the lines naming what failed.
|
||||
func triedOf(v builder.CheckVerdict) link.Tried {
|
||||
t := link.Tried{Verdict: v.Verdict, Summary: v.Summary, Took: v.Took.Round(time.Second).String(),
|
||||
Head: v.Head, Trunk: v.Trunk}
|
||||
if v.Repo != nil && (v.Gate == nil || v.Gate.Verdict == "pass") {
|
||||
t.Verdict, t.Summary = v.Repo.Verdict, v.Repo.Summary
|
||||
}
|
||||
if v.Repo != nil && v.Repo.Failed != "" {
|
||||
t.Failed = strings.Split(v.Repo.Failed, "\n")
|
||||
}
|
||||
if t.Verdict == "" {
|
||||
t.Verdict = "error"
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// namedOf are the builds the seat's verdicts name, as the builder believes a scratch record by them (novox/hq ADR
|
||||
// 0350 §2, issue 703): each module's key with every artifact's digest.
|
||||
func namedOf(builds []link.CheckedBuild) []builder.NamedBuild {
|
||||
var out []builder.NamedBuild
|
||||
for _, b := range builds {
|
||||
named := builder.NamedBuild{Pull: builder.PullRef{Owner: b.Pull.Owner, Repo: b.Pull.Repo,
|
||||
Number: b.Pull.Number}, Check: b.Check, Module: b.Built.Module, Key: b.Built.Key,
|
||||
Digests: map[string]string{}}
|
||||
for _, m := range b.Built.Made {
|
||||
named.Digests[m.Name] = builder.DigestOf(m.Reference)
|
||||
}
|
||||
out = append(out, named)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// checkBuiltOf is a check's builds for its merge as the outcome carries them (novox/hq ADR 0350 §1).
|
||||
func checkBuiltOf(built []builder.CheckBuilt) []link.CheckBuilt {
|
||||
var out []link.CheckBuilt
|
||||
for _, b := range built {
|
||||
c := link.CheckBuilt{Module: b.Module, Key: b.Key, NotBuilt: b.NotBuilt, Took: b.Took.Round(time.Second).String(),
|
||||
Kept: b.Kept, Same: b.Same, SameAs: b.SameAs}
|
||||
for _, m := range b.Made {
|
||||
c.Made = append(c.Made, link.MadeArtifact{Name: m.Name, Kind: m.Kind, Reference: m.Reference})
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// layerOf is one layer of a check as the outcome carries it.
|
||||
func layerOf(l *builder.Layer) *link.CheckLayer {
|
||||
if l == nil {
|
||||
@@ -587,171 +365,79 @@ func layerOf(l *builder.Layer) *link.CheckLayer {
|
||||
}
|
||||
|
||||
// packagesFrom is where a build resolves the mesh's own published packages — the SDK above all
|
||||
// (novox/hq ADR 0076, issue 053) — on this machine.
|
||||
// (novox/hq ADR 0076, issue 053).
|
||||
//
|
||||
// **The binding wins, then the genesis document** (novox/hq ADR 0336 §3, §4). The build agent's
|
||||
// npm-package-registry requirement is optional: on genesis's node, before the gitea module is assigned,
|
||||
// nothing provides it, and the installer sealed the forge it raised to this agent as one document
|
||||
// instead. Once the requirement binds, the binding names the endpoint the way the artifact store's
|
||||
// does and a sealed token file the credential, and the genesis document is left unread. A binding
|
||||
// file named and absent is a requirement not bound here, never a fault: the environment names the
|
||||
// path whether or not the mesh wrote it.
|
||||
//
|
||||
// The environment variables remain for a builder run by a person. With none of the three the result
|
||||
// is disabled and says this machine, so a build that needs the registry is refused naming it (§6),
|
||||
// and a build that needs none builds anyway.
|
||||
func packagesFrom(machine string) (builder.Npmrc, error) {
|
||||
// Preferably from the mesh: a package-registry binding names the endpoint the way the artifact
|
||||
// store's binding does, and a sealed token file the credential the way the broker's does. The
|
||||
// environment variables remain for a builder run by a person, and for the bootstrap, where there is
|
||||
// no registry yet — there the result is disabled and a build that needs no mesh-published dependency
|
||||
// builds anyway.
|
||||
func packagesFrom() (builder.Npmrc, error) {
|
||||
scope := strings.TrimSpace(os.Getenv("MESH_NPM_SCOPE"))
|
||||
if scope == "" {
|
||||
scope = "@novox"
|
||||
}
|
||||
none := builder.Npmrc{Machine: machine}
|
||||
|
||||
registry := strings.TrimSpace(os.Getenv("MESH_NPM_REGISTRY"))
|
||||
var username string
|
||||
if path := strings.TrimSpace(os.Getenv("MESH_PACKAGE_BINDING")); path != "" {
|
||||
raw, err := os.ReadFile(path)
|
||||
switch {
|
||||
case errors.Is(err, os.ErrNotExist):
|
||||
// Not bound on this machine: the genesis document, or nothing.
|
||||
case err != nil:
|
||||
return none, fmt.Errorf("cannot read what the mesh said about the package registry: %w", err)
|
||||
default:
|
||||
return fromBinding(path, raw, scope, machine)
|
||||
}
|
||||
}
|
||||
if path := strings.TrimSpace(os.Getenv("MESH_GENESIS_PACKAGE_REGISTRY")); path != "" {
|
||||
raw, err := os.ReadFile(path)
|
||||
switch {
|
||||
case errors.Is(err, os.ErrNotExist):
|
||||
// Not this mesh's genesis node, or the installer has not sealed it yet.
|
||||
case err != nil:
|
||||
return none, fmt.Errorf("cannot read the genesis document for the package registry: %w", err)
|
||||
default:
|
||||
return fromGenesis(path, raw, scope, machine)
|
||||
}
|
||||
}
|
||||
|
||||
// A builder run by a person.
|
||||
registry := strings.TrimSpace(os.Getenv("MESH_NPM_REGISTRY"))
|
||||
secret := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN"))
|
||||
username := strings.TrimSpace(os.Getenv("MESH_NPM_USER"))
|
||||
if registry == "" && secret == "" {
|
||||
return none, nil
|
||||
}
|
||||
n := builder.Npmrc{Scope: scope, Registry: registry, From: builder.FromEnvironment, Machine: machine}
|
||||
if username != "" {
|
||||
n.Username, n.Password = username, secret
|
||||
} else {
|
||||
n.Token = secret
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// fromBinding composes the registry from the npm-package-registry binding the mesh wrote and the
|
||||
// credential it sealed beside it. The mesh authenticates the ordinary way — a generated password the
|
||||
// provider only applies (novox/hq ADR 0048) — so with a username the secret is a password (basic
|
||||
// auth); without one it is a bearer token a provider minted.
|
||||
func fromBinding(path string, raw []byte, scope, machine string) (builder.Npmrc, error) {
|
||||
none := builder.Npmrc{Machine: machine}
|
||||
var told struct {
|
||||
From string `json:"from"`
|
||||
At string `json:"at"`
|
||||
As string `json:"as"`
|
||||
Serves map[string]any `json:"serves"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &told); err != nil {
|
||||
return none, fmt.Errorf("%s is not a binding: %w", path, err)
|
||||
}
|
||||
if told.At == "" {
|
||||
return none, fmt.Errorf(
|
||||
"%s says the package registry is on %q and gives no address for it", path, told.From)
|
||||
}
|
||||
// Composed from what the provider serves, so nothing here knows gitea's URL shape from
|
||||
// another registry's: it states its port, the path its registry answers on, and the scheme.
|
||||
scheme := "https"
|
||||
if s, ok := told.Serves["scheme"]; ok {
|
||||
scheme = fmt.Sprintf("%v", s)
|
||||
}
|
||||
port, ok := told.Serves["port"]
|
||||
if !ok {
|
||||
return none, fmt.Errorf("%s says nothing about which port the package registry answers on", path)
|
||||
}
|
||||
npmPath, ok := told.Serves["npm-path"]
|
||||
if !ok {
|
||||
return none, fmt.Errorf("%s says nothing about the path the package registry answers on", path)
|
||||
}
|
||||
secret := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN"))
|
||||
if file := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN_FILE")); file != "" {
|
||||
raw, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
return none, fmt.Errorf("cannot read this builder's package-registry credential: %w", err)
|
||||
return builder.Npmrc{}, fmt.Errorf("cannot read what the mesh said about the package registry: %w", err)
|
||||
}
|
||||
var told struct {
|
||||
From string `json:"from"`
|
||||
At string `json:"at"`
|
||||
As string `json:"as"`
|
||||
Serves map[string]any `json:"serves"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &told); err != nil {
|
||||
return builder.Npmrc{}, fmt.Errorf("%s is not a binding: %w", path, err)
|
||||
}
|
||||
if told.At == "" {
|
||||
return builder.Npmrc{}, fmt.Errorf(
|
||||
"%s says the package registry is on %q and gives no address for it", path, told.From)
|
||||
}
|
||||
// Composed from what the provider serves, so nothing here knows gitea's URL shape from
|
||||
// another registry's: it states its port, the path its registry answers on, and the scheme.
|
||||
scheme := "https"
|
||||
if s, ok := told.Serves["scheme"]; ok {
|
||||
scheme = fmt.Sprintf("%v", s)
|
||||
}
|
||||
port, ok := told.Serves["port"]
|
||||
if !ok {
|
||||
return builder.Npmrc{}, fmt.Errorf("%s says nothing about which port the package registry answers on", path)
|
||||
}
|
||||
npmPath, ok := told.Serves["npm-path"]
|
||||
if !ok {
|
||||
return builder.Npmrc{}, fmt.Errorf("%s says nothing about the path the package registry answers on", path)
|
||||
}
|
||||
registry = fmt.Sprintf("%s://%s:%v%v", scheme, told.At, port, npmPath)
|
||||
username = told.As
|
||||
}
|
||||
|
||||
// The credential the mesh sealed to this machine. The mesh authenticates the ordinary way — a
|
||||
// generated password the provider only applies (novox/hq ADR 0048) — so with a username this is
|
||||
// a password (basic auth); without one it is a bearer token a provider minted.
|
||||
secret := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN"))
|
||||
if path := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN_FILE")); path != "" {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return builder.Npmrc{}, fmt.Errorf("cannot read this builder's package-registry credential: %w", err)
|
||||
}
|
||||
secret = strings.TrimSpace(string(raw))
|
||||
}
|
||||
n := builder.Npmrc{Scope: scope, Registry: fmt.Sprintf("%s://%s:%v%v", scheme, told.At, port, npmPath),
|
||||
From: builder.FromBinding, Machine: machine}
|
||||
if told.As != "" {
|
||||
n.Username, n.Password = told.As, secret
|
||||
} else {
|
||||
n.Token = secret
|
||||
if u := strings.TrimSpace(os.Getenv("MESH_NPM_USER")); u != "" {
|
||||
username = u
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// GenesisRegistry is the genesis document (novox/hq ADR 0336 §3): what the installer sealed to the
|
||||
// build agent on its node at genesis's packages step, as the `genesis-package-registry` own-secret —
|
||||
// the forge it raised, the port this node gave the package registry, its npm path, and the account it
|
||||
// made with that account's password. mesh-host writes it; the field names are the two repositories'
|
||||
// agreement.
|
||||
type GenesisRegistry struct {
|
||||
Scheme string `json:"scheme"`
|
||||
At string `json:"at"`
|
||||
Port int `json:"port"`
|
||||
NpmPath string `json:"npm-path"`
|
||||
As string `json:"as"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
|
||||
// fromGenesis composes the registry from the genesis document.
|
||||
func fromGenesis(path string, raw []byte, scope, machine string) (builder.Npmrc, error) {
|
||||
none := builder.Npmrc{Machine: machine}
|
||||
var g GenesisRegistry
|
||||
if err := json.Unmarshal(raw, &g); err != nil {
|
||||
return none, fmt.Errorf("%s is not the genesis document for the package registry: %w", path, err)
|
||||
if registry == "" && secret == "" {
|
||||
return builder.Npmrc{}, nil
|
||||
}
|
||||
var missing []string
|
||||
for name, value := range map[string]string{"at": g.At, "npm-path": g.NpmPath, "as": g.As, "password": g.Password} {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
missing = append(missing, name)
|
||||
}
|
||||
if username != "" {
|
||||
return builder.Npmrc{Scope: scope, Registry: registry, Username: username, Password: secret}, nil
|
||||
}
|
||||
if g.Port <= 0 {
|
||||
missing = append(missing, "port")
|
||||
}
|
||||
if len(missing) > 0 {
|
||||
sort.Strings(missing)
|
||||
return none, fmt.Errorf("%s, the genesis document for the package registry, says nothing of %s",
|
||||
path, strings.Join(missing, ", "))
|
||||
}
|
||||
scheme := g.Scheme
|
||||
if scheme == "" {
|
||||
scheme = "http"
|
||||
}
|
||||
return builder.Npmrc{Scope: scope, Registry: fmt.Sprintf("%s://%s:%d%s", scheme, g.At, g.Port, g.NpmPath),
|
||||
Username: g.As, Password: g.Password, From: builder.FromGenesis, Machine: machine}, nil
|
||||
}
|
||||
|
||||
// saysRegistry is the line each build's log carries about the package registry (novox/hq ADR 0336
|
||||
// §4): which of the binding and the genesis document it resolved from, or that it has neither here.
|
||||
func saysRegistry(n builder.Npmrc) string {
|
||||
if n.Enabled() {
|
||||
return "the package registry is resolved from " + n.From
|
||||
}
|
||||
machine := n.Machine
|
||||
if machine == "" {
|
||||
machine = "this machine"
|
||||
}
|
||||
return fmt.Sprintf("no package registry: %s is unbound on %s and no genesis document is here; "+
|
||||
"a build needing it is refused", builder.RegistryRequirement, machine)
|
||||
return builder.Npmrc{Scope: scope, Registry: registry, Token: secret}, nil
|
||||
}
|
||||
|
||||
// forgeFrom is the git credential this builder may offer a clone, composed from the same binding
|
||||
|
||||
@@ -84,15 +84,10 @@ func buildOnce(ctx context.Context, args []string) error {
|
||||
}
|
||||
fmt.Fprintln(os.Stderr)
|
||||
|
||||
machine := os.Getenv("MESH_NODE")
|
||||
if machine == "" {
|
||||
machine, _ = os.Hostname()
|
||||
}
|
||||
npmrc, err := packagesFrom(machine)
|
||||
npmrc, err := packagesFrom()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, " [packages] %s\n", saysRegistry(npmrc))
|
||||
built, buildErr := builder.Build(ctx, builder.Command, publisher, repository, *path, *ref, where, bases, npmrc,
|
||||
forgeFrom(),
|
||||
func(step, message string) { fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message) })
|
||||
|
||||
@@ -1,102 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
)
|
||||
|
||||
// The build agent's package registry (novox/hq ADR 0336, issue 121): its npm-package-registry
|
||||
// requirement is optional, so the binding may be absent, and on genesis's node the installer seals
|
||||
// the forge it raised as one genesis document instead.
|
||||
|
||||
// registryFiles names the three paths the build agent's environment always names, writing only those
|
||||
// given.
|
||||
func registryFiles(t *testing.T, binding, token, genesis string) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
for env, content := range map[string]string{
|
||||
"MESH_PACKAGE_BINDING": binding,
|
||||
"MESH_NPM_TOKEN_FILE": token,
|
||||
"MESH_GENESIS_PACKAGE_REGISTRY": genesis,
|
||||
} {
|
||||
path := filepath.Join(dir, env)
|
||||
t.Setenv(env, path)
|
||||
if content != "" {
|
||||
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, env := range []string{"MESH_NPM_REGISTRY", "MESH_NPM_TOKEN", "MESH_NPM_USER", "MESH_NPM_SCOPE"} {
|
||||
t.Setenv(env, "")
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
aBinding = `{"from":"gitea","at":"anchor.mesh","as":"anchor-agent",
|
||||
"serves":{"scheme":"http","port":3000,"npm-path":"/api/packages/novox/npm/"}}`
|
||||
aGenesisDocument = `{"scheme":"http","at":"127.0.0.1","port":3100,"npm-path":"/api/packages/novox/npm/",
|
||||
"as":"mesh-builder","password":"genesis-secret"}`
|
||||
)
|
||||
|
||||
func TestUntilTheRequirementBindsTheGenesisDocumentIsRead(t *testing.T) {
|
||||
registryFiles(t, "", "", aGenesisDocument)
|
||||
n, err := packagesFrom("anchor")
|
||||
if err != nil {
|
||||
t.Fatalf("an absent binding beside a genesis document was refused: %v", err)
|
||||
}
|
||||
if n.From != builder.FromGenesis || n.Registry != "http://127.0.0.1:3100/api/packages/novox/npm/" ||
|
||||
n.Username != "mesh-builder" || n.Password != "genesis-secret" || n.Scope != "@novox" {
|
||||
t.Fatalf("the genesis document was not what the registry came from: %+v", n)
|
||||
}
|
||||
file, err := n.File()
|
||||
if err != nil || !strings.Contains(file, "//127.0.0.1:3100/api/packages/novox/npm/:username=mesh-builder") {
|
||||
t.Fatalf("the .npmrc is not written from the genesis document: %v\n%s", err, file)
|
||||
}
|
||||
if got := saysRegistry(n); !strings.Contains(got, "genesis document") {
|
||||
t.Errorf("the log does not say the genesis document: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARealBindingWinsOverTheGenesisDocument(t *testing.T) {
|
||||
registryFiles(t, aBinding, "bound-secret\n", aGenesisDocument)
|
||||
n, err := packagesFrom("anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n.From != builder.FromBinding || n.Registry != "http://anchor.mesh:3000/api/packages/novox/npm/" ||
|
||||
n.Username != "anchor-agent" || n.Password != "bound-secret" {
|
||||
t.Fatalf("the binding did not win over the genesis document: %+v", n)
|
||||
}
|
||||
if got := saysRegistry(n); !strings.Contains(got, "binding") {
|
||||
t.Errorf("the log does not say the binding: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithNeitherTheAgentHasNoRegistryAndSaysWhere(t *testing.T) {
|
||||
registryFiles(t, "", "", "")
|
||||
n, err := packagesFrom("workstation")
|
||||
if err != nil {
|
||||
t.Fatalf("an unbound optional requirement was a fault: %v", err)
|
||||
}
|
||||
if n.Enabled() || n.Machine != "workstation" {
|
||||
t.Fatalf("a registry came from nowhere: %+v", n)
|
||||
}
|
||||
got := saysRegistry(n)
|
||||
for _, want := range []string{"npm-package-registry", "unbound", "workstation"} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("the log does not say %q: %s", want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAGenesisDocumentMissingAFieldIsRefusedByName(t *testing.T) {
|
||||
registryFiles(t, "", "", `{"at":"127.0.0.1","port":3000,"npm-path":"/api/packages/novox/npm/","as":"mesh-builder"}`)
|
||||
if _, err := packagesFrom("anchor"); err == nil || !strings.Contains(err.Error(), "password") {
|
||||
t.Fatalf("a genesis document with no password was taken: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// **The build agent says it is ready** (novox/hq issue 544). It listens on nothing, so no http or tcp
|
||||
// check can see it work; it writes a mark once it is bound to its worker and pulls build work, and its
|
||||
// container's health check runs `mesh-builder ready`, which passes only while the mark is there. Without
|
||||
// it the node-engine could judge the agent only alive, and every walk that moved it waited out the whole
|
||||
// grace on each machine.
|
||||
//
|
||||
// The mark lives in the container's own temporary directory, never in the workspace the machine keeps:
|
||||
// a new container starts without it, and a restarted one removes it before it binds again.
|
||||
func readyMark() string { return filepath.Join(os.TempDir(), "mesh-builder.ready") }
|
||||
|
||||
// notReady removes the mark: a holder starting is not taking work until it says so again.
|
||||
func notReady() error {
|
||||
if err := os.Remove(readyMark()); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("cannot remove the mark that this agent takes build work (%s): %w", readyMark(), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// taking writes the mark. A mark that cannot be written is said, and the agent goes on building: its
|
||||
// check then says it is not ready, which is the truth the gate should hear.
|
||||
func taking(seat string) {
|
||||
if err := os.WriteFile(readyMark(), []byte(seat+"\n"), 0o644); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "cannot mark that this agent takes build work (%s): %v\n", readyMark(), err)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "taking build work from %s\n", seat)
|
||||
}
|
||||
|
||||
// ready is the health check: nil when this container's agent is bound and pulling build work.
|
||||
func ready() error {
|
||||
if _, err := os.Stat(readyMark()); err != nil {
|
||||
return fmt.Errorf("not taking build work yet: %s is not there", readyMark())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The health check passes only between the agent saying it takes work and its next start (novox/hq
|
||||
// issue 544).
|
||||
func TestTheAgentIsReadyOnlyOnceItTakesWork(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
if err := ready(); err == nil {
|
||||
t.Fatal("an agent that never took work is ready")
|
||||
}
|
||||
taking("mesh-build-machine")
|
||||
if err := ready(); err != nil {
|
||||
t.Fatalf("an agent taking work is not ready: %v", err)
|
||||
}
|
||||
// Started again, it is not ready until it binds again.
|
||||
if err := notReady(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ready(); err == nil {
|
||||
t.Fatal("an agent starting again is ready before it takes work")
|
||||
}
|
||||
if err := notReady(); err != nil {
|
||||
t.Fatalf("removing a mark that is not there failed: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(readyMark()); err == nil {
|
||||
t.Fatal("the mark is still there")
|
||||
}
|
||||
}
|
||||
@@ -16,7 +16,6 @@ func TestBuilderDiagnosticsStayOffStdout(t *testing.T) {
|
||||
"version)": true, // --version
|
||||
`"stopping")`: true, // the loop.s shutdown line
|
||||
"usage)": true, // --help text, for a human
|
||||
"said)": true, // busy: the answer its node-engine reads (novox/hq ADR 0335)
|
||||
}
|
||||
for _, file := range []string{"once.go", "main.go"} {
|
||||
src, err := os.ReadFile(file)
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
)
|
||||
|
||||
// A try's outcome, as the build seat says it in the build's log (novox/hq issue 686): the repository's own
|
||||
// check's verdict, how long it took, and each line naming what failed.
|
||||
func TestATrysOutcomeNamesWhatFailed(t *testing.T) {
|
||||
tried := triedOf(builder.CheckVerdict{Verdict: "pass", Summary: "the change touches no module", Took: 303 * time.Second,
|
||||
Head: "c0ffee", Trunk: "ca3f43c", Gate: &builder.Layer{Verdict: "pass"},
|
||||
Repo: &builder.Layer{Verdict: "fail", Summary: "its merge-check.sh failed: TestX",
|
||||
Failed: "--- FAIL: TestX (0.01s)\n x_test.go:12: got 1, want 2"}})
|
||||
if tried.Verdict != "fail" || tried.Took != "5m3s" || len(tried.Failed) != 2 || tried.Trunk != "ca3f43c" ||
|
||||
tried.Summary != "its merge-check.sh failed: TestX" {
|
||||
t.Fatalf("a failed try: %+v", tried)
|
||||
}
|
||||
// A head that does not merge fails both layers: the check's own word.
|
||||
conflict := triedOf(builder.CheckVerdict{Verdict: "fail", Summary: "does not merge", Gate: &builder.Layer{Verdict: "fail"},
|
||||
Repo: &builder.Layer{Verdict: "fail", Summary: "does not merge"}})
|
||||
if conflict.Verdict != "fail" || conflict.Summary != "does not merge" {
|
||||
t.Fatalf("a try that does not merge: %+v", conflict)
|
||||
}
|
||||
// No verdict is an error, never a pass.
|
||||
if none := triedOf(builder.CheckVerdict{}); none.Verdict != "error" {
|
||||
t.Fatalf("a try with no verdict: %+v", none)
|
||||
}
|
||||
}
|
||||
@@ -1,145 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A provision whose credential only a person outside the mesh can give (novox/hq ADR 0337, issue 164, issue 165):
|
||||
// given once for its provider, at the controller's terminal, and sealed in one act to every consumer that binds the
|
||||
// provision from that provider now. The value is never read back, printed or kept.
|
||||
|
||||
// outsideOffer is the provider module's own name when `secret accept <node> <module> <name>` names a provision that
|
||||
// module offers with a credential issued outside the mesh, and "" when it names anything else (an own secret, or a
|
||||
// pair with --provider).
|
||||
func outsideOffer(ctx context.Context, module, name, provider string) (string, error) {
|
||||
if provider != "" {
|
||||
return "", nil
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer open.Close()
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if m, known := shelf[module]; known && m.IssuedOutsideOf(name) {
|
||||
return module, nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// offeredOutsideBy is a module that offers the provision with a credential issued outside the mesh, or "".
|
||||
func offeredOutsideBy(ctx context.Context, provision string) (string, error) {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer open.Close()
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
module, _ := catalogue.OffersIssuedOutside(shelf, provision)
|
||||
return module, nil
|
||||
}
|
||||
|
||||
// consumersFrom is every consumer that binds a provision from one provider now, found by resolving every machine as
|
||||
// its next send would, and the machines that could not be resolved, with why.
|
||||
func consumersFrom(ctx context.Context, open *stores, provider, providerModule, provision string) (
|
||||
[]inventory.PairEnd, []string, error) {
|
||||
nodes, err := open.inventory.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var out []inventory.PairEnd
|
||||
var unresolved []string
|
||||
for _, n := range nodes {
|
||||
plan, _, err := planFor(ctx, open, n.Name)
|
||||
if err != nil {
|
||||
unresolved = append(unresolved, fmt.Sprintf("%s: %s", n.Name, oneLine(err.Error())))
|
||||
continue
|
||||
}
|
||||
out = append(out, bindsFrom(plan, n.Name, provider, providerModule, provision)...)
|
||||
}
|
||||
return out, unresolved, nil
|
||||
}
|
||||
|
||||
// bindsFrom is every consumer on one machine whose resolution answers the provision from that provider. Pure.
|
||||
func bindsFrom(plan catalogue.Resolution, machine, provider, providerModule, provision string) []inventory.PairEnd {
|
||||
var out []inventory.PairEnd
|
||||
for _, need := range plan.Needs {
|
||||
if need.ByRecord || need.Name != provision || need.From != provider {
|
||||
continue
|
||||
}
|
||||
if need.Module != "" && need.Module != providerModule {
|
||||
continue
|
||||
}
|
||||
out = append(out, inventory.PairEnd{Consumer: machine, ConsumerModule: need.For, Local: need.Local})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
return out[i].ConsumerModule+"\x00"+out[i].Local < out[j].ConsumerModule+"\x00"+out[j].Local
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// acceptForProvision gives a provision's credential issued outside the mesh, once for its provider (ADR 0337 §3).
|
||||
func acceptForProvision(ctx context.Context, provider, providerModule, provision, value string) error {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
consumers, unresolved, err := consumersFrom(ctx, open, provider, providerModule, provision)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
accepted, err := open.inventory.AcceptSecretForProvision(ctx, provision, provider, providerModule, value, consumers)
|
||||
value = ""
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Print(acceptedSaid(provider, providerModule, provision, accepted, unresolved))
|
||||
if len(accepted.Sealed) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := sendTo(ctx, open, accepted.Machines); err != nil {
|
||||
return fmt.Errorf("%s is sealed and could not be sent: %w; `push %s` sends it", provision, err,
|
||||
strings.Join(accepted.Machines, "`, `push "))
|
||||
}
|
||||
fmt.Printf(" sent %s; each consumer starts again on it\n", strings.Join(accepted.Machines, ", "))
|
||||
return nil
|
||||
}
|
||||
|
||||
// acceptedSaid is what one accept for a provider did, in words that never carry the value. Pure.
|
||||
func acceptedSaid(provider, providerModule, provision string, a inventory.ProvisionAccepted, unresolved []string) string {
|
||||
var b strings.Builder
|
||||
if len(a.Sealed) == 0 {
|
||||
fmt.Fprintf(&b, "%s from %s on %s: no consumer binds it now, so nothing was sealed; a consumer that binds "+
|
||||
"later waits for it, and this line given again seals it to every consumer then\n", provision,
|
||||
providerModule, provider)
|
||||
} else {
|
||||
var names []string
|
||||
for _, c := range a.Sealed {
|
||||
names = append(names, fmt.Sprintf("%s on %s%s", c.ConsumerModule, c.Consumer, asLocal(c.Local)))
|
||||
}
|
||||
fmt.Fprintf(&b, "%s from %s on %s is given, sealed to %s and to %s\n", provision, providerModule, provider,
|
||||
strings.Join(names, ", "), provider)
|
||||
fmt.Fprintf(&b, " the mesh cannot read it back, never makes one of its own for it, and will not rotate it\n")
|
||||
}
|
||||
for _, c := range a.Unsealable {
|
||||
fmt.Fprintf(&b, " not sealed to %s on %s: no sealing key on that machine; give it again once it has one\n",
|
||||
c.ConsumerModule, c.Consumer)
|
||||
}
|
||||
for _, u := range unresolved {
|
||||
fmt.Fprintf(&b, " not known whether a consumer on %s binds it, which did not resolve; give it again once it does\n", u)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -26,11 +25,8 @@ import (
|
||||
//
|
||||
// - **The serving controller** takes it before it does anything else — before it asserts the bus's
|
||||
// objects, which are the controller's to write — waiting while another holds it, and renews it.
|
||||
// Another holder seen, or the key's age passed with no renewal made, is the lease lost: the gate
|
||||
// closes at once and the process exits, so its service manager restarts it as a candidate (serve,
|
||||
// in push.go). A renewal the bus does not answer in time is not a loss (novox/hq issue 477): the
|
||||
// lease is kept, no new act starts once half the key's age passed unrenewed, and the next renewal
|
||||
// made opens the gate again.
|
||||
// A renewal refused or failed is the lease lost: the gate closes at once and the process exits, so
|
||||
// its service manager restarts it as a candidate (serve, in push.go).
|
||||
// - **A command run at a shell** — `push` in the installer, the lab, a person repairing a mesh whose
|
||||
// controller is down (issue 201) — acts **under the holder's epoch** when a controller holds the
|
||||
// lease: it is the same mesh's word, composed and sent under the store's hold of each machine like
|
||||
@@ -67,11 +63,6 @@ type actor struct {
|
||||
// the highest epoch issued, and what was said of it; zero when it was not (S12).
|
||||
reset time.Time
|
||||
resetSaid string
|
||||
// renewEvery and poll, when set, are how often the serving controller renews the lease it holds and
|
||||
// looks again at one another holds, in place of the lease's own: a test's, so two controllers hand the
|
||||
// lease over in a second rather than in a renewal of five (novox/hq issue 591). A renewal is given
|
||||
// the lease's own RenewEvery to answer however often it is made, so a slow bus does not lose it.
|
||||
renewEvery, poll time.Duration
|
||||
}
|
||||
|
||||
// theLease is this process's standing to the lease.
|
||||
@@ -188,21 +179,7 @@ func (a *actor) release() {
|
||||
// holderOf is this process as the lease's holder.
|
||||
func holderOf(instance string) lease.Holder {
|
||||
host, _ := os.Hostname()
|
||||
build := runningBuild()
|
||||
if build == "" {
|
||||
build = version
|
||||
}
|
||||
return lease.Holder{Instance: instance, Host: host, Build: build}
|
||||
}
|
||||
|
||||
// RunningBuildVar is where the declaration tells this process which build it is (module.json, the
|
||||
// controller process's env): the version its bundle is delivered as, `${version}` composed by the
|
||||
// catalogue from the bundle's digest. Empty for a process placed by hand.
|
||||
const RunningBuildVar = "MESH_CONTROLLER_VERSION"
|
||||
|
||||
// runningBuild is the version of the build this process is, or empty when the declaration did not say.
|
||||
func runningBuild() string {
|
||||
return strings.TrimSpace(os.Getenv(RunningBuildVar))
|
||||
return lease.Holder{Instance: instance, Host: host, Build: version}
|
||||
}
|
||||
|
||||
// serveUnderTheLease takes the lease for the serving controller, waiting while another holds it, and
|
||||
@@ -231,7 +208,7 @@ func (a *actor) serveUnderTheLease(ctx context.Context, inv *inventory.Inventory
|
||||
}
|
||||
say := func(format string, args ...any) { fmt.Printf(format+"\n", args...) }
|
||||
l, err := lease.Open(ctx, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance),
|
||||
RenewEvery: a.renewEvery, RenewWithin: lease.RenewEvery, Poll: a.poll, Floor: inv.HighestEpoch, Say: say, Health: controllerHealth, Moved: func(was, floor uint64) {
|
||||
Floor: inv.HighestEpoch, Say: say, Health: controllerHealth, Moved: func(was, floor uint64) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
a.reset = time.Now()
|
||||
|
||||
+21
-98
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"io"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -52,12 +51,6 @@ func assignWith(ctx context.Context, open *stores, node string, opts assignOptio
|
||||
if len(modules) == 0 {
|
||||
return "", fmt.Errorf("assign %s names no module", node)
|
||||
}
|
||||
if err := refusedMovingTheController(modules); err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Every line of the answer comes from the one resolution after the write (novox/hq issue 237):
|
||||
// what the reads before it would say about a machine left out is not the answer's.
|
||||
ctx = leftOutSaidBy(ctx, io.Discard)
|
||||
// Held while it is recorded, so it cannot land between a converge's preview and its flip and
|
||||
// be taken without ever having been previewed (novox/hq ADR 0100).
|
||||
ctx, release, err := holdNodes(ctx, open, []string{node})
|
||||
@@ -129,28 +122,22 @@ func assignWith(ctx context.Context, open *stores, node string, opts assignOptio
|
||||
for _, line := range settled {
|
||||
answer += "\n " + line
|
||||
}
|
||||
// What this act changed about this node's unmet seat dependencies, and nothing else (novox/hq
|
||||
// ADR 0207): a dependency of a module just assigned, or one this assignment met. The rest of the
|
||||
// node's list, and every other node's, is `status`'s.
|
||||
for _, line := range unheldChange(shelf, node, before, append(append([]string(nil), before...), added...)) {
|
||||
answer += "\n " + line
|
||||
}
|
||||
// Its bus credential, in the same act (novox/hq issue 203): an assignment pushed before its
|
||||
// credential exists delivers a process that cannot authenticate and crash-loops until somebody
|
||||
// runs a second verb and a second push. Issued here when the module speaks on the bus and has
|
||||
// no credential yet; kept when it has one, so re-assigning rotates nothing.
|
||||
var issued []string
|
||||
for _, module := range added {
|
||||
if line := issueOnAssign(ctx, open, node, module); line != "" {
|
||||
issued = append(issued, line)
|
||||
answer += "\n " + line
|
||||
}
|
||||
}
|
||||
// **The one resolution the rest of the answer is said from** (novox/hq issue 237), taken after
|
||||
// the write: what this act changed about this node's unmet seat dependencies, and nothing else
|
||||
// (novox/hq ADR 0207) — a dependency of a module just assigned, or one this assignment met; the
|
||||
// rest of the node's list, and every other node's, is `status`'s — and whether the node resolves.
|
||||
after := resolvedAfter(ctx, open, shelf, node)
|
||||
for _, line := range unheldChangeTo(shelf, node, before, after.modules, after.unheld) {
|
||||
answer += "\n " + line
|
||||
}
|
||||
for _, line := range issued {
|
||||
answer += "\n " + line
|
||||
}
|
||||
plan, err := after.plan, after.err
|
||||
plan, _, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
// Kept, and still refused. Both halves are the answer, and the rest of the mesh is still
|
||||
// worth reporting: this machine's refusal is rarely the only consequence.
|
||||
@@ -203,9 +190,8 @@ func seatDependenciesOnAssign(ctx context.Context, open *stores, node string, mo
|
||||
if _, err := catalogue.AssignRefusal(shelf, node, assigned, adding); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
// Two modules declaring one path or unit, or one package present and absent, is refused before anything
|
||||
// is recorded (novox/hq ADR 0210, ADR 0303, 04-ISSUES/235): kept, the node would not resolve until one came
|
||||
// off again. Two modules declaring one package present is not a collision (ADR 0303, issue 628).
|
||||
// Two modules declaring one package, path or unit is refused before anything is recorded
|
||||
// (novox/hq ADR 0210, 04-ISSUES/235): kept, the node would not resolve until one came off again.
|
||||
if err := catalogue.CollisionRefusal(shelf, node, assigned, adding); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
@@ -223,14 +209,9 @@ func seatDependenciesOnAssign(ctx context.Context, open *stores, node string, mo
|
||||
// (novox/hq ADR 0207) — the other side of refusing that module's assignment without one. Several
|
||||
// modules in one act are judged together, so a holder and its dependents come off in one command.
|
||||
func unassign(ctx context.Context, open *stores, node string, modules ...string) (string, error) {
|
||||
if err := refusedMovingTheController(modules); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(modules) == 0 {
|
||||
return "", fmt.Errorf("unassign %s names no module", node)
|
||||
}
|
||||
// One resolution after the write, as assign's answer (novox/hq issue 237).
|
||||
ctx = leftOutSaidBy(ctx, io.Discard)
|
||||
ctx, release, err := holdNodes(ctx, open, []string{node})
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -275,21 +256,14 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string)
|
||||
}
|
||||
answer := fmt.Sprintf("%s no longer runs %s — run `push %s` to make it so",
|
||||
node, strings.Join(modules, ", "), node)
|
||||
after := resolvedAfter(ctx, open, shelf, node)
|
||||
for _, line := range unheldChangeTo(shelf, node, assigned, after.modules, after.unheld) {
|
||||
answer += "\n " + line
|
||||
var left []string
|
||||
for _, a := range assigned {
|
||||
if !slices.Contains(modules, a) {
|
||||
left = append(left, a)
|
||||
}
|
||||
}
|
||||
// Whether the node itself still resolves, from the same resolution: before, only the rest of the
|
||||
// mesh's reads said it, on standard error and once per other machine.
|
||||
if after.err != nil {
|
||||
if unresolvable(after.err) {
|
||||
answer += fmt.Sprintf("\n but %s does not resolve now, so nothing will be sent to it:", node)
|
||||
} else {
|
||||
answer += fmt.Sprintf("\n but whether %s still resolves could not be read:", node)
|
||||
}
|
||||
for _, line := range strings.Split(strings.TrimRight(after.err.Error(), "\n"), "\n") {
|
||||
answer += "\n " + strings.TrimSpace(line)
|
||||
}
|
||||
for _, line := range unheldChange(shelf, node, assigned, left) {
|
||||
answer += "\n " + line
|
||||
}
|
||||
// What it leaves behind that is irreplaceable is kept and retired, never removed (novox/hq ADR 0233).
|
||||
for _, line := range keptOnUnassign(ctx, open.inventory, node, modules) {
|
||||
@@ -298,39 +272,6 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string)
|
||||
return answer + blockedElsewhere(ctx, open, node), nil
|
||||
}
|
||||
|
||||
// afterAct is the one resolution of a node an act's answer is said from, taken after its write
|
||||
// (novox/hq issue 237): the plan or why there is none, the modules it has on the node, and its unmet
|
||||
// seat dependencies.
|
||||
type afterAct struct {
|
||||
plan catalogue.Resolution
|
||||
err error
|
||||
modules []string
|
||||
unheld []catalogue.Unheld
|
||||
}
|
||||
|
||||
// resolvedAfter resolves the node as `plan` does, once, after the act's write. When the node resolves,
|
||||
// its modules and unmet seat dependencies are the resolution's — the closure, so a holder pulled in by
|
||||
// a requirement counts as `plan` counts it. When it does not, they are judged over the assignments read
|
||||
// after the write, which can only find more unmet than the closure would: a seat said held there is
|
||||
// held in the closure too.
|
||||
func resolvedAfter(ctx context.Context, open *stores, shelf map[string]catalogue.Manifest, node string) afterAct {
|
||||
plan, _, err := planFor(ctx, open, node)
|
||||
if err == nil {
|
||||
out := afterAct{plan: plan, unheld: plan.Unheld}
|
||||
for _, m := range plan.Modules {
|
||||
out.modules = append(out.modules, m.Module)
|
||||
}
|
||||
return out
|
||||
}
|
||||
out := afterAct{err: err}
|
||||
assigned, readErr := open.inventory.Assigned(ctx, node)
|
||||
if readErr != nil {
|
||||
return out
|
||||
}
|
||||
out.modules, out.unheld = assigned, unheldOver(shelf, node, assigned)
|
||||
return out
|
||||
}
|
||||
|
||||
// splitModules is a surface's one `module` field as the modules it names: several, comma-separated,
|
||||
// are one act (novox/hq ADR 0207), so the holders that depend on each other go on together from the
|
||||
// command API and the controller seat's verbs as they do from the command line.
|
||||
@@ -391,10 +332,10 @@ func blockedElsewhere(ctx context.Context, open *stores, except string) string {
|
||||
}
|
||||
|
||||
// issueOnAssign gives a newly assigned module its bus credential, the way `module issue` does, and
|
||||
// says what it did in one line. Nothing for a module that declares no broker secret and runs no
|
||||
// provisioner client; nothing for one whose user is already minted (a credential is rotated on
|
||||
// purpose, never by re-assigning); and when the bus cannot be reached from here, the line names the
|
||||
// verb and the push that would refuse the module until it is run — never a silent placeholder (novox/hq issue 203).
|
||||
// says what it did in one line. Nothing for a module that declares no broker secret; nothing for one
|
||||
// whose user is already minted (a credential is rotated on purpose, never by re-assigning); and when
|
||||
// the bus cannot be reached from here, the line names the verb and the push that would refuse the
|
||||
// module until it is run — never a silent placeholder (novox/hq issue 203).
|
||||
func issueOnAssign(ctx context.Context, open *stores, node, module string) string {
|
||||
inv := open.inventory
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
@@ -406,10 +347,6 @@ func issueOnAssign(ctx context.Context, open *stores, node, module string) strin
|
||||
return ""
|
||||
}
|
||||
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username()
|
||||
if _, reads := m.OwnSecrets["broker"]; !reads {
|
||||
// A module running the provisioner client alone is minted the client's account (issue 917).
|
||||
user = broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: module}.Username()
|
||||
}
|
||||
if _, minted, err := inv.BusUserHash(ctx, user); err != nil || minted {
|
||||
return ""
|
||||
}
|
||||
@@ -423,17 +360,3 @@ func issueOnAssign(ctx context.Context, open *stores, node, module string) strin
|
||||
}
|
||||
return fmt.Sprintf("its bus credential is issued and sealed to %s, and arrives with the push", node)
|
||||
}
|
||||
|
||||
// refusedMovingTheController refuses assigning or unassigning the controller's module through a verb (review of
|
||||
// novox/hq ADR 0272): the node it is assigned to is the control-node, and the control-node's operator account is
|
||||
// the controller's terminal, so whoever moves the module chooses the terminal. At the terminal alone, as every
|
||||
// change that says who may change what.
|
||||
func refusedMovingTheController(modules []string) error {
|
||||
verb, through := throughAVerb()
|
||||
if !through || !slices.Contains(modules, controllerModule) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%s is assigned and unassigned at the controller's terminal only (mesh-cli on the control-node), "+
|
||||
"never through a verb (this came through %q): the node it runs on is the control-node, whose operator is the "+
|
||||
"terminal (novox/hq ADR 0272). Nothing was assigned", controllerModule, verb)
|
||||
}
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
|
||||
@@ -1,228 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/licences"
|
||||
)
|
||||
|
||||
func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aStore())
|
||||
if _, err := assign(ctx, open, "anchor", "distribution"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "distribution",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5101}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A module the mesh built, recorded by digest and path, running on the other machine.
|
||||
if err := open.inventory.RecordBuild(ctx, inventory.Build{
|
||||
ID: "b1", Repository: "r", Module: "app", Commit: "abc",
|
||||
Made: []inventory.Artifact{{Name: "server", Kind: "image",
|
||||
Reference: catalogue.ArtifactStoreScheme + "app/server@" + aDigest}},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "app", Version: "1",
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-app",
|
||||
"image": catalogue.ArtifactStoreScheme + "app/server@" + aDigest}}})
|
||||
if _, err := assign(ctx, open, "laptop", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The runtime's trust is the runtime's module's to write (novox/hq ADR 0222): a stand-in for it
|
||||
// asks where this machine reaches the store, as the docker module does.
|
||||
register(t, open, catalogue.Manifest{Module: "runtime", Version: "1",
|
||||
Resources: []map[string]any{{"id": "daemon", "type": "file", "path": "/etc/docker/daemon.json",
|
||||
"into": "json", "content": `{"insecure-registries": ["${seat:mesh-artifact-store:reach}"]}` + "\n"}}})
|
||||
if _, err := assign(ctx, open, "laptop", "runtime"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
on := map[string]bool{"anchor": true, "laptop": true}
|
||||
node, port, found, err := artifactStoreOnNetwork(ctx, open.inventory, on)
|
||||
if err != nil || !found || node != "anchor" || port != "5101" {
|
||||
t.Fatalf("the store is found on %q:%q (%v, %v); the node put it on 5101", node, port, found, err)
|
||||
}
|
||||
|
||||
var trust string
|
||||
for _, r := range composed(t, open, "laptop").Resources {
|
||||
if r["path"] == "/etc/docker/daemon.json" {
|
||||
trust, _ = r["content"].(string)
|
||||
}
|
||||
if r["id"] == "app.server" && r["image"] != "anchor.internal:5101/app/server@"+aDigest {
|
||||
t.Errorf("the image the mesh built is fetched as %v", r["image"])
|
||||
}
|
||||
}
|
||||
if !strings.Contains(trust, "anchor.internal:5101") || strings.Contains(trust, ":5000") {
|
||||
t.Fatalf("the runtime is told to trust %q; the node put the store on 5101", trust)
|
||||
}
|
||||
|
||||
// And a replay to the catalogue says where the store is now.
|
||||
announced, err := following{open: open}.Announceable(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(announced) != 1 || announced[0].Made[0].Reference != "anchor.internal:5101/app/server@"+aDigest {
|
||||
t.Fatalf("the replay announces %+v", announced)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
control, err := withSeatPorts(m).Resolve([]catalogue.Built{{Name: "controller", Kind: catalogue.ArtifactBundle,
|
||||
Reference: "https://registry.example/mesh-controller/controller.tar.gz", Digest: aDigest}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, control)
|
||||
register(t, open, catalogue.Manifest{Module: "postgres", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}},
|
||||
Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store",
|
||||
"ports": []any{"5432:5432"}, "image": "pg@" + aDigest}}})
|
||||
register(t, open, catalogue.Manifest{Module: "lavinmq", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}},
|
||||
Listens: []catalogue.Listening{{Port: 5671, From: catalogue.FromMesh},
|
||||
{Port: 5672, From: catalogue.FromMesh}},
|
||||
Guards: []int{15672},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-broker",
|
||||
"ports": []any{"5671:5671", "5672:5672", "127.0.0.1:15672:15672"}, "image": "mq@" + aDigest}}})
|
||||
// The control plane's own bus user is the installer's, seeded at genesis before the controller
|
||||
// runs (SeedBusUser); without it a push now refuses the credential nobody issued (issue 203).
|
||||
if err := open.inventory.SeedBusUser(ctx, inventory.BusUser{Username: "anchor.mesh-controller",
|
||||
Kind: inventory.BusController, Node: "anchor", Module: "mesh-controller"}, "bootstrap"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "anchor", "mesh-controller"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The store's module is registered and given its port, and NOT assigned: the state genesis
|
||||
// leaves a given-port node in before the foundation is adopted as modules (04-ISSUES/085).
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "postgres",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5432": 6852}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "anchor", "lavinmq"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "lavinmq",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5672": 5679}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var env map[string]any
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
if r["id"] == "mesh-controller.controller" {
|
||||
env, _ = r["env"].(map[string]any)
|
||||
}
|
||||
}
|
||||
if env == nil {
|
||||
t.Fatal("the control plane's process is not in its own node's declaration")
|
||||
}
|
||||
for key, want := range map[string]string{
|
||||
"MESH_STORE_INVENTORY_PORT": "6852",
|
||||
"MESH_STORE_IDENTITY_PORT": "6852",
|
||||
"MESH_STORE_LICENCES_PORT": "6852",
|
||||
"MESH_BROKER_AMQP_PORT": "5679",
|
||||
// Neither given nor assigned by the mesh: the manifest's own number is NOT the answer,
|
||||
// because the sealed value beside it carries the port genesis wrote (finding F3).
|
||||
"MESH_BROKER_ADDRESS_PORT": "",
|
||||
"MESH_BROKER_MANAGEMENT_PORT": "",
|
||||
} {
|
||||
if env[key] != want {
|
||||
t.Errorf("the control plane is told %s=%v; the node says %q", key, env[key], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func aLoneNode(t *testing.T) *stores {
|
||||
t.Helper()
|
||||
inventory.ForTest(t)
|
||||
licences.ForTest(t)
|
||||
open, err := openStores(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(open.Close)
|
||||
for _, m := range provided {
|
||||
if err := open.inventory.Provide(t.Context(), m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
record, err := open.inventory.AddNode(t.Context(), "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reported, _ := json.Marshal(map[string]any{"capabilities": []map[string]any{
|
||||
{"name": "container-runtime", "present": true}}})
|
||||
var profile map[string]any
|
||||
_ = json.Unmarshal(reported, &profile)
|
||||
if err := open.inventory.RecordProfile(t.Context(), record.ID, profile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSealingKey(t.Context(), record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return open
|
||||
}
|
||||
|
||||
func TestOnANodeWithNoNetworkTheStoreIsReachedByLoopback(t *testing.T) {
|
||||
open := aLoneNode(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aStore())
|
||||
register(t, open, catalogue.Manifest{Module: "builder", Version: "1",
|
||||
Requires: []string{catalogue.ArtifactStoreProvision},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-builder",
|
||||
"image": "registry.example/mesh-builder@" + aDigest}}})
|
||||
if err := open.inventory.RecordBuild(ctx, inventory.Build{
|
||||
ID: "b1", Repository: "r", Module: "postgres", Commit: "abc",
|
||||
Made: []inventory.Artifact{{Name: "runtime", Kind: "image",
|
||||
Reference: catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "postgres", Version: "1",
|
||||
Resources: []map[string]any{{"id": "runtime", "type": "container", "name": "mesh-postgres",
|
||||
"image": catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}}})
|
||||
for _, module := range []string{"distribution", "builder", "postgres"} {
|
||||
if _, err := assign(ctx, open, "anchor", module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "distribution",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5100}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var image any
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
if r["id"] == "postgres.runtime" {
|
||||
image = r["image"]
|
||||
}
|
||||
}
|
||||
if image != "127.0.0.1:5100/postgres/runtime@"+aDigest {
|
||||
t.Fatalf("on the store's own node, off any network, the image is fetched as %v", image)
|
||||
}
|
||||
held := heldBy(ctx)
|
||||
if got := held["postgres/runtime"]; got != "127.0.0.1:5100/postgres/runtime@"+aDigest {
|
||||
t.Fatalf("a builder beside the store is handed the base %q", got)
|
||||
}
|
||||
}
|
||||
@@ -2,10 +2,13 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/licences"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
@@ -83,12 +86,150 @@ func aStore() catalogue.Manifest {
|
||||
|
||||
// **The trust a machine writes for the store, and the address every built image is fetched
|
||||
// through, say the port the node gave the store** — not the catalogue's number.
|
||||
func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aStore())
|
||||
if _, err := assign(ctx, open, "anchor", "distribution"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "distribution",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5101}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A module the mesh built, recorded by digest and path, running on the other machine.
|
||||
if err := open.inventory.RecordBuild(ctx, inventory.Build{
|
||||
ID: "b1", Repository: "r", Module: "app", Commit: "abc",
|
||||
Made: []inventory.Artifact{{Name: "server", Kind: "image",
|
||||
Reference: catalogue.ArtifactStoreScheme + "app/server@" + aDigest}},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "app", Version: "1",
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-app",
|
||||
"image": catalogue.ArtifactStoreScheme + "app/server@" + aDigest}}})
|
||||
if _, err := assign(ctx, open, "laptop", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The runtime's trust is the runtime's module's to write (novox/hq ADR 0222): a stand-in for it
|
||||
// asks where this machine reaches the store, as the docker module does.
|
||||
register(t, open, catalogue.Manifest{Module: "runtime", Version: "1",
|
||||
Resources: []map[string]any{{"id": "daemon", "type": "file", "path": "/etc/docker/daemon.json",
|
||||
"into": "json", "content": `{"insecure-registries": ["${seat:mesh-artifact-store:reach}"]}` + "\n"}}})
|
||||
if _, err := assign(ctx, open, "laptop", "runtime"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
on := map[string]bool{"anchor": true, "laptop": true}
|
||||
node, port, found, err := artifactStoreOnNetwork(ctx, open.inventory, on)
|
||||
if err != nil || !found || node != "anchor" || port != "5101" {
|
||||
t.Fatalf("the store is found on %q:%q (%v, %v); the node put it on 5101", node, port, found, err)
|
||||
}
|
||||
|
||||
var trust string
|
||||
for _, r := range composed(t, open, "laptop").Resources {
|
||||
if r["path"] == "/etc/docker/daemon.json" {
|
||||
trust, _ = r["content"].(string)
|
||||
}
|
||||
if r["id"] == "app.server" && r["image"] != "anchor.internal:5101/app/server@"+aDigest {
|
||||
t.Errorf("the image the mesh built is fetched as %v", r["image"])
|
||||
}
|
||||
}
|
||||
if !strings.Contains(trust, "anchor.internal:5101") || strings.Contains(trust, ":5000") {
|
||||
t.Fatalf("the runtime is told to trust %q; the node put the store on 5101", trust)
|
||||
}
|
||||
|
||||
// And a replay to the catalogue says where the store is now.
|
||||
announced, err := following{open}.Announceable(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(announced) != 1 || announced[0].Made[0].Reference != "anchor.internal:5101/app/server@"+aDigest {
|
||||
t.Fatalf("the replay announces %+v", announced)
|
||||
}
|
||||
}
|
||||
|
||||
// **The control plane's own connections say the port the node gave the store and the broker.**
|
||||
//
|
||||
// Composed from the control plane's own manifest against a real inventory: the store's module is
|
||||
// given 6852 on this node the way genesis or an operator gives it, and the control plane's
|
||||
// process is told so beside the sealed connection genesis wrote.
|
||||
func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
control, err := withSeatPorts(m).Resolve([]catalogue.Built{{Name: "controller", Kind: catalogue.ArtifactBundle,
|
||||
Reference: "https://registry.example/mesh-controller/controller.tar.gz", Digest: aDigest}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, control)
|
||||
register(t, open, catalogue.Manifest{Module: "postgres", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}},
|
||||
Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store",
|
||||
"ports": []any{"5432:5432"}, "image": "pg@" + aDigest}}})
|
||||
register(t, open, catalogue.Manifest{Module: "lavinmq", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}},
|
||||
Listens: []catalogue.Listening{{Port: 5671, From: catalogue.FromMesh},
|
||||
{Port: 5672, From: catalogue.FromMesh}},
|
||||
Guards: []int{15672},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-broker",
|
||||
"ports": []any{"5671:5671", "5672:5672", "127.0.0.1:15672:15672"}, "image": "mq@" + aDigest}}})
|
||||
// The control plane's own bus user is the installer's, seeded at genesis before the controller
|
||||
// runs (SeedBusUser); without it a push now refuses the credential nobody issued (issue 203).
|
||||
if err := open.inventory.SeedBusUser(ctx, inventory.BusUser{Username: "anchor.mesh-controller",
|
||||
Kind: inventory.BusController, Node: "anchor", Module: "mesh-controller"}, "bootstrap"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "anchor", "mesh-controller"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The store's module is registered and given its port, and NOT assigned: the state genesis
|
||||
// leaves a given-port node in before the foundation is adopted as modules (04-ISSUES/085).
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "postgres",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5432": 6852}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "anchor", "lavinmq"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "lavinmq",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5672": 5679}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var env map[string]any
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
if r["id"] == "mesh-controller.controller" {
|
||||
env, _ = r["env"].(map[string]any)
|
||||
}
|
||||
}
|
||||
if env == nil {
|
||||
t.Fatal("the control plane's process is not in its own node's declaration")
|
||||
}
|
||||
for key, want := range map[string]string{
|
||||
"MESH_STORE_INVENTORY_PORT": "6852",
|
||||
"MESH_STORE_IDENTITY_PORT": "6852",
|
||||
"MESH_STORE_LICENCES_PORT": "6852",
|
||||
"MESH_BROKER_AMQP_PORT": "5679",
|
||||
// Neither given nor assigned by the mesh: the manifest's own number is NOT the answer,
|
||||
// because the sealed value beside it carries the port genesis wrote (finding F3).
|
||||
"MESH_BROKER_ADDRESS_PORT": "",
|
||||
"MESH_BROKER_MANAGEMENT_PORT": "",
|
||||
} {
|
||||
if env[key] != want {
|
||||
t.Errorf("the control plane is told %s=%v; the node says %q", key, env[key], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// withSeatPorts is the control plane's manifest with the seat placeholders in its environment —
|
||||
// added here until module.json carries them (the manifest lands one commit after the code that
|
||||
@@ -133,7 +274,79 @@ func withSeatPorts(m catalogue.Manifest) catalogue.Manifest {
|
||||
// aLoneNode is one capable machine with nothing placed on any network — the control-node during
|
||||
// genesis, before the "network" step, which is after the store, the broker, the vault and the
|
||||
// catalogue have each been built and pushed (finding F2).
|
||||
func aLoneNode(t *testing.T) *stores {
|
||||
t.Helper()
|
||||
inventory.ForTest(t)
|
||||
licences.ForTest(t)
|
||||
open, err := openStores(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(open.Close)
|
||||
for _, m := range provided {
|
||||
if err := open.inventory.Provide(t.Context(), m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
record, err := open.inventory.AddNode(t.Context(), "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reported, _ := json.Marshal(map[string]any{"capabilities": []map[string]any{
|
||||
{"name": "container-runtime", "present": true}}})
|
||||
var profile map[string]any
|
||||
_ = json.Unmarshal(reported, &profile)
|
||||
if err := open.inventory.RecordProfile(t.Context(), record.ID, profile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSealingKey(t.Context(), record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return open
|
||||
}
|
||||
|
||||
// **Before the network exists, the store's own node reaches it by loopback** — never refused,
|
||||
// never handed the scheme: a genesis pushes the store, the broker, the vault and the catalogue to
|
||||
// a node on no network, and builds the catalogue on a base it must be able to pull.
|
||||
func TestOnANodeWithNoNetworkTheStoreIsReachedByLoopback(t *testing.T) {
|
||||
open := aLoneNode(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aStore())
|
||||
register(t, open, catalogue.Manifest{Module: "builder", Version: "1",
|
||||
Requires: []string{catalogue.ArtifactStoreProvision},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-builder",
|
||||
"image": "registry.example/mesh-builder@" + aDigest}}})
|
||||
if err := open.inventory.RecordBuild(ctx, inventory.Build{
|
||||
ID: "b1", Repository: "r", Module: "postgres", Commit: "abc",
|
||||
Made: []inventory.Artifact{{Name: "runtime", Kind: "image",
|
||||
Reference: catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "postgres", Version: "1",
|
||||
Resources: []map[string]any{{"id": "runtime", "type": "container", "name": "mesh-postgres",
|
||||
"image": catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}}})
|
||||
for _, module := range []string{"distribution", "builder", "postgres"} {
|
||||
if _, err := assign(ctx, open, "anchor", module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "distribution",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5100}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var image any
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
if r["id"] == "postgres.runtime" {
|
||||
image = r["image"]
|
||||
}
|
||||
}
|
||||
if image != "127.0.0.1:5100/postgres/runtime@"+aDigest {
|
||||
t.Fatalf("on the store's own node, off any network, the image is fetched as %v", image)
|
||||
}
|
||||
held := heldBy(ctx)
|
||||
if got := held["postgres/runtime"]; got != "127.0.0.1:5100/postgres/runtime@"+aDigest {
|
||||
t.Fatalf("a builder beside the store is handed the base %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
|
||||
@@ -408,9 +408,6 @@ func comparisonOf(module string, c comparison, opts takeOptions) (preview string
|
||||
switch {
|
||||
case sec.Origin == inventory.OriginAccepted:
|
||||
fmt.Fprintf(&b, " %s %s: accepted from a person, carried in as it is\n", what, name)
|
||||
case sec.Origin == inventory.OriginIssued:
|
||||
// Taken from the provider's software (novox/hq ADR 0305): the value the running service uses.
|
||||
fmt.Fprintf(&b, " %s %s: issued by the provider's software, carried in as it is\n", what, name)
|
||||
case opts.Mint[sec.Name]:
|
||||
fmt.Fprintf(&b, " %s %s: minted by the mesh; the service takes the new value, as --mint said\n", what, name)
|
||||
case foundData:
|
||||
@@ -424,9 +421,8 @@ func comparisonOf(module string, c comparison, opts takeOptions) (preview string
|
||||
}
|
||||
// And its settings on this machine, composed against its definition (rule 1, rule 6).
|
||||
for _, layer := range c.layers {
|
||||
said := layer.Whole()
|
||||
keys := make([]string, 0, len(said))
|
||||
for k := range said {
|
||||
keys := make([]string, 0, len(layer.Values))
|
||||
for k := range layer.Values {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
@@ -723,7 +719,7 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s
|
||||
return "", err
|
||||
}
|
||||
derived := derivedFilter{rules: rules, foundation: with.Foundation, mesh: with.Mesh,
|
||||
outward: plan.PublicDomain != "", outwardLinks: with.OutwardLinks, local: with.LocalNetworks}
|
||||
outward: plan.PublicDomain != "", outwardLinks: with.OutwardLinks}
|
||||
filtering, err := inv.FilteringOf(ctx, node)
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -931,9 +927,6 @@ type derivedFilter struct {
|
||||
// The filter constrains what arrives on them; everything arriving elsewhere is this machine's
|
||||
// own guest and is not filtered.
|
||||
outwardLinks []string
|
||||
// local is the machine's local networks, which a port from `local-network` stays open to (novox/hq
|
||||
// ADR 0340).
|
||||
local []string
|
||||
}
|
||||
|
||||
// closesOutside is what a narrowing from everywhere to the private network is called: it closes.
|
||||
@@ -981,17 +974,6 @@ func (d derivedFilter) fate(r inventory.Reach) string {
|
||||
if !onMesh {
|
||||
return fmt.Sprintf("%s — declared by %s from the mesh only", closesOutside, by)
|
||||
}
|
||||
case catalogue.FromLocalNetwork:
|
||||
// The private network and this machine's local networks (novox/hq ADR 0340): it closes to
|
||||
// everything else outside, and says which local networks stay.
|
||||
if !onMesh {
|
||||
if len(d.local) == 0 {
|
||||
return fmt.Sprintf("%s — declared by %s from the local network, and this machine "+
|
||||
"reported none", closesOutside, by)
|
||||
}
|
||||
return fmt.Sprintf("WILL CLOSE to everything outside the private network and %s — declared "+
|
||||
"by %s from the local network", strings.Join(d.local, ", "), by)
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("declared by %s (from %s)", by, rule.From)
|
||||
}
|
||||
|
||||
@@ -1,180 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A refusal the bus said once is counted once** (novox/hq issue 402). The bus refused the controller
|
||||
// one publish at 18:52:59 UTC on 2026-10-10 and never again, yet S9 looked at the one remembered
|
||||
// advisory every half minute for the hour it is kept, and the condition said "observed 15 time(s), last
|
||||
// 13s ago" with a refusal in its evidence every 30 seconds: two of us read it as a refusal going on and
|
||||
// went looking for a missing grant. The condition counts what the bus said, with when it last said it,
|
||||
// never how often the controller looked.
|
||||
func TestARefusalSaidOnceIsCountedOnceHoweverOftenItIsLookedAt(t *testing.T) {
|
||||
refused := time.Date(2026, 10, 10, 18, 52, 59, 0, time.UTC)
|
||||
now := refused.Add(10 * time.Second)
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
||||
t.Cleanup(func() { k.Close(t.Context()) })
|
||||
|
||||
// The words the live condition bus.controller.refused carried on 2026-10-10.
|
||||
said := "the bus refused the controller: nats: permissions violation: Permissions Violation for " +
|
||||
`Publish to "mesh.seat.mesh-delivery.tool.times"`
|
||||
advisory := link.Advisory{Kind: link.AdvisoryRefused, ID: "controller", Said: said,
|
||||
First: refused, Last: refused, Count: 1}
|
||||
look := func() conditions.Condition {
|
||||
t.Helper()
|
||||
f := &signalFacts{now: now, host: "novox", advisories: []link.Advisory{advisory}}
|
||||
if err := k.Reconcile(t.Context(), "S9", watchAdvisories(f)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, found, err := conditions.ReadOne(t.Context(), store, "bus.controller.refused")
|
||||
if err != nil || !found {
|
||||
t.Fatalf("bus.controller.refused: found %v, %v", found, err)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
var c conditions.Condition
|
||||
for range 15 {
|
||||
c = look()
|
||||
now = now.Add(30 * time.Second)
|
||||
}
|
||||
if c.Observations != 1 || len(c.Evidence) != 1 {
|
||||
t.Fatalf("one refusal, looked at 15 times, reads as observed %d time(s) with %d evidence lines: %+v",
|
||||
c.Observations, len(c.Evidence), c.Evidence)
|
||||
}
|
||||
if !c.LastObserved.Equal(refused) || !c.Evidence[0].At.Equal(refused) {
|
||||
t.Fatalf("last observed %s, evidence at %s: the refusal was at %s", c.LastObserved, c.Evidence[0].At, refused)
|
||||
}
|
||||
|
||||
// The bus refuses it three times more between two looks: the condition counts four refusals, says
|
||||
// the newest, and adds one line of evidence for the look that saw them.
|
||||
again := now.Add(-5 * time.Second)
|
||||
advisory.Last, advisory.Count = again, 4
|
||||
c = look()
|
||||
if c.Observations != 4 || len(c.Evidence) != 2 || !c.LastObserved.Equal(again) || !c.Evidence[0].At.Equal(again) {
|
||||
t.Fatalf("four refusals read as observed %d time(s), last %s, evidence %+v", c.Observations, c.LastObserved, c.Evidence)
|
||||
}
|
||||
if !strings.Contains(c.Summary, "(4 times since 18:52 UTC)") {
|
||||
t.Fatalf("summary %q", c.Summary)
|
||||
}
|
||||
now = now.Add(30 * time.Second)
|
||||
if c = look(); c.Observations != 4 || len(c.Evidence) != 2 {
|
||||
t.Fatalf("a look with nothing new counted: observed %d time(s), evidence %+v", c.Observations, c.Evidence)
|
||||
}
|
||||
}
|
||||
|
||||
// **A condition that looks at the present still counts its looks** (novox/hq issue 402 changes only
|
||||
// what reads a record): a machine silent for three looks was observed three times, and says so.
|
||||
func TestAConditionOfThePresentCountsItsLooks(t *testing.T) {
|
||||
now := time.Date(2026, 10, 10, 19, 0, 0, 0, time.UTC)
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
||||
t.Cleanup(func() { k.Close(t.Context()) })
|
||||
var c conditions.Condition
|
||||
for range 3 {
|
||||
var err error
|
||||
if c, err = k.Observe(t.Context(), conditions.Observation{Scope: conditions.ScopeMachine, ID: "ace",
|
||||
Kind: "silent", Machine: "ace", Severity: conditions.Warning, Summary: "ace has not been heard from",
|
||||
Source: "S1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now = now.Add(30 * time.Second)
|
||||
}
|
||||
if c.Observations != 3 || len(c.Evidence) != 3 || !c.LastObserved.Equal(now.Add(-30*time.Second)) {
|
||||
t.Fatalf("observed %d time(s), %d evidence lines, last %s", c.Observations, len(c.Evidence), c.LastObserved)
|
||||
}
|
||||
}
|
||||
|
||||
// **One key, one watcher** (novox/hq issue 440). A consumer's max-deliveries condition is said from
|
||||
// DEAD_LETTERS while it holds a message the consumer gave up on (issue 330). A max-deliveries advisory
|
||||
// without a token names the same key; read beside it, each look added an observation and a line of
|
||||
// evidence through the dead-letter half, and the two summaries overwrote each other on every look. The
|
||||
// dead-letter row alone says that key, and counts the messages given up on, never the looks.
|
||||
func TestAHeldDeadLetterIsCountedByWhatWasGivenUpNotByTheLooks(t *testing.T) {
|
||||
gaveUp := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
|
||||
now := gaveUp.Add(20 * time.Second)
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
||||
t.Cleanup(func() { k.Close(t.Context()) })
|
||||
|
||||
const key = "bus.EVENTS.media_sonarr.max-deliveries"
|
||||
held := map[string]int{"EVENTS.media_sonarr": 1}
|
||||
newest := map[string]time.Time{"EVENTS.media_sonarr": gaveUp}
|
||||
advisory := link.Advisory{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.media_sonarr", Stream: "EVENTS",
|
||||
Consumer: "media_sonarr", Said: "sonarr on media handed message 7 over 5 times and gave up on it",
|
||||
First: gaveUp, Last: gaveUp, Count: 1}
|
||||
look := func() conditions.Condition {
|
||||
t.Helper()
|
||||
f := &signalFacts{now: now, host: "novox", advisories: []link.Advisory{advisory},
|
||||
deadLetters: held, deadLettersNewest: newest}
|
||||
if err := k.Reconcile(t.Context(), "S9", watchAdvisories(f)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, found, err := conditions.ReadOne(t.Context(), store, key)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("%s: found %v, %v", key, found, err)
|
||||
}
|
||||
if !strings.Contains(c.Summary, "dead-letters verb") {
|
||||
t.Fatalf("the summary is not the dead-letter row's: %q", c.Summary)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
var c conditions.Condition
|
||||
for range 5 {
|
||||
c = look()
|
||||
now = now.Add(30 * time.Second)
|
||||
}
|
||||
if c.Observations != 1 || len(c.Evidence) != 1 || !c.LastObserved.Equal(gaveUp) {
|
||||
t.Fatalf("one message given up on, looked at five times, reads as observed %d time(s), last %s, "+
|
||||
"with %d evidence lines: %+v", c.Observations, c.LastObserved, len(c.Evidence), c.Evidence)
|
||||
}
|
||||
|
||||
// The consumer gives up on a second message between two looks: two given up on, one more line.
|
||||
again := now.Add(-10 * time.Second)
|
||||
held["EVENTS.media_sonarr"], newest["EVENTS.media_sonarr"] = 2, again
|
||||
c = look()
|
||||
if c.Observations != 2 || len(c.Evidence) != 2 || !c.LastObserved.Equal(again) {
|
||||
t.Fatalf("two given up on read as observed %d time(s), last %s, evidence %+v", c.Observations,
|
||||
c.LastObserved, c.Evidence)
|
||||
}
|
||||
now = now.Add(30 * time.Second)
|
||||
if c = look(); c.Observations != 2 || len(c.Evidence) != 2 {
|
||||
t.Fatalf("a look with nothing new counted: observed %d time(s), evidence %+v", c.Observations, c.Evidence)
|
||||
}
|
||||
}
|
||||
|
||||
// The advisory still says the max-deliveries it alone knows: a message given up on that could not be
|
||||
// kept, under a key of its own (issue 330), which issue 440's change leaves as it was.
|
||||
func TestAMessageThatCouldNotBeKeptIsStillSaidFromTheAdvisory(t *testing.T) {
|
||||
at := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
|
||||
f := &signalFacts{now: at, host: "novox", advisories: []link.Advisory{{Kind: link.AdvisoryMaxDeliveries,
|
||||
ID: "EVENTS.media_sonarr", Stream: "EVENTS", Consumer: "media_sonarr", Token: link.AdvisoryNotKept,
|
||||
Said: "sonarr on media gave up on message 7, and it could not be kept", First: at, Last: at, Count: 1}}}
|
||||
said := watchAdvisories(f)
|
||||
if len(said) != 1 || said[0].Key() != "bus.EVENTS.media_sonarr.not-kept" {
|
||||
t.Fatalf("%+v", said)
|
||||
}
|
||||
}
|
||||
|
||||
// A max-deliveries advisory without a token names a consumer's dead-letter key, which only DEAD_LETTERS
|
||||
// says (novox/hq issues 330 and 440): with nothing held, the advisory alone raises nothing.
|
||||
func TestAMaxDeliveriesAdvisoryAloneRaisesNothing(t *testing.T) {
|
||||
at := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
|
||||
f := &signalFacts{now: at, host: "novox", advisories: []link.Advisory{{Kind: link.AdvisoryMaxDeliveries,
|
||||
ID: "EVENTS.media_sonarr", Stream: "EVENTS", Consumer: "media_sonarr",
|
||||
Said: "sonarr on media handed message 7 over 5 times and gave up on it", First: at, Last: at, Count: 1}}}
|
||||
if said := watchAdvisories(f); len(said) != 0 {
|
||||
t.Fatalf("said %+v", said)
|
||||
}
|
||||
}
|
||||
@@ -48,8 +48,7 @@ const agentAccountProbe = "DA"
|
||||
//
|
||||
// The one judgement: `node show`, the self-check, and ADR 0259's router honouring a verified sender read
|
||||
// it here.
|
||||
// now is the judging clock, threaded so a caller judging several things at one instant judges them all at it.
|
||||
func agentConfined(ctx context.Context, inv *inventory.Inventory, node string, now time.Time) (named, confined bool, why string, err error) {
|
||||
func agentConfined(ctx context.Context, inv *inventory.Inventory, node string) (named, confined bool, why string, err error) {
|
||||
n, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return false, false, "", err
|
||||
@@ -62,7 +61,7 @@ func agentConfined(ctx context.Context, inv *inventory.Inventory, node string, n
|
||||
if err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
confined, why = judgedConfined(n.AgentAccount, h, had, now)
|
||||
confined, why = judgedConfined(n.AgentAccount, h, had, time.Now())
|
||||
return true, confined, why, nil
|
||||
}
|
||||
|
||||
@@ -118,71 +117,6 @@ func judgedConfined(agent string, h inventory.NodeHealth, had bool, now time.Tim
|
||||
h.SaidAt.Local().Format("2006-01-02 15:04"))
|
||||
}
|
||||
|
||||
// searchQuietFor is how long the controller lets an agent account's verdict wait for the node-engine's setuid
|
||||
// search before that is itself the urgent condition: the longest a search is expected to take (link.RootSearchQuiet,
|
||||
// the engine's own value; novox/hq issue 361 — the search runs to completion, with no bound of its own), counted
|
||||
// from when this controller first saw it waiting, never from the engine's start. Quiet raises nothing; it never
|
||||
// makes the agent account confined, which only a healthy verdict from a complete, fresh search does.
|
||||
const searchQuietFor = link.RootSearchQuiet
|
||||
|
||||
// The kinds of an agent account's verdict, for the quiet a search earns.
|
||||
const (
|
||||
verdictOther = iota // anything else: a stale statement, an older engine, no verdict, another unknown
|
||||
verdictPending // waiting for the search, and otherwise healthy
|
||||
verdictComplete // judged: healthy, or a way to root found
|
||||
)
|
||||
|
||||
// rootVerdictKind reads a statement for the agent account (novox/hq ADR 0266): pending when every verdict on it
|
||||
// is healthy or not judged yet because the engine's setuid search runs, at least one of them that; complete when
|
||||
// every verdict is healthy or one found a way to root. The engine's own "since" is not read: it starts again at
|
||||
// every restart of the engine.
|
||||
func rootVerdictKind(agent string, h inventory.NodeHealth, had bool, now time.Time) int {
|
||||
if !had || now.Sub(h.HeardAt) > verdictFreshFor || h.Contract < link.RootContract {
|
||||
return verdictOther
|
||||
}
|
||||
pending, any := false, false
|
||||
for _, r := range h.Resources {
|
||||
if r.Kind != link.KindAccount || r.Target != agent || r.Root != link.RootNever {
|
||||
continue
|
||||
}
|
||||
any = true
|
||||
switch {
|
||||
case r.State == link.StateHealthy:
|
||||
case r.State == link.StateUnhealthy:
|
||||
return verdictComplete
|
||||
case r.State == link.StateUnknown && strings.HasPrefix(r.Reason, link.ReasonRootPending):
|
||||
pending = true
|
||||
default:
|
||||
return verdictOther
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case !any:
|
||||
return verdictOther
|
||||
case pending:
|
||||
return verdictPending
|
||||
}
|
||||
return verdictComplete
|
||||
}
|
||||
|
||||
// searchStillRunning says the one thing keeping an agent account from being judged is the node-engine's setuid
|
||||
// search, and that this controller first saw it waiting less than searchQuietFor ago — kept in the store, so a
|
||||
// node-engine restarted in a loop does not keep it quiet. A complete verdict forgets when it began.
|
||||
func searchStillRunning(ctx context.Context, inv *inventory.Inventory, node, agent string, h inventory.NodeHealth,
|
||||
had bool, now time.Time) (bool, error) {
|
||||
switch rootVerdictKind(agent, h, had, now) {
|
||||
case verdictComplete:
|
||||
return false, inv.RootSearchJudged(ctx, node)
|
||||
case verdictPending:
|
||||
since, err := inv.RootSearchPending(ctx, node, now)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return now.Sub(since) <= searchQuietFor, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// probeAgentAccounts is DA: every machine that names an agent account has it judged, on its node-engine's
|
||||
// newest statement, unable to become root without a person (ADR 0266).
|
||||
func probeAgentAccounts(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
@@ -200,22 +134,10 @@ func probeAgentAccounts(ctx context.Context, d *doctor) ([]conditions.Observatio
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := time.Now()
|
||||
quiet, err := searchStillRunning(ctx, inv, n.Name, n.AgentAccount, h, had, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
confined, why := judgedConfined(n.AgentAccount, h, had, now)
|
||||
confined, why := judgedConfined(n.AgentAccount, h, had, time.Now())
|
||||
if confined {
|
||||
continue
|
||||
}
|
||||
// Not judged yet only because the node-engine's search runs and no complete, fresh one judges: not the
|
||||
// urgent condition after every restart. The agent is still not confined — ADR 0259's router reads
|
||||
// agentConfined, not this — and `node show` still says not judged. Loud again once the search fails,
|
||||
// waits past searchQuietFor, or the statement goes stale.
|
||||
if quiet {
|
||||
continue
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "agent-root",
|
||||
Machine: n.Name, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("on %s, %s (ADR 0266): an agent there may become root without a person, and "+
|
||||
@@ -231,7 +153,7 @@ func agentAccountLines(ctx context.Context, inv *inventory.Inventory, n inventor
|
||||
return []string{fmt.Sprintf(" agents run as the operator account (%s); no agent account is named",
|
||||
orNoneKnown(n.Account))}
|
||||
}
|
||||
_, confined, why, err := agentConfined(ctx, inv, n.Name, time.Now())
|
||||
_, confined, why, err := agentConfined(ctx, inv, n.Name)
|
||||
if err != nil {
|
||||
return []string{fmt.Sprintf(" agents run as %s (home %s); whether it can become root could NOT be read: %v",
|
||||
n.AgentAccount, n.AgentHome(), err)}
|
||||
|
||||
@@ -1,161 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
snapshot "github.com/novox/mesh-controller/internal/facts"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-host/rootsearch"
|
||||
)
|
||||
|
||||
func TestTheSelfCheckSaysAnAgentAccountThatCanBecomeRoot(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.NodeByName(ctx, n); err != nil {
|
||||
if _, err := inv.AddNode(ctx, n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.SetAccount(ctx, n, "ops", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.SetAgentAccount(ctx, "anchor", "agent", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &doctor{open: open}
|
||||
found, err := probeAgentAccounts(ctx, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found = onlyMachine(found, "anchor")
|
||||
if len(found) != 1 || found[0].Machine != "anchor" || found[0].Severity != conditions.Urgent ||
|
||||
!strings.Contains(found[0].Said, "not judged") {
|
||||
t.Fatalf("a named agent account with no verdict: %+v", found)
|
||||
}
|
||||
w := plainWordings[kindAgentCanBecomeRoot](conditions.Observation{Kind: kindAgentCanBecomeRoot, Machine: "anchor"})
|
||||
if w.Headline == "" || w.Needs == "" || w.Resolved == "" {
|
||||
t.Errorf("the condition has no plain words: %+v", w)
|
||||
}
|
||||
|
||||
healthy := inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account",
|
||||
Kind: link.KindAccount, Target: "agent", State: link.StateHealthy, Root: link.RootNever, Account: "agent"}
|
||||
if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.RootContract,
|
||||
SaidAt: time.Now(), HeardAt: time.Now(), Resources: []inventory.ResourceHealth{healthy}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if found, err = probeAgentAccounts(ctx, d); err != nil || len(onlyMachine(found, "anchor")) != 0 {
|
||||
t.Fatalf("a judged agent account still fails: %+v %v", found, err)
|
||||
}
|
||||
if named, confined, why, err := agentConfined(ctx, inv, "anchor", time.Now()); err != nil || !named || !confined {
|
||||
t.Fatalf("agentConfined on anchor: %v %v %q %v", named, confined, why, err)
|
||||
}
|
||||
if named, _, why, err := agentConfined(ctx, inv, "laptop", time.Now()); err != nil || named ||
|
||||
!strings.Contains(why, "operator account") {
|
||||
t.Fatalf("agentConfined on a machine naming none: %v %q %v", named, why, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheFactsCarryTheAgentAccountAsAPseudonym(t *testing.T) {
|
||||
open, _ := aMeshWithSecrets(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.SetAccount(ctx, "anchor", "keeper", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetAgentAccount(ctx, "anchor", "warden", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := gatherFacts(ctx, open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, _ := f.Encode()
|
||||
if strings.Contains(string(body), "warden") {
|
||||
t.Error("the agent account's name is in the snapshot")
|
||||
}
|
||||
m, ok := f.Machine(snapshot.Pseudonym("machine", "anchor"))
|
||||
if !ok || m.AgentAccount != snapshot.Pseudonym("account", "warden") || m.Account == m.AgentAccount {
|
||||
t.Fatalf("the anchor's agent account reads as %q (operator %q)", m.AgentAccount, m.Account)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASearchStillRunningAfterARestartIsNotUrgent(t *testing.T) {
|
||||
if searchQuietFor != rootsearch.Quiet {
|
||||
t.Fatalf("the quiet is %s and the node-engine's %s: they are one value", searchQuietFor, rootsearch.Quiet)
|
||||
}
|
||||
// A daily search that finishes within the quiet never leaves the account unjudged between two of them.
|
||||
if rootsearch.FreshFor < rootsearch.Every+rootsearch.Quiet {
|
||||
t.Fatalf("a complete search judges for %s, less than a day's search (%s) and the quiet (%s)",
|
||||
rootsearch.FreshFor, rootsearch.Every, rootsearch.Quiet)
|
||||
}
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if _, err := inv.NodeByName(ctx, "anchor"); err != nil {
|
||||
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.SetAgentAccount(ctx, "anchor", "agent", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &doctor{open: open}
|
||||
say := func(state, reason string) []conditions.Observation {
|
||||
t.Helper()
|
||||
// The engine's since is always now: it was just restarted.
|
||||
v := inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account",
|
||||
Kind: link.KindAccount, Target: "agent", State: state, Reason: reason, Root: link.RootNever,
|
||||
Account: "agent", Since: time.Now()}
|
||||
if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.RootContract,
|
||||
SaidAt: time.Now(), HeardAt: time.Now(), Resources: []inventory.ResourceHealth{v}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found, err := probeAgentAccounts(ctx, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return onlyMachine(found, "anchor")
|
||||
}
|
||||
running := link.ReasonRootPending + ": the search for setuid programs, started at 19:21, has not finished yet"
|
||||
healthy := func() {
|
||||
t.Helper()
|
||||
if found := say(link.StateHealthy, ""); len(found) != 0 {
|
||||
t.Fatalf("a healthy verdict raised: %+v", found)
|
||||
}
|
||||
}
|
||||
if found := say(link.StateUnknown, running); len(found) != 0 {
|
||||
t.Fatalf("a search first seen now was raised: %+v", found)
|
||||
}
|
||||
if _, confined, why, _ := agentConfined(ctx, inv, "anchor", time.Now()); confined || !strings.Contains(why, "not judged") {
|
||||
t.Fatalf("an account whose search runs was read as confined: %q", why)
|
||||
}
|
||||
// The engine restarted again and again, each statement's own since fresh: the controller's clock runs on.
|
||||
if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
healthy() // a complete verdict forgets when the waiting began …
|
||||
if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil {
|
||||
t.Fatal(err) // … and this restart loop began past the bound
|
||||
}
|
||||
if found := say(link.StateUnknown, running); len(found) != 1 || found[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("a search pending past the bound, by the controller's clock, was not urgent: %+v", found)
|
||||
}
|
||||
healthy()
|
||||
incomplete := rootsearch.ReasonIncomplete + ": the search for setuid programs did not finish (last tried at " +
|
||||
"19:23: timeout); it is tried again later"
|
||||
if found := say(link.StateUnknown, incomplete); len(found) != 1 || found[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("a search that did not finish was not urgent: %+v", found)
|
||||
}
|
||||
if found := say(link.StateUnhealthy, link.ReasonRoot+": in the group docker; "+running); len(found) != 1 ||
|
||||
found[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("a way to root found while the search runs was not urgent: %+v", found)
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
snapshot "github.com/novox/mesh-controller/internal/facts"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
@@ -63,6 +64,55 @@ func TestAnAgentAccountIsConfinedOnlyOnAHealthyVerdictJudgedForRoot(t *testing.T
|
||||
|
||||
// DA raises an urgent condition, with plain words, on a machine whose agent account is not judged unable to
|
||||
// become root; a machine that names none is not its to judge.
|
||||
func TestTheSelfCheckSaysAnAgentAccountThatCanBecomeRoot(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.NodeByName(ctx, n); err != nil {
|
||||
if _, err := inv.AddNode(ctx, n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.SetAccount(ctx, n, "ops", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.SetAgentAccount(ctx, "anchor", "agent", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &doctor{open: open}
|
||||
found, err := probeAgentAccounts(ctx, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found = onlyMachine(found, "anchor")
|
||||
if len(found) != 1 || found[0].Machine != "anchor" || found[0].Severity != conditions.Urgent ||
|
||||
!strings.Contains(found[0].Said, "not judged") {
|
||||
t.Fatalf("a named agent account with no verdict: %+v", found)
|
||||
}
|
||||
w := plainWordings[kindAgentCanBecomeRoot](conditions.Observation{Kind: kindAgentCanBecomeRoot, Machine: "anchor"})
|
||||
if w.Headline == "" || w.Needs == "" || w.Resolved == "" {
|
||||
t.Errorf("the condition has no plain words: %+v", w)
|
||||
}
|
||||
|
||||
healthy := inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account",
|
||||
Kind: link.KindAccount, Target: "agent", State: link.StateHealthy, Root: link.RootNever, Account: "agent"}
|
||||
if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.RootContract,
|
||||
SaidAt: time.Now(), HeardAt: time.Now(), Resources: []inventory.ResourceHealth{healthy}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if found, err = probeAgentAccounts(ctx, d); err != nil || len(onlyMachine(found, "anchor")) != 0 {
|
||||
t.Fatalf("a judged agent account still fails: %+v %v", found, err)
|
||||
}
|
||||
if named, confined, why, err := agentConfined(ctx, inv, "anchor"); err != nil || !named || !confined {
|
||||
t.Fatalf("agentConfined on anchor: %v %v %q %v", named, confined, why, err)
|
||||
}
|
||||
if named, _, why, err := agentConfined(ctx, inv, "laptop"); err != nil || named ||
|
||||
!strings.Contains(why, "operator account") {
|
||||
t.Fatalf("agentConfined on a machine naming none: %v %q %v", named, why, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheAgentRootWordsArePlain(t *testing.T) {
|
||||
w := plainWordings[kindAgentCanBecomeRoot](conditions.Observation{Kind: kindAgentCanBecomeRoot, Machine: "anchor"})
|
||||
@@ -83,6 +133,28 @@ func onlyMachine(obs []conditions.Observation, machine string) []conditions.Obse
|
||||
|
||||
// The snapshot a merge check composes from carries the agent account as a pseudonym (novox/hq ADR 0266),
|
||||
// so a change is judged against machines that name one, and the name never leaves.
|
||||
func TestTheFactsCarryTheAgentAccountAsAPseudonym(t *testing.T) {
|
||||
open, _ := aMeshWithSecrets(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.SetAccount(ctx, "anchor", "keeper", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetAgentAccount(ctx, "anchor", "warden", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := gatherFacts(ctx, open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, _ := f.Encode()
|
||||
if strings.Contains(string(body), "warden") {
|
||||
t.Error("the agent account's name is in the snapshot")
|
||||
}
|
||||
m, ok := f.Machine(snapshot.Pseudonym("machine", "anchor"))
|
||||
if !ok || m.AgentAccount != snapshot.Pseudonym("account", "warden") || m.Account == m.AgentAccount {
|
||||
t.Fatalf("the anchor's agent account reads as %q (operator %q)", m.AgentAccount, m.Account)
|
||||
}
|
||||
}
|
||||
|
||||
// No verb runs a `node` command that sets something: through the generic `command` verb, `node account`,
|
||||
// `node agent-account` and every other `node` subcommand but list and show are refused, naming the terminal.
|
||||
@@ -125,10 +197,3 @@ func TestTheNodeVerbOnlyShows(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Until a complete search for setuid programs judges — none since the node-engine's state was kept, or the last
|
||||
// older than the engine lets one judge — the agent account is not judged, and the search runs to its end with no
|
||||
// bound (novox/hq issue 361). DA does not raise that as urgent within searchQuietFor, counted from when this
|
||||
// controller first saw it waiting — never from the engine's own "since", which a restart resets, so an engine
|
||||
// restarted in a loop does not keep it quiet. The account is still not confined, and `node show` still says not
|
||||
// judged; a search that failed, or a way to root found, is urgent at once.
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
|
||||
@@ -1,164 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// kindApplyFailing is S21's kind: a machine whose declarations have been refused, or applied only in part,
|
||||
// for longer than one apply interval (novox/hq issue 233).
|
||||
const kindApplyFailing = "apply-failing"
|
||||
|
||||
// applyEvery is how often a node-engine applies what it was last told when nothing new arrives: its
|
||||
// ReconcileEvery. A failure still there after one more of them is not one the next apply clears.
|
||||
const applyEvery = 5 * time.Minute
|
||||
|
||||
// packageManagerKey is how a node-engine names a package manager that does not answer among what failed
|
||||
// (mesh-host's probe.go): "the package manager (pacman)".
|
||||
const packageManagerKey = "the package manager ("
|
||||
|
||||
// watchApplyFailing is S21: **a machine that does not apply what it was sent is said by the mesh, not only
|
||||
// to someone who asks for `status`** (novox/hq issue 233, issue 187's family). A workstation refused every
|
||||
// declaration for an hour and a half because its package manager did not answer, and the mesh's only word
|
||||
// on it was a line in `status`. Raised once the machine's first report of this failure is older than one apply
|
||||
// interval **while the machine is heard** — judged by time, not by how many reports said it: a node-engine
|
||||
// says a re-apply only when how it went changed, so the same failure is one report however long it lasts.
|
||||
// Names the refusal's or the failure's words; cleared by the clean apply the node-engine then says. Not while
|
||||
// the machine is silent, asleep or not known: S1 says silent, and asleep is not lost.
|
||||
func watchApplyFailing(f *signalFacts) []conditions.Observation {
|
||||
heard := map[string]bool{}
|
||||
for _, m := range f.machines {
|
||||
if !m.asleep() && !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) <= heartbeatBound(m.every) {
|
||||
heard[m.name] = true
|
||||
}
|
||||
}
|
||||
var out []conditions.Observation
|
||||
said := map[string]bool{}
|
||||
for _, d := range f.doing {
|
||||
if !d.Wrong() || d.Since == nil || f.now.Sub(*d.Since) <= applyEvery || !heard[d.Node] {
|
||||
continue
|
||||
}
|
||||
out = append(out, applyFailing(d, f.now))
|
||||
said[d.Node] = true
|
||||
}
|
||||
// **A module's files the provisioner failed to place are a failed apply too** (novox/hq ADR 0362 §3): its client's
|
||||
// newest word, judged by the same bound — the client says a failure once, and fills again on every change —
|
||||
// on a machine whose node-engine applied what it was sent.
|
||||
failed := map[string][]inventory.ProvReport{}
|
||||
for _, r := range f.provFailed {
|
||||
if r.Status != inventory.ProvFailed || f.now.Sub(r.At) <= applyEvery || !heard[r.Node] || said[r.Node] {
|
||||
continue
|
||||
}
|
||||
failed[r.Node] = append(failed[r.Node], r)
|
||||
}
|
||||
for _, node := range sortedNames(failed) {
|
||||
out = append(out, provisioningFailing(node, failed[node], f.now))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// provisioningFailing is one machine's S21 condition for the modules whose files its provisioner client failed to
|
||||
// place.
|
||||
func provisioningFailing(m string, failed []inventory.ProvReport, now time.Time) conditions.Observation {
|
||||
sort.Slice(failed, func(a, b int) bool { return failed[a].Module < failed[b].Module })
|
||||
var names, words []string
|
||||
since := failed[0].At
|
||||
for _, r := range failed {
|
||||
names = append(names, r.Module)
|
||||
words = append(words, r.Module+": "+oneLine(orNoReason(r.Reason)))
|
||||
if r.At.Before(since) {
|
||||
since = r.At
|
||||
}
|
||||
}
|
||||
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m,
|
||||
Severity: conditions.Warning, Said: strings.Join(words, "; "),
|
||||
Resolved: m + " applies what it is sent again"}
|
||||
o.Summary = fmt.Sprintf("%s's provisioner has failed to place the files of %s since %s (bound %s)", m,
|
||||
strings.Join(names, ", "), since.UTC().Format("2006-01-02 15:04 MST"), ago(applyEvery))
|
||||
o.Headline = m + " is not applying its update"
|
||||
o.Explanation = fmt.Sprintf("The provisioner on %s has not been able to write the files of %s for %s. The "+
|
||||
"details say why.", m, strings.Join(names, ", "), humanDuration(now.Sub(since)))
|
||||
return o
|
||||
}
|
||||
|
||||
func orNoReason(s string) string {
|
||||
if s == "" {
|
||||
return "no reason given"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// applyFailing is one machine's S21 condition.
|
||||
func applyFailing(d inventory.Doing, now time.Time) conditions.Observation {
|
||||
m := d.Node
|
||||
since := d.Since.UTC().Format("2006-01-02 15:04 MST")
|
||||
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m,
|
||||
Severity: conditions.Warning, Said: doingWords(d),
|
||||
Resolved: m + " applies what it is sent again"}
|
||||
how := "refused its declaration whole"
|
||||
if d.Outcome != inventory.OutcomeRefused {
|
||||
how = "applied its declaration only in part"
|
||||
}
|
||||
o.Summary = fmt.Sprintf("%s has %s since %s (bound %s): %s", m, how, since, ago(applyEvery), failedNames(d))
|
||||
o.Headline = m + " is not applying its update"
|
||||
o.Explanation = fmt.Sprintf("%s has %s for %s, and trying again has not helped. The details say what fails.",
|
||||
m, plainHow(d), humanDuration(now.Sub(*d.Since)))
|
||||
if tool := unansweringTool(d); tool != "" {
|
||||
o.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent "+
|
||||
"is applied, since %s (bound %s)", m, tool, since, ago(applyEvery))
|
||||
o.Headline = fmt.Sprintf("%s cannot install packages", m)
|
||||
o.Explanation = fmt.Sprintf("The package manager on %s (%s) has not answered for %s, so no package reaches "+
|
||||
"it. Everything else is still applied, so an update that repairs it can land; if none does, it needs a "+
|
||||
"look on the machine.", m, tool, humanDuration(now.Sub(*d.Since)))
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
// plainHow is how a machine failed, as the operator reads it.
|
||||
func plainHow(d inventory.Doing) string {
|
||||
if d.Outcome == inventory.OutcomeRefused {
|
||||
return "refused every update"
|
||||
}
|
||||
return "applied its updates only in part"
|
||||
}
|
||||
|
||||
// unansweringTool is the package manager a machine said does not answer, or "".
|
||||
func unansweringTool(d inventory.Doing) string {
|
||||
for _, r := range d.Failed {
|
||||
if rest, ok := strings.CutPrefix(r.ID, packageManagerKey); ok {
|
||||
return strings.TrimSuffix(rest, ")")
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// failedNames is what failed, by name: the refusal, or the failed resources' ids.
|
||||
func failedNames(d inventory.Doing) string {
|
||||
if d.Outcome == inventory.OutcomeRefused {
|
||||
return "refused"
|
||||
}
|
||||
ids := make([]string, 0, len(d.Failed))
|
||||
for _, r := range d.Failed {
|
||||
ids = append(ids, r.ID)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
return "failed " + strings.Join(ids, ", ")
|
||||
}
|
||||
|
||||
// doingWords is the machine's own words on what failed: the evidence, kept inside the mesh.
|
||||
func doingWords(d inventory.Doing) string {
|
||||
if d.Outcome == inventory.OutcomeRefused {
|
||||
return "refused: " + oneLine(d.Refused)
|
||||
}
|
||||
var parts []string
|
||||
for _, r := range d.Failed {
|
||||
parts = append(parts, r.ID+": "+oneLine(r.Error))
|
||||
}
|
||||
sort.Strings(parts)
|
||||
return strings.Join(parts, "; ")
|
||||
}
|
||||
@@ -1,123 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// unansweringPackageManager is anchor's last report as the workstation's read on 2026-10-04, once its
|
||||
// node-engine applies the rest (novox/hq issue 233): the package failed and the package manager did not answer,
|
||||
// the same failure on times reports since for before now.
|
||||
func unansweringPackageManager(now time.Time, forHowLong time.Duration, times int) inventory.Doing {
|
||||
since := now.Add(-forHowLong)
|
||||
return inventory.Doing{Node: "anchor", Outcome: inventory.OutcomeFailed, At: now.Add(-time.Second),
|
||||
Since: &since, Times: times, Applied: 41, Failed: []inventory.FailedResource{
|
||||
{ID: "apply", Error: `applying "pacman.base": nothing can be said about "base": this is the arch host and ` +
|
||||
"pacman does not answer here. Either this machine is not Arch, or its package database is broken: " +
|
||||
"pacman exited 1: error: config file /etc/pacman.conf could not be read: No such file or directory"},
|
||||
{ID: "the package manager (pacman)", Error: "pacman does not answer here, so every package in this " +
|
||||
"declaration fails and the rest of it is applied (novox/hq issue 233): pacman exited 1: error: config " +
|
||||
"file /etc/pacman.conf could not be read: No such file or directory"}}}
|
||||
}
|
||||
|
||||
// **A machine whose package manager does not answer is raised by the mesh itself, naming the tool and the
|
||||
// probe's words** (novox/hq issue 233): the words in the evidence, the path kept out of the summary.
|
||||
func TestAPackageManagerThatDoesNotAnswerIsRaisedNamingItsWords(t *testing.T) {
|
||||
now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
|
||||
f := calm(now)
|
||||
f.doing = []inventory.Doing{unansweringPackageManager(now, 90*time.Minute, 18)}
|
||||
got := watchApplyFailing(f)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("raised %d conditions, want one: %+v", len(got), got)
|
||||
}
|
||||
o := got[0]
|
||||
if o.Key() != "machine.anchor.apply-failing" || o.Machine != "anchor" {
|
||||
t.Errorf("the condition is not the machine's: %s", o.Key())
|
||||
}
|
||||
if !strings.Contains(o.Summary, "pacman does not answer") || !strings.Contains(o.Summary, "since 2026-10-04 12:30") {
|
||||
t.Errorf("the summary does not name the tool and since when: %s", o.Summary)
|
||||
}
|
||||
if strings.Contains(o.Summary, "/etc/") {
|
||||
t.Errorf("the summary carries a path, which belongs in the evidence: %s", o.Summary)
|
||||
}
|
||||
if !strings.Contains(o.Said, "could not be read") {
|
||||
t.Errorf("the evidence does not carry the probe's words: %s", o.Said)
|
||||
}
|
||||
if !strings.Contains(o.Headline, "cannot install packages") || o.Explanation == "" || o.Resolved == "" {
|
||||
t.Errorf("the plain words do not say it: %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
// **A whole refusal is raised too**, with the node-engine's words: what the workstation's report said before the
|
||||
// node-engine applied the rest.
|
||||
func TestARefusalThatGoesOnIsRaised(t *testing.T) {
|
||||
now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
|
||||
since := now.Add(-time.Hour)
|
||||
f := calm(now)
|
||||
f.doing = []inventory.Doing{{Node: "anchor", Outcome: inventory.OutcomeRefused, Since: &since, Times: 12,
|
||||
Refused: "this is the arch host and pacman does not answer here", At: now.Add(-time.Minute)}}
|
||||
got := watchApplyFailing(f)
|
||||
if len(got) != 1 || !strings.Contains(got[0].Summary, "refused its declaration whole") ||
|
||||
!strings.Contains(got[0].Said, "pacman does not answer") {
|
||||
t.Fatalf("a refusal that went on for an hour was not raised with its words: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **Judged by time while the machine is heard, not by how many reports said it** (review of novox/hq issue 233):
|
||||
// a node-engine says the same failure once, so one report an apply interval old is a fault that goes on; one
|
||||
// younger is not yet; and a silent or sleeping machine is S1's.
|
||||
func TestOneReportOlderThanAnApplyIsRaisedAndAQuietMachineIsNot(t *testing.T) {
|
||||
now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
|
||||
f := calm(now)
|
||||
f.doing = []inventory.Doing{unansweringPackageManager(now, 6*time.Minute, 1)}
|
||||
if got := watchApplyFailing(f); len(got) != 1 {
|
||||
t.Errorf("one report of a failure six minutes old was not raised: %+v", got)
|
||||
}
|
||||
f.doing = []inventory.Doing{unansweringPackageManager(now, 4*time.Minute, 1)}
|
||||
if got := watchApplyFailing(f); len(got) != 0 {
|
||||
t.Errorf("a failure four minutes old was raised before one apply could clear it: %+v", got)
|
||||
}
|
||||
f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 1)}
|
||||
f.machines[0].lastHeard = now.Add(-time.Hour)
|
||||
if got := watchApplyFailing(f); len(got) != 0 {
|
||||
t.Errorf("a silent machine was raised for its last report: %+v", got)
|
||||
}
|
||||
f = calm(now)
|
||||
f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 1)}
|
||||
f.doing[0].Node = "a-machine-the-mesh-does-not-know"
|
||||
if got := watchApplyFailing(f); len(got) != 0 {
|
||||
t.Errorf("a machine not heard of was raised: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **A failed re-apply followed by a clean one clears S21** (review of novox/hq issue 233): the node-engine says
|
||||
// the clean re-apply, the store keeps it as applied with no failure since, and the condition the failure raised
|
||||
// is cleared through the keeper on the next tick.
|
||||
func TestACleanReapplyAfterAFailedOneClearsTheCondition(t *testing.T) {
|
||||
now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
||||
defer k.Close(context.Background())
|
||||
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
||||
|
||||
failing := calm(now)
|
||||
failing.doing = []inventory.Doing{unansweringPackageManager(now, 20*time.Minute, 1)}
|
||||
w.see(t.Context(), failing)
|
||||
open, err := k.Open(t.Context())
|
||||
if err != nil || len(open) != 1 || open[0].Kind != kindApplyFailing {
|
||||
t.Fatalf("the failed re-apply raised %+v (%v), want apply-failing", open, err)
|
||||
}
|
||||
|
||||
clean := calm(now.Add(5 * time.Minute))
|
||||
clean.doing = nil // the clean re-apply recorded: NotDoingWhatTheyWereTold no longer lists the machine
|
||||
w.see(t.Context(), clean)
|
||||
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||
t.Fatalf("the condition stayed open after the clean re-apply: %+v", open)
|
||||
}
|
||||
}
|
||||
@@ -1,946 +0,0 @@
|
||||
package main
|
||||
|
||||
// The controller asks, and acts on the operator's warrant (novox/hq ADR 0259 §6). It holds no channel, no
|
||||
// identity and no factor: it asks the router like any other module, and performs the answer chosen with its
|
||||
// own grant.
|
||||
//
|
||||
// - **For every open, unsilenced condition that needs the operator and names its answers**, one ask is
|
||||
// published on the `operator-channel` seat under the controller's own name: the condition's words, its
|
||||
// actions as options at their levels (Silence acknowledges; Release, Stop, Start and Restart approve),
|
||||
// answered by the operator, expiring after a day (a week when every option only acknowledges). A
|
||||
// condition that clears, is silenced, or changes its answers has its ask cancelled; an ask that expired
|
||||
// unanswered is asked again while the condition lasts; one the router refused is asked again after a wait
|
||||
// that grows with each refusal in a row, at most half an hour, so a refusal is never believed longer. Each ask is kept in the controller's bucket
|
||||
// `asked`, so a restart neither asks twice nor forgets.
|
||||
// - **On a warrant**, heard on the seat's event under the controller's own name (which only the router may
|
||||
// say), the controller acts once per ask: only for an ask it holds, only for the option it offered at
|
||||
// that option's level, and only while the condition is still open. It performs the action as itself —
|
||||
// a silence through its own conditions, any other through the verb the action names — with the warrant's
|
||||
// words as its why, and records it in the hand-act log as the operator's decision, naming the channel,
|
||||
// the ask and the proofs. An ask that ended without a choice is recorded and nothing is done.
|
||||
// - **A warrant it missed** while away is read from the router's record of its asks, under its own name.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The asker's name on the seat: the controller's module.
|
||||
const askerName = broker.ControllerSeat
|
||||
|
||||
// How long an ask lasts: a day when an answer approves, a week when every answer only acknowledges.
|
||||
const (
|
||||
// askApproveFor is a day less a margin, so an ask is never refused at the router for lasting a day and
|
||||
// a moment (the SDK's bound is a day).
|
||||
askApproveFor = 24*time.Hour - 10*time.Minute
|
||||
askAcknowledgeFor = 7 * 24 * time.Hour
|
||||
// askEvery is how often what is open is asked about again, beside every change.
|
||||
askEvery = time.Minute
|
||||
// askCatchUpAfter is how old an open ask is before the router's record of it is read: a warrant heard
|
||||
// on the event needs no reading.
|
||||
askCatchUpAfter = 2 * time.Minute
|
||||
// askAgainAfterAnswer is how long a condition the operator answered is not asked about again with the
|
||||
// same answers: what was chosen takes a while to clear it, and asking again at once would ask twice.
|
||||
askAgainAfterAnswer = time.Hour
|
||||
// askMostOpen is how many asks the controller holds open at once (the router refuses a fourth): the
|
||||
// most urgent conditions first, then the oldest.
|
||||
askMostOpen = asks.MostOpen
|
||||
// askRefusedRetryMost is the longest a refusal is believed without asking again (novox/hq issue 369): the
|
||||
// router refused while its channels had not yet said they could send, the channels could send twenty
|
||||
// minutes later, and the controller repeated that refusal for eleven hours. A refused ask is asked again
|
||||
// after askEvery, then twice as long after each refusal in a row, never longer than this — and at once when
|
||||
// the channels change.
|
||||
askRefusedRetryMost = 30 * time.Minute
|
||||
// askRefusedCounted is how far back refusals in a row are counted for that wait.
|
||||
askRefusedCounted = 6 * time.Hour
|
||||
// askVerdictWait is how long an ask made again after a refusal waits for the router's word before it is
|
||||
// taken as taken: the router refuses an ask as it reads it, so a refusal comes within seconds. Meanwhile
|
||||
// the condition saying it was not delivered stands as it was, neither cleared nor raised again.
|
||||
askVerdictWait = 2 * time.Minute
|
||||
)
|
||||
|
||||
// refusedRetryAfter is how long a part refused n times in a row waits before it is asked again.
|
||||
func refusedRetryAfter(n int) time.Duration {
|
||||
wait := askEvery
|
||||
for i := 1; i < n && wait < askRefusedRetryMost; i++ {
|
||||
wait *= 2
|
||||
}
|
||||
return min(wait, askRefusedRetryMost)
|
||||
}
|
||||
|
||||
// What became of an ask, as the controller keeps it.
|
||||
const (
|
||||
askOpen = "open"
|
||||
askCancelled = "cancelled"
|
||||
)
|
||||
|
||||
// asked is one ask the controller made, as it keeps it.
|
||||
type asked struct {
|
||||
ID string `json:"id"`
|
||||
Condition string `json:"condition"`
|
||||
// Channels is what the channels were when it was asked (asker.channels): an ask the router refused is asked
|
||||
// again at once when the condition's answers or the channels change, and otherwise after a wait that grows
|
||||
// with each refusal in a row (refusedRetryAfter, novox/hq issue 369).
|
||||
Channels string `json:"channels,omitempty"`
|
||||
Ask asks.Ask `json:"ask"`
|
||||
Actions []conditions.Action `json:"actions"`
|
||||
// Options are the actions by option id.
|
||||
Options map[string]int `json:"options"`
|
||||
State string `json:"state"`
|
||||
Opened time.Time `json:"opened"`
|
||||
Ended time.Time `json:"ended,omitempty"`
|
||||
Warrant *asks.Warrant `json:"warrant,omitempty"`
|
||||
// Acted is what the controller did on the warrant: empty before it did anything, "acting" while it acts,
|
||||
// then "done", "failed: …" or "nothing: …". Anything but empty is never acted on again.
|
||||
Acted string `json:"acted,omitempty"`
|
||||
// Part is which ask of its condition this is (askPart): empty for the one that carries the condition's
|
||||
// answers, or the authorising ones where it has both; "acknowledge" for its acknowledging answers asked
|
||||
// apart (the review of 2026-10-09, M1).
|
||||
Part string `json:"part,omitempty"`
|
||||
// Rehearsal is an ask started at the controller's terminal (rehearse.go): about no condition, its answers
|
||||
// perform nothing, and the reconciling of conditions leaves it alone.
|
||||
Rehearsal bool `json:"rehearsal,omitempty"`
|
||||
// Proposal is a settings layer proposed through a verb (proposals.go, novox/hq ADR 0277): about no
|
||||
// condition, set on Approve by the serving controller itself, and left alone by the reconciling of conditions.
|
||||
Proposal *settingsProposal `json:"proposal,omitempty"`
|
||||
}
|
||||
|
||||
// ofACondition says an ask is one of a condition's: not a rehearsal, not a proposal.
|
||||
func (r asked) ofACondition() bool { return !r.Rehearsal && r.Proposal == nil }
|
||||
|
||||
// partKey is an ask's place among what is asked: its condition and its part.
|
||||
func partKey(condition, part string) string { return condition + "#" + part }
|
||||
|
||||
// partAcknowledge is the part of a condition asked apart for its acknowledging answers.
|
||||
const partAcknowledge = "acknowledge"
|
||||
|
||||
// askPart is one ask a condition is asked with: its part, what it is about, and its answers.
|
||||
type askPart struct {
|
||||
name string
|
||||
about string
|
||||
actions []conditions.Action
|
||||
}
|
||||
|
||||
// levelOf is an action's level as an option offers it: one that says none is never taken for less than
|
||||
// approve.
|
||||
func levelOf(act conditions.Action) asks.Level {
|
||||
if act.Level == "" {
|
||||
return asks.Approve
|
||||
}
|
||||
return asks.Level(act.Level)
|
||||
}
|
||||
|
||||
// partsOf is the asks a condition is asked with (the review of 2026-10-09, M1): one, when its answers are all
|
||||
// of one kind; else its authorising answers (Release, Stop, Restart) in one ask, about the condition, and its
|
||||
// acknowledging ones (Silence) in another. **An acknowledgement never shares an ask with an approval**: a
|
||||
// channel that only acknowledges would otherwise answer the ask, and end the approval with it.
|
||||
func partsOf(c conditions.Condition) []askPart {
|
||||
var ack, auth []conditions.Action
|
||||
for _, act := range c.Actions {
|
||||
if levelOf(act) == asks.Acknowledge {
|
||||
ack = append(ack, act)
|
||||
} else {
|
||||
auth = append(auth, act)
|
||||
}
|
||||
}
|
||||
if len(ack) == 0 || len(auth) == 0 {
|
||||
return []askPart{{about: c.Key, actions: c.Actions}}
|
||||
}
|
||||
return []askPart{{about: c.Key, actions: auth},
|
||||
{name: partAcknowledge, about: c.Key + "." + partAcknowledge, actions: ack}}
|
||||
}
|
||||
|
||||
// askedStore keeps the asks (broker.AskedBucket). **Every write after the first is a compare-and-set** (the
|
||||
// review of 2026-10-09, L2): an ask is created once, and changed only over the revision it was read at, the
|
||||
// change decided again on what is read — so two controllers, or two deliveries of one warrant, never write
|
||||
// over each other, and of two that would act only the one whose write stands does.
|
||||
type askedStore interface {
|
||||
Get(ctx context.Context, id string) (*asked, error)
|
||||
// Create keeps a new ask, and refuses one already kept under its id.
|
||||
Create(ctx context.Context, a asked) error
|
||||
// Change applies change to the ask kept under id, by compare-and-set, and says whether its write stood.
|
||||
// change says whether to write at all; on a write that came between, it is asked again on what is read.
|
||||
Change(ctx context.Context, id string, change func(*asked) bool) (bool, error)
|
||||
All(ctx context.Context) ([]asked, error)
|
||||
}
|
||||
|
||||
// askChangeTries is how often a change is read and tried again when another write came between.
|
||||
const askChangeTries = 5
|
||||
|
||||
// asker is the controller asking the operator and acting on the answer.
|
||||
type asker struct {
|
||||
open func(ctx context.Context) ([]conditions.Condition, error)
|
||||
silence func(ctx context.Context, key string, d time.Duration, by, why string) error
|
||||
store askedStore
|
||||
// publish puts a message on a subject's stream, de-duplicated by id.
|
||||
publish func(ctx context.Context, subject string, body []byte, id string) error
|
||||
// call performs an action's verb with its arguments, as the controller.
|
||||
call func(ctx context.Context, a conditions.Action, args map[string]string) error
|
||||
// setLayer sets a proposed settings layer on the operator's warrant (novox/hq ADR 0277); nil cannot.
|
||||
setLayer setOnWarrant
|
||||
// record writes the hand-act log.
|
||||
record func(ctx context.Context, act link.HandAct) error
|
||||
// routerRecord reads the router's record of an ask for a warrant missed; nil reads nothing.
|
||||
routerRecord func(ctx context.Context, id string) (*asks.Warrant, error)
|
||||
// routerHere says whether a router holds the seat and takes asks under the asker's name; nil is yes.
|
||||
routerHere func(ctx context.Context) (bool, error)
|
||||
// grantHeld says whether the bus holds the controller's grant to ask: the user list the bus's machine was
|
||||
// last sent is the one the mesh composes now (novox/hq issue 353). The grant is composed from the router's
|
||||
// assignment and reaches the bus only when that machine is next pushed, so between `assign` and `push`
|
||||
// the record says a router is here and the bus refuses every ask. why says what to do; nil is yes.
|
||||
grantHeld func(ctx context.Context) (held bool, why string, err error)
|
||||
// channels is what the channels are now, as a fingerprint: who holds which kind, promising what.
|
||||
channels func(ctx context.Context) string
|
||||
// raise keeps the asker's own condition (sourceAsker): which conditions needing the operator could not be
|
||||
// asked, and why. Nil raises nothing (a test that does not look).
|
||||
raise func(ctx context.Context, obs []conditions.Observation) error
|
||||
now func() time.Time
|
||||
logf func(string, ...any)
|
||||
|
||||
saidNoRouter bool
|
||||
saidNoGrant bool
|
||||
// saidGrantCaveat is whether a grant held with a caveat was said, so it is said once.
|
||||
saidGrantCaveat bool
|
||||
|
||||
mu sync.Mutex
|
||||
nudged chan struct{}
|
||||
}
|
||||
|
||||
func (a *asker) nudge() {
|
||||
if a == nil {
|
||||
return
|
||||
}
|
||||
a.mu.Lock()
|
||||
if a.nudged == nil {
|
||||
a.nudged = make(chan struct{}, 1)
|
||||
}
|
||||
ch := a.nudged
|
||||
a.mu.Unlock()
|
||||
select {
|
||||
case ch <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// keep asks until ctx ends: now, on every change of a condition, and every askEvery.
|
||||
func (a *asker) keep(ctx context.Context) {
|
||||
a.nudge()
|
||||
tick := time.NewTicker(askEvery)
|
||||
defer tick.Stop()
|
||||
a.mu.Lock()
|
||||
nudged := a.nudged
|
||||
a.mu.Unlock()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
case <-nudged:
|
||||
}
|
||||
if err := a.reconcile(ctx); err != nil {
|
||||
a.logf("what the operator is asked could not be brought up to date: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// wants says whether a condition is one to ask about now.
|
||||
func wants(c conditions.Condition, now time.Time) bool {
|
||||
return len(c.Actions) > 0 && c.Needs != "" && !c.SilencedAt(now)
|
||||
}
|
||||
|
||||
func sameAsked(a []conditions.Action, b []conditions.Action) bool {
|
||||
x, _ := json.Marshal(a)
|
||||
y, _ := json.Marshal(b)
|
||||
return string(x) == string(y)
|
||||
}
|
||||
|
||||
// reconcile brings what is asked in line with what is open.
|
||||
func (a *asker) reconcile(ctx context.Context) error {
|
||||
now := a.now()
|
||||
if a.routerHere != nil {
|
||||
here, err := a.routerHere(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !here {
|
||||
if !a.saidNoRouter {
|
||||
a.logf("no router takes asks under the controller's name (a module declaring %s with its ask "+
|
||||
"named by its caller, assigned): the operator is asked nothing until one is", broker.AsksSeat)
|
||||
a.saidNoRouter = true
|
||||
}
|
||||
open, err := a.open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var unasked []conditions.Condition
|
||||
for _, c := range open {
|
||||
if wants(c, now) {
|
||||
unasked = append(unasked, c)
|
||||
}
|
||||
}
|
||||
return a.sayUnasked(ctx, unasked, "no router takes the controller's asks: no module holding "+
|
||||
broker.AsksSeat+" that takes an ask under its asker's name is assigned")
|
||||
}
|
||||
a.saidNoRouter = false
|
||||
}
|
||||
if a.grantHeld != nil {
|
||||
held, why, err := a.grantHeld(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !held {
|
||||
if !a.saidNoGrant {
|
||||
a.logf("the bus does not hold the controller's grant to ask yet: %s; the operator is asked nothing until it does", why)
|
||||
a.saidNoGrant = true
|
||||
}
|
||||
open, err := a.open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var unasked []conditions.Condition
|
||||
for _, c := range open {
|
||||
if wants(c, now) {
|
||||
unasked = append(unasked, c)
|
||||
}
|
||||
}
|
||||
return a.sayUnasked(ctx, unasked, "the bus does not hold the controller's grant to ask yet: "+why)
|
||||
}
|
||||
if why != "" && !a.saidGrantCaveat {
|
||||
// Held, with a caveat worth one line (an older bus module cannot confirm the reload).
|
||||
a.logf("the controller asks: %s", why)
|
||||
a.saidGrantCaveat = true
|
||||
} else if why == "" {
|
||||
a.saidGrantCaveat = false
|
||||
}
|
||||
a.saidNoGrant = false
|
||||
}
|
||||
channels := ""
|
||||
if a.channels != nil {
|
||||
channels = a.channels(ctx)
|
||||
}
|
||||
open, err := a.open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
all, err := a.store.All(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
byCondition := map[string]asked{} // by partKey
|
||||
// What is open and not a condition's — a rehearsal, a proposal — still counts toward what the router holds
|
||||
// open for the controller (askMostOpen); expired unanswered, it is kept so, as the router says it too.
|
||||
otherOpen := 0
|
||||
for _, r := range all {
|
||||
if r.State == askOpen && !r.ofACondition() && !now.Before(r.Ask.Expires) {
|
||||
if _, err := a.store.Change(ctx, r.ID, func(x *asked) bool {
|
||||
if x.State != askOpen || x.Acted != "" {
|
||||
return false
|
||||
}
|
||||
x.State, x.Ended, x.Acted = string(asks.OutcomeExpired), now, "nothing: the ask expired unanswered"
|
||||
return true
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if r.State == askOpen && !r.ofACondition() {
|
||||
otherOpen++
|
||||
}
|
||||
if r.State == askOpen && r.ofACondition() {
|
||||
k := partKey(r.Condition, r.Part)
|
||||
if prior, held := byCondition[k]; !held || r.Opened.After(prior.Opened) {
|
||||
byCondition[k] = r
|
||||
}
|
||||
}
|
||||
}
|
||||
// A warrant missed while away, read from the router's record: for a condition's ask, and for a proposal's or a
|
||||
// rehearsal's alike.
|
||||
if a.routerRecord != nil {
|
||||
var lookedUp []asked
|
||||
for _, r := range byCondition {
|
||||
lookedUp = append(lookedUp, r)
|
||||
}
|
||||
for _, r := range all {
|
||||
if r.State == askOpen && !r.ofACondition() {
|
||||
lookedUp = append(lookedUp, r)
|
||||
}
|
||||
}
|
||||
for _, r := range lookedUp {
|
||||
if now.Sub(r.Opened) < askCatchUpAfter {
|
||||
continue
|
||||
}
|
||||
if w, err := a.routerRecord(ctx, r.ID); err == nil && w != nil {
|
||||
body, _ := json.Marshal(w)
|
||||
if err := a.Decided(ctx, body); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if all, err = a.store.All(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
byCondition = map[string]asked{}
|
||||
for _, r := range all {
|
||||
if r.State == askOpen && r.ofACondition() {
|
||||
byCondition[partKey(r.Condition, r.Part)] = r
|
||||
}
|
||||
}
|
||||
}
|
||||
// What the operator answered lately, by condition: not asked again at once; and what the router refused,
|
||||
// newest first: not asked again until the answers or the channels change.
|
||||
answered, refused := map[string]asked{}, map[string]asked{}
|
||||
for _, r := range all {
|
||||
k := partKey(r.Condition, r.Part)
|
||||
if r.State == string(asks.OutcomeChosen) && now.Sub(r.Ended) < askAgainAfterAnswer {
|
||||
answered[k] = r
|
||||
}
|
||||
if r.State == string(asks.OutcomeRefused) {
|
||||
if prior, has := refused[k]; !has || r.Opened.After(prior.Opened) {
|
||||
refused[k] = r
|
||||
}
|
||||
}
|
||||
}
|
||||
// Refusals in a row, by part: those since the last ask that was not refused, within askRefusedCounted.
|
||||
// superseded: a part asked since its last refusal, by an ask the router did not refuse — open, answered,
|
||||
// expired or cancelled. Its refusal is history then, never said again (the review of PR 198: an answered
|
||||
// retry brought the refusal back).
|
||||
inRow, superseded := map[string]int{}, map[string]asked{}
|
||||
for k, last := range refused {
|
||||
for _, r := range all {
|
||||
if partKey(r.Condition, r.Part) == k && r.State != string(asks.OutcomeRefused) && r.State != askUnsent &&
|
||||
r.Opened.After(last.Opened) && r.Opened.After(superseded[k].Opened) {
|
||||
superseded[k] = r
|
||||
}
|
||||
}
|
||||
var since time.Time
|
||||
for _, r := range all {
|
||||
if partKey(r.Condition, r.Part) == k && r.State != string(asks.OutcomeRefused) && r.State != askUnsent &&
|
||||
!r.Opened.After(last.Opened) && r.Opened.After(since) {
|
||||
since = r.Opened
|
||||
}
|
||||
}
|
||||
for _, r := range all {
|
||||
if partKey(r.Condition, r.Part) == k && r.State == string(asks.OutcomeRefused) && r.Opened.After(since) &&
|
||||
now.Sub(r.Opened) < askRefusedCounted {
|
||||
inRow[k]++
|
||||
}
|
||||
}
|
||||
}
|
||||
wanted := map[string]bool{}
|
||||
var unasked []conditions.Condition // refused by the router, and nothing it was refused for changed
|
||||
var refusedWords []string
|
||||
// The most urgent first, then the oldest: those are asked when no more than askMostOpen may be.
|
||||
sort.SliceStable(open, func(i, j int) bool {
|
||||
ui, uj := open[i].Severity == conditions.Urgent, open[j].Severity == conditions.Urgent
|
||||
if ui != uj {
|
||||
return ui
|
||||
}
|
||||
if !open[i].Raised.Equal(open[j].Raised) {
|
||||
return open[i].Raised.Before(open[j].Raised)
|
||||
}
|
||||
return open[i].Key < open[j].Key
|
||||
})
|
||||
openNow := otherOpen
|
||||
for _, c := range open {
|
||||
if !wants(c, now) {
|
||||
continue
|
||||
}
|
||||
for _, p := range partsOf(c) {
|
||||
if r, held := byCondition[partKey(c.Key, p.name)]; held && sameAsked(r.Actions, p.actions) && now.Before(r.Ask.Expires) {
|
||||
openNow++
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, c := range open {
|
||||
if !wants(c, now) {
|
||||
continue
|
||||
}
|
||||
saidUnasked := false
|
||||
for _, p := range partsOf(c) {
|
||||
key := partKey(c.Key, p.name)
|
||||
wanted[key] = true
|
||||
if r, was := refused[key]; was && sameAsked(r.Actions, p.actions) {
|
||||
cur, held := byCondition[key]
|
||||
ended := r.Ended
|
||||
if ended.IsZero() {
|
||||
ended = r.Opened
|
||||
}
|
||||
later, asked := superseded[key]
|
||||
waiting := !held && !asked && r.Channels == channels && now.Sub(ended) < refusedRetryAfter(inRow[key])
|
||||
// Asked again now (the wait over), or lately and the router's word not in yet: said as it was until
|
||||
// that word, so the condition neither clears nor is raised again at each try.
|
||||
retrying := !held && !asked && !waiting
|
||||
verdictDue := held && asked && later.ID == cur.ID && now.Sub(cur.Opened) < askVerdictWait
|
||||
if waiting || retrying || verdictDue {
|
||||
if !saidUnasked {
|
||||
unasked, saidUnasked = append(unasked, c), true
|
||||
}
|
||||
if r.Warrant != nil && r.Warrant.Words != "" {
|
||||
refusedWords = append(refusedWords, r.Warrant.Words)
|
||||
}
|
||||
}
|
||||
if waiting {
|
||||
continue // refused lately, and nothing it was refused for has changed: asked again after the wait
|
||||
}
|
||||
}
|
||||
if r, done := answered[key]; done && sameAsked(r.Actions, p.actions) {
|
||||
if _, held := byCondition[key]; !held {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if r, held := byCondition[key]; held {
|
||||
switch {
|
||||
case !sameAsked(r.Actions, p.actions):
|
||||
if err := a.cancel(ctx, r, "its answers changed"); err != nil {
|
||||
return err
|
||||
}
|
||||
case !now.Before(r.Ask.Expires):
|
||||
// Expired unanswered: the router says so too; asked again below while it lasts.
|
||||
if _, err := a.store.Change(ctx, r.ID, func(x *asked) bool {
|
||||
if x.State != askOpen {
|
||||
return false
|
||||
}
|
||||
x.State, x.Ended = string(asks.OutcomeExpired), now
|
||||
return true
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
openNow--
|
||||
default:
|
||||
continue
|
||||
}
|
||||
}
|
||||
if openNow >= askMostOpen {
|
||||
continue // asked when one of the open ones ends, most urgent first
|
||||
}
|
||||
if err := a.ask(ctx, c, p, channels); err != nil {
|
||||
a.logf("the operator could not be asked about %s: %v", c.Key, err)
|
||||
continue
|
||||
}
|
||||
openNow++
|
||||
}
|
||||
}
|
||||
stillOpen := map[string]conditions.Condition{}
|
||||
for _, c := range open {
|
||||
stillOpen[c.Key] = c
|
||||
}
|
||||
for key, r := range byCondition {
|
||||
if wanted[key] {
|
||||
continue
|
||||
}
|
||||
// **A silence never takes an approval back** (the confirmation review of 2026-10-09, M1). Silence is an
|
||||
// acknowledgement — anyone at the desk may give it — so a condition silenced while its approval is asked
|
||||
// keeps that ask open, unchanged, until it is answered on a channel that proves who answered, or expires.
|
||||
// It is not asked again once it ends, while the silence lasts.
|
||||
if c, open := stillOpen[r.Condition]; open && c.SilencedAt(now) && r.Ask.Highest() != asks.Acknowledge &&
|
||||
now.Before(r.Ask.Expires) && keepsItsAnswers(c, r) {
|
||||
continue
|
||||
}
|
||||
if err := a.cancel(ctx, r, "the condition ended, was silenced or needs nothing now"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
why := "the router refused the ask"
|
||||
if len(refusedWords) > 0 {
|
||||
why += ": " + refusedWords[0]
|
||||
}
|
||||
return a.sayUnasked(ctx, unasked, why)
|
||||
}
|
||||
|
||||
// keepsItsAnswers says a condition still offers the answers an ask kept was asked with.
|
||||
func keepsItsAnswers(c conditions.Condition, r asked) bool {
|
||||
for _, p := range partsOf(c) {
|
||||
if partKey(c.Key, p.name) == partKey(r.Condition, r.Part) {
|
||||
return sameAsked(r.Actions, p.actions)
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// sourceAsker raises the asker's own condition.
|
||||
const sourceAsker = "asker"
|
||||
|
||||
// sayUnasked keeps the asker's one condition: while a condition that needs the operator could not be asked
|
||||
// on any channel, said loudly (failure must be loud), cleared when every one could be.
|
||||
func (a *asker) sayUnasked(ctx context.Context, unasked []conditions.Condition, why string) error {
|
||||
if a.raise == nil {
|
||||
return nil
|
||||
}
|
||||
var obs []conditions.Observation
|
||||
if len(unasked) > 0 {
|
||||
keys := make([]string, 0, len(unasked))
|
||||
severity := conditions.Warning
|
||||
for _, c := range unasked {
|
||||
keys = append(keys, c.Key)
|
||||
if c.Severity == conditions.Urgent {
|
||||
severity = conditions.Urgent
|
||||
}
|
||||
}
|
||||
sort.Strings(keys)
|
||||
obs = append(obs, conditions.Observation{Scope: conditions.ScopeSeat, ID: broker.AsksSeat, Token: "unasked",
|
||||
Kind: "asks-undelivered", Severity: severity, Source: sourceAsker,
|
||||
Summary: fmt.Sprintf("%d condition(s) that need the operator could not be asked on any channel: %s; %s",
|
||||
len(keys), strings.Join(keys, ", "), why),
|
||||
Headline: "Questions for you not delivered",
|
||||
Explanation: "The mesh could not send you its questions on any channel.",
|
||||
Needs: "answer them from the mesh MCP server, and check why no channel carries them.",
|
||||
Resolved: "The mesh can ask you again"})
|
||||
}
|
||||
if err := a.raise(ctx, obs); err != nil {
|
||||
a.logf("whether the operator could be asked could not be kept as a condition: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// optionID is an action's label as an option's id: "Silence for a week" is silence-for-a-week.
|
||||
func optionID(label string) string {
|
||||
var b strings.Builder
|
||||
dash := false
|
||||
for _, r := range strings.ToLower(label) {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
|
||||
b.WriteRune(r)
|
||||
dash = false
|
||||
case !dash && b.Len() > 0:
|
||||
b.WriteByte('-')
|
||||
dash = true
|
||||
}
|
||||
}
|
||||
return strings.TrimSuffix(b.String(), "-")
|
||||
}
|
||||
|
||||
// doesWords is what an action does, in the words an option says it with.
|
||||
func doesWords(act conditions.Action) string {
|
||||
switch {
|
||||
case act.Arguments["silence"] != "":
|
||||
return "nothing more is said of it for a week"
|
||||
case act.Verb == "mesh-delivery.release":
|
||||
return "the delivery goes on"
|
||||
case act.Verb == "mesh-delivery.stop":
|
||||
return "the delivery ends"
|
||||
case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["go"] != "":
|
||||
return "the delivery starts"
|
||||
case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["stop"] != "":
|
||||
return "the delivery is stopped"
|
||||
case strings.HasSuffix(act.Verb, ".restart"):
|
||||
return "its service is restarted on " + act.Machine
|
||||
}
|
||||
return strings.ToLower(act.Label)
|
||||
}
|
||||
|
||||
// askText is a condition's words as an ask says them: without where an answer is given when no channel can
|
||||
// give it (FromMeshMCPServer), since the ask is answered on a channel and the router says where else.
|
||||
func askText(s string) string {
|
||||
for _, with := range []string{", " + FromMeshMCPServer, " " + FromMeshMCPServer} {
|
||||
s = strings.ReplaceAll(s, with, ".")
|
||||
}
|
||||
return strings.ReplaceAll(s, "..", ".")
|
||||
}
|
||||
|
||||
// askOf is the ask one part of a condition is asked with.
|
||||
func askOf(id string, c conditions.Condition, p askPart, now time.Time) (asks.Ask, map[string]int) {
|
||||
q := asks.Ask{ID: id, Headline: c.Headline, Explanation: askText(c.Explanation), Who: asks.Operator,
|
||||
OnExpiry: "nothing is done, and you are asked again while it lasts", About: p.about,
|
||||
Urgent: c.Severity == conditions.Urgent}
|
||||
options := map[string]int{}
|
||||
approves := false
|
||||
for i, act := range p.actions {
|
||||
level := levelOf(act) // an action that says nothing of its level is never taken for less than approve
|
||||
approves = approves || level != asks.Acknowledge
|
||||
oid := optionID(act.Label)
|
||||
options[oid] = i
|
||||
// Every option binds the exact act it stands for (novox/hq ADR 0259 §6): the verb, the machine and
|
||||
// every argument. The warrant then authorises that act and no other.
|
||||
binds, _ := asks.ActDigest(boundAct(act))
|
||||
q.Options = append(q.Options, asks.Option{ID: oid, Label: act.Label, Does: doesWords(act), Level: level,
|
||||
Binds: binds})
|
||||
}
|
||||
q.Expires = now.Add(askAcknowledgeFor)
|
||||
if approves {
|
||||
q.Expires = now.Add(askApproveFor)
|
||||
}
|
||||
return q, options
|
||||
}
|
||||
|
||||
// boundAct is what an option's Binds digests: the act exactly as the controller will perform it — its verb,
|
||||
// machine, level, and each argument as "arg.<name>" — and never its label or words.
|
||||
func boundAct(act conditions.Action) asks.Act {
|
||||
out := asks.Act{"verb": act.Verb, "machine": act.Machine, "level": act.Level}
|
||||
for k, v := range act.Arguments {
|
||||
out["arg."+k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func newAskID() string {
|
||||
var b [8]byte
|
||||
_, _ = rand.Read(b[:])
|
||||
return "c" + hex.EncodeToString(b[:])
|
||||
}
|
||||
|
||||
// askUnsent is an ask kept and never published: asked again at the next look.
|
||||
const askUnsent = "unsent"
|
||||
|
||||
// ask publishes one ask about a part of a condition, kept before it is published (the review of 2026-10-09,
|
||||
// L3): a warrant for it then always finds it, and one whose publishing failed is marked so and asked again.
|
||||
func (a *asker) ask(ctx context.Context, c conditions.Condition, p askPart, channels string) error {
|
||||
now := a.now()
|
||||
id := newAskID()
|
||||
q, options := askOf(id, c, p, now)
|
||||
if err := q.Check(now); err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := json.Marshal(q)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := a.store.Create(ctx, asked{ID: id, Condition: c.Key, Part: p.name, Ask: q, Actions: p.actions,
|
||||
Options: options, State: askOpen, Opened: now, Channels: channels}); err != nil {
|
||||
return fmt.Errorf("the ask could not be kept, so it was not asked: %w", err)
|
||||
}
|
||||
if err := a.publish(ctx, asks.AskSubject(askerName), body, "ask."+id); err != nil {
|
||||
if _, cerr := a.store.Change(ctx, id, func(x *asked) bool {
|
||||
if x.State != askOpen || x.Acted != "" {
|
||||
return false
|
||||
}
|
||||
x.State, x.Ended, x.Acted = askUnsent, a.now(), "nothing: it could not be published: "+err.Error()
|
||||
return true
|
||||
}); cerr != nil {
|
||||
a.logf("the ask %s could not be published, and could not be marked so: %v", id, cerr)
|
||||
}
|
||||
return err
|
||||
}
|
||||
a.logf("asked the operator about %s (%s): %d answer(s)", c.Key, id, len(q.Options))
|
||||
return nil
|
||||
}
|
||||
|
||||
// cancel takes an ask back: kept cancelled first, so a warrant that comes after is refused, then said to the
|
||||
// router; a cancel the router did not hear leaves the ask to expire there, and nothing is done on it here.
|
||||
func (a *asker) cancel(ctx context.Context, r asked, why string) error {
|
||||
stood, err := a.store.Change(ctx, r.ID, func(x *asked) bool {
|
||||
if x.State != askOpen {
|
||||
return false
|
||||
}
|
||||
x.State, x.Ended = askCancelled, a.now()
|
||||
return true
|
||||
})
|
||||
if err != nil || !stood {
|
||||
return err
|
||||
}
|
||||
body, _ := json.Marshal(map[string]string{"id": r.ID})
|
||||
if err := a.publish(ctx, asks.CancelSubject(askerName), body, "cancel."+r.ID); err != nil {
|
||||
a.logf("the ask %s about %s is taken back here, and the router could not be told (%v): it expires there, "+
|
||||
"and no answer to it is acted on", r.ID, r.Condition, err)
|
||||
return nil
|
||||
}
|
||||
a.logf("took back the ask %s about %s: %s", r.ID, r.Condition, why)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Decided takes the router's word on one of the controller's asks (link.Decider). An error is returned only
|
||||
// when what was decided could not be kept, so the word is held and heard again.
|
||||
func (a *asker) Decided(ctx context.Context, body []byte) error {
|
||||
var w asks.Warrant
|
||||
if err := json.Unmarshal(body, &w); err != nil {
|
||||
a.logf("the router's word on an ask could not be read; ignored: %v", err)
|
||||
return nil
|
||||
}
|
||||
if w.Asker != askerName {
|
||||
a.logf("REFUSED a warrant for %s's ask %s: the controller acts only on its own", w.Asker, w.Ask)
|
||||
return nil
|
||||
}
|
||||
r, err := a.store.Get(ctx, w.Ask)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if r == nil {
|
||||
a.logf("REFUSED a warrant for the ask %s, which the controller does not hold", w.Ask)
|
||||
return nil
|
||||
}
|
||||
if r.Acted != "" {
|
||||
return nil // heard again: acted on once
|
||||
}
|
||||
now := a.now()
|
||||
if w.Outcome != asks.OutcomeChosen {
|
||||
acted := "nothing: the ask " + string(w.Outcome)
|
||||
if w.Words != "" {
|
||||
acted += ": " + w.Words
|
||||
}
|
||||
if _, err := a.store.Change(ctx, r.ID, func(x *asked) bool {
|
||||
if x.Acted != "" {
|
||||
return false
|
||||
}
|
||||
x.State, x.Ended, x.Warrant, x.Acted = string(w.Outcome), now, &w, acted
|
||||
return true
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
a.logf("the ask %s about %s ended %s; nothing is done", r.ID, r.Condition, w.Outcome)
|
||||
return nil
|
||||
}
|
||||
if r.State != askOpen {
|
||||
// Cancelled, replaced or expired in the controller's own record: no answer to it is acted on.
|
||||
a.logf("REFUSED a warrant for the ask %s, which is %s in the controller's own record", r.ID, r.State)
|
||||
return nil
|
||||
}
|
||||
option, err := w.For(askerName, r.Ask)
|
||||
if err != nil {
|
||||
a.logf("REFUSED a warrant for the ask %s: %v", r.ID, err)
|
||||
return nil
|
||||
}
|
||||
index, offered := r.Options[option.ID]
|
||||
if !offered || index >= len(r.Actions) {
|
||||
a.logf("REFUSED a warrant for the ask %s: it chose %s, which no action stands for", r.ID, option.ID)
|
||||
return nil
|
||||
}
|
||||
act := r.Actions[index]
|
||||
// The act about to be performed is the one the option bound when the controller asked: a record changed
|
||||
// since is refused, never performed.
|
||||
if err := option.Performs(boundAct(act)); err != nil {
|
||||
a.logf("REFUSED a warrant for the ask %s: %v", r.ID, err)
|
||||
return nil
|
||||
}
|
||||
open, err := a.open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stillOpen := r.Rehearsal || r.Proposal != nil // a rehearsal and a proposal are about no condition
|
||||
for _, c := range open {
|
||||
stillOpen = stillOpen || c.Key == r.Condition
|
||||
}
|
||||
if !stillOpen {
|
||||
// The asker checks the state is still what it asked about before it acts (to-be 46 §10, step 7).
|
||||
if _, err := a.store.Change(ctx, r.ID, func(x *asked) bool {
|
||||
if x.State != askOpen || x.Acted != "" {
|
||||
return false
|
||||
}
|
||||
x.State, x.Warrant, x.Ended, x.Acted = string(asks.OutcomeChosen), &w, now,
|
||||
"nothing: the condition ended before the answer"
|
||||
return true
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
a.logf("%s, for %s, which ended meanwhile: nothing is done", w.Says(), r.Condition)
|
||||
return nil
|
||||
}
|
||||
// Claimed before acting, by compare-and-set: only the delivery whose write stands acts (security review
|
||||
// of 2026-10-08, finding 9). Not by the warrant's message id, which another publisher could take first:
|
||||
// the controller's own record decides.
|
||||
claimed, err := a.store.Change(ctx, r.ID, func(x *asked) bool {
|
||||
if x.State != askOpen || x.Acted != "" {
|
||||
return false
|
||||
}
|
||||
x.State, x.Warrant, x.Acted = string(asks.OutcomeChosen), &w, "acting"
|
||||
return true
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !claimed {
|
||||
a.logf("the warrant for the ask %s was already taken by another delivery; nothing more is done", r.ID)
|
||||
return nil
|
||||
}
|
||||
r.Acted = "acting"
|
||||
|
||||
why := fmt.Sprintf("%s (ask %s)", w.Says(), r.ID)
|
||||
args := map[string]string{}
|
||||
for k, v := range act.Arguments {
|
||||
args[k] = v
|
||||
}
|
||||
if v, takes := args["why"]; takes && v == "" {
|
||||
args["why"] = why
|
||||
}
|
||||
var acted error
|
||||
outcome := "done"
|
||||
switch {
|
||||
case r.Rehearsal && act.Verb == rehearsalVerb:
|
||||
// A rehearsal's answer performs nothing: it is recorded below as the operator's decision.
|
||||
case r.Proposal != nil && act.Verb == proposalVerb:
|
||||
// A proposed settings layer, set by this controller itself on Approve (novox/hq ADR 0277): the act's
|
||||
// digest of the values is held to the record's own values before anything is set.
|
||||
outcome, acted = a.decideProposal(ctx, *r, act, w)
|
||||
if acted == nil && outcome != "" && !strings.HasPrefix(outcome, "nothing") {
|
||||
outcome = "done: " + outcome
|
||||
}
|
||||
case act.Arguments["silence"] != "":
|
||||
acted = a.silence(ctx, act.Arguments["silence"], conditions.MaxSilence, byWords(w), why)
|
||||
default:
|
||||
acted = a.call(ctx, act, args)
|
||||
}
|
||||
ended := a.now()
|
||||
if acted != nil {
|
||||
outcome = "failed: " + acted.Error()
|
||||
}
|
||||
r.Ended, r.Acted = ended, outcome
|
||||
if _, err := a.store.Change(ctx, r.ID, func(x *asked) bool {
|
||||
if x.Acted != "acting" {
|
||||
return false
|
||||
}
|
||||
x.Ended, x.Acted = ended, outcome
|
||||
return true
|
||||
}); err != nil {
|
||||
a.logf("%s was acted on (%s), and how it ended could NOT be kept: %v", r.ID, outcome, err)
|
||||
}
|
||||
verbArgs := []string{act.Verb}
|
||||
if act.Machine != "" {
|
||||
verbArgs = append(verbArgs, "on "+act.Machine)
|
||||
}
|
||||
keys := make([]string, 0, len(args))
|
||||
for k := range args {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
if k != "why" {
|
||||
verbArgs = append(verbArgs, k+"="+args[k])
|
||||
}
|
||||
}
|
||||
if err := a.record(ctx, link.HandAct{Verb: handActWarrant, Args: verbArgs, Why: why, By: byWords(w),
|
||||
Cause: conditions.CauseOperatorAnswer, Condition: r.Condition, Via: viaWords(w), Ask: r.ID,
|
||||
Proofs: w.Proofs, RequestedBy: r.Condition, Outcome: r.Acted}); err != nil {
|
||||
a.logf("%s was done, and could NOT be recorded in the hand-act log: %v", why, err)
|
||||
}
|
||||
a.logf("%s: %s", why, r.Acted)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handActWarrant is the verb an act the operator chose on a warrant is recorded under: a person's decision,
|
||||
// never a repair (handActVerbs).
|
||||
const handActWarrant = "warrant"
|
||||
|
||||
// byWords is who chose, as the hand-act log says it: "the operator, as telegram identity 42".
|
||||
func byWords(w asks.Warrant) string {
|
||||
if w.By == nil {
|
||||
return "the operator"
|
||||
}
|
||||
return fmt.Sprintf("the %s, as %s identity %s", w.By.Who, w.By.Kind, w.By.Identity)
|
||||
}
|
||||
|
||||
// viaWords is the channel an answer came through: its module and kind, and how the sender was known.
|
||||
func viaWords(w asks.Warrant) string {
|
||||
if w.By == nil {
|
||||
return w.Channel
|
||||
}
|
||||
via := w.Channel + " (" + w.By.Kind + ")"
|
||||
if w.By.Verified != "" {
|
||||
via += ", " + w.By.Verified
|
||||
}
|
||||
return via
|
||||
}
|
||||
|
||||
// errNotGranted is an action whose verb the controller's grant does not name.
|
||||
var errNotGranted = errors.New("the controller's grant does not name this verb")
|
||||
@@ -1,153 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// busAsker is an asker on a real bus's `asked` bucket, counting what it performs: two of them are two
|
||||
// controllers sharing one record.
|
||||
type busAskerRig struct {
|
||||
mu sync.Mutex
|
||||
called int
|
||||
acts int
|
||||
open []conditions.Condition
|
||||
sent [][]byte
|
||||
}
|
||||
|
||||
func (rig *busAskerRig) asker(t *testing.T, conn *nats.Conn, now time.Time) *asker {
|
||||
return &asker{
|
||||
open: func(context.Context) ([]conditions.Condition, error) {
|
||||
rig.mu.Lock()
|
||||
defer rig.mu.Unlock()
|
||||
return rig.open, nil
|
||||
},
|
||||
silence: func(context.Context, string, time.Duration, string, string) error { return nil },
|
||||
store: busAsked{conn: conn},
|
||||
publish: func(_ context.Context, subject string, body []byte, _ string) error {
|
||||
rig.mu.Lock()
|
||||
defer rig.mu.Unlock()
|
||||
if subject == asks.AskSubject(askerName) {
|
||||
rig.sent = append(rig.sent, body)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
call: func(context.Context, conditions.Action, map[string]string) error {
|
||||
time.Sleep(20 * time.Millisecond) // long enough for the other delivery to arrive meanwhile
|
||||
rig.mu.Lock()
|
||||
defer rig.mu.Unlock()
|
||||
rig.called++
|
||||
return nil
|
||||
},
|
||||
record: func(context.Context, link.HandAct) error {
|
||||
rig.mu.Lock()
|
||||
defer rig.mu.Unlock()
|
||||
rig.acts++
|
||||
return nil
|
||||
},
|
||||
now: func() time.Time { return now },
|
||||
logf: t.Logf,
|
||||
}
|
||||
}
|
||||
|
||||
func askedBus(t *testing.T) *nats.Conn {
|
||||
t.Helper()
|
||||
conn, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(conn.Close)
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.CreateKeyValue(context.Background(), jetstream.KeyValueConfig{Bucket: broker.AskedBucket}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return conn
|
||||
}
|
||||
|
||||
// The review of 2026-10-09 (L7): two deliveries of one warrant, to two controllers at once, perform its act
|
||||
// exactly once and record it once — the record's compare-and-set decides, never the warrant's message id.
|
||||
func TestTwoAnswersAtOnceActOnce(t *testing.T) {
|
||||
conn := askedBus(t)
|
||||
now := time.Date(2026, 10, 9, 14, 0, 0, 0, time.UTC)
|
||||
rig := &busAskerRig{open: []conditions.Condition{heldCondition()}}
|
||||
first, second := rig.asker(t, conn, now), rig.asker(t, conn, now)
|
||||
if err := first.reconcile(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rig.sent) != 1 {
|
||||
t.Fatalf("asked %d times", len(rig.sent))
|
||||
}
|
||||
var q asks.Ask
|
||||
_ = json.Unmarshal(rig.sent[0], &q)
|
||||
release, _ := q.Option("release")
|
||||
w := asks.Warrant{Ask: q.ID, Asker: askerName, About: q.About, Outcome: asks.OutcomeChosen, Option: release.ID,
|
||||
Label: release.Label, Level: release.Level, Channel: "telegram", Proofs: []string{"P1"}, At: now,
|
||||
AskDigest: q.Digest(), By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}}
|
||||
body, _ := json.Marshal(w)
|
||||
var wg sync.WaitGroup
|
||||
for _, a := range []*asker{first, second, first, second} {
|
||||
wg.Add(1)
|
||||
go func(a *asker) {
|
||||
defer wg.Done()
|
||||
if err := a.Decided(context.Background(), body); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}(a)
|
||||
}
|
||||
wg.Wait()
|
||||
if rig.called != 1 || rig.acts != 1 {
|
||||
t.Fatalf("performed %d time(s), recorded %d time(s)", rig.called, rig.acts)
|
||||
}
|
||||
got, err := busAsked{conn: conn}.Get(context.Background(), q.ID)
|
||||
if err != nil || got == nil || got.Acted != "done" {
|
||||
t.Fatalf("kept as %+v (%v)", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The review of 2026-10-09 (L2): a write decided on a record read earlier never lands over one made since. A
|
||||
// cancel read before the answer was acted on leaves the act's record as it is.
|
||||
func TestAStaleCancelDoesNotWriteOverAnAct(t *testing.T) {
|
||||
conn := askedBus(t)
|
||||
now := time.Date(2026, 10, 9, 14, 0, 0, 0, time.UTC)
|
||||
rig := &busAskerRig{open: []conditions.Condition{heldCondition()}}
|
||||
a := rig.asker(t, conn, now)
|
||||
if err := a.reconcile(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var q asks.Ask
|
||||
_ = json.Unmarshal(rig.sent[0], &q)
|
||||
stale, _ := busAsked{conn: conn}.Get(context.Background(), q.ID)
|
||||
release, _ := q.Option("release")
|
||||
w := asks.Warrant{Ask: q.ID, Asker: askerName, About: q.About, Outcome: asks.OutcomeChosen, Option: release.ID,
|
||||
Label: release.Label, Level: release.Level, Channel: "telegram", At: now, AskDigest: q.Digest(),
|
||||
By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}}
|
||||
body, _ := json.Marshal(w)
|
||||
if err := a.Decided(context.Background(), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := a.cancel(context.Background(), *stale, "the condition ended"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ := busAsked{conn: conn}.Get(context.Background(), q.ID)
|
||||
if got.State != string(asks.OutcomeChosen) || got.Acted != "done" {
|
||||
t.Errorf("a stale cancel wrote over the act: %+v", got)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,589 +0,0 @@
|
||||
package main
|
||||
|
||||
// The asker on the bus: its asks in the controller's bucket `asked`, its asks and cancels published on the
|
||||
// seat under the controller's name, the verbs a warrant chooses called with the controller's grant, and the
|
||||
// router's record of its asks read under its name (novox/hq ADR 0259).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// askerFrom is the serving controller's asker; nil in any other process.
|
||||
var askerFrom *asker
|
||||
|
||||
// askWithin is how long a verb a warrant chose is given to answer.
|
||||
const askWithin = time.Minute
|
||||
|
||||
type busAsked struct{ conn *nats.Conn }
|
||||
|
||||
func (b busAsked) kv(ctx context.Context) (jetstream.KeyValue, error) {
|
||||
js, err := jetstream.New(b.conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return js.KeyValue(ctx, broker.AskedBucket)
|
||||
}
|
||||
|
||||
func (b busAsked) Get(ctx context.Context, id string) (*asked, error) {
|
||||
kv, err := b.kv(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e, err := kv.Get(ctx, id)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var r asked
|
||||
return &r, json.Unmarshal(e.Value(), &r)
|
||||
}
|
||||
|
||||
// Create keeps a new ask under its id, and only where none is kept: never over another.
|
||||
func (b busAsked) Create(ctx context.Context, r asked) error {
|
||||
kv, err := b.kv(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = kv.Create(ctx, r.ID, body)
|
||||
return err
|
||||
}
|
||||
|
||||
// Change applies change to the ask kept under id by compare-and-set on its key's revision (the review of
|
||||
// 2026-10-09, L2): read, changed, and written only over the revision read; when another write came between,
|
||||
// read again and asked again, at most askChangeTries times. change says whether to write at all.
|
||||
func (b busAsked) Change(ctx context.Context, id string, change func(*asked) bool) (bool, error) {
|
||||
kv, err := b.kv(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for try := 0; try < askChangeTries; try++ {
|
||||
e, err := kv.Get(ctx, id)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
var r asked
|
||||
if err := json.Unmarshal(e.Value(), &r); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !change(&r) {
|
||||
return false, nil
|
||||
}
|
||||
body, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if _, err := kv.Update(ctx, id, body, e.Revision()); err != nil {
|
||||
var api *jetstream.APIError
|
||||
if errors.Is(err, jetstream.ErrKeyExists) || (errors.As(err, &api) && api.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence) {
|
||||
continue
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
return false, fmt.Errorf("the ask %s changed under every one of %d tries", id, askChangeTries)
|
||||
}
|
||||
|
||||
func (b busAsked) All(ctx context.Context) ([]asked, error) {
|
||||
kv, err := b.kv(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lister, err := kv.ListKeys(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = lister.Stop() }()
|
||||
var out []asked
|
||||
for k := range lister.Keys() {
|
||||
e, err := kv.Get(ctx, k)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var r asked
|
||||
if json.Unmarshal(e.Value(), &r) == nil {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// callAction performs an action's verb as the controller, through the grant that names it.
|
||||
func callAction(conn *nats.Conn) func(ctx context.Context, a conditions.Action, args map[string]string) error {
|
||||
return func(ctx context.Context, a conditions.Action, args map[string]string) error {
|
||||
seat, verb, ok := strings.Cut(a.Verb, ".")
|
||||
if !ok {
|
||||
return fmt.Errorf("%q names no seat and verb", a.Verb)
|
||||
}
|
||||
body := map[string]any{}
|
||||
for k, v := range args {
|
||||
body[k] = v
|
||||
}
|
||||
if seat == catalogue.DeliverySeat {
|
||||
_, err := askDeliveryOwner(ctx, conn, verb, body)
|
||||
return err
|
||||
}
|
||||
granted := false
|
||||
for _, v := range broker.VerbsTheControllerActsOnAWarrant {
|
||||
granted = granted || (v.Seat == seat && v.Verb == verb)
|
||||
}
|
||||
if !granted {
|
||||
return fmt.Errorf("%s: %w", a.Verb, errNotGranted)
|
||||
}
|
||||
var answer link.Answer
|
||||
var err error
|
||||
if a.Machine != "" {
|
||||
answer, err = link.AskSeatTool(ctx, conn, seat, verb, a.Machine, body, askWithin)
|
||||
} else {
|
||||
answer, err = link.AskMeshSeatTool(ctx, conn, seat, verb, body, askWithin)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return fmt.Errorf("%s refused: %s", a.Verb, answer.Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// routerRecordOf reads the router's record of one of the controller's asks, under its name, and answers
|
||||
// how it ended when it did: the bucket is the one the asks seat's declarer names as its records.
|
||||
func routerRecordOf(conn *nats.Conn, inv *inventory.Inventory) func(ctx context.Context, id string) (*asks.Warrant, error) {
|
||||
return func(ctx context.Context, id string) (*asks.Warrant, error) {
|
||||
bucket, err := asksRecords(ctx, inv)
|
||||
if err != nil || bucket == "" {
|
||||
return nil, err
|
||||
}
|
||||
reply, err := conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET.KV_"+bucket+".$KV."+bucket+"."+askerName+"."+id, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if reply.Header.Get("Status") != "" {
|
||||
return nil, nil // none, or not readable: the event says it
|
||||
}
|
||||
var rec struct {
|
||||
State string `json:"state"`
|
||||
Warrant *asks.Warrant `json:"warrant"`
|
||||
}
|
||||
if json.Unmarshal(reply.Data, &rec) != nil || rec.State == "open" || rec.Warrant == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return rec.Warrant, nil
|
||||
}
|
||||
}
|
||||
|
||||
// asksRecords is the bucket the asks seat's declarer keeps its record of asks in.
|
||||
func asksRecords(ctx context.Context, inv *inventory.Inventory) (string, error) {
|
||||
declared, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, m := range declared {
|
||||
for _, s := range m.DefinesSeats {
|
||||
if s.Name == broker.AsksSeat && len(s.Records) > 0 {
|
||||
return broker.BucketName(m.Module, s.Records[0]), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// routerHereIn says whether a module declaring the asks seat, with its ask named by its caller, is assigned:
|
||||
// without it nothing takes an ask, and asking would only fill a queue nobody reads.
|
||||
func routerHereIn(inv *inventory.Inventory) func(ctx context.Context) (bool, error) {
|
||||
return func(ctx context.Context) (bool, error) {
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
for _, s := range e.Manifest.DefinesSeats {
|
||||
if s.Name == broker.AsksSeat && s.NamedByCaller("ask") && len(e.On) > 0 {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
// grantHeldIn says whether the bus holds the controller's grant to ask (novox/hq issue 353), judged on the two
|
||||
// facts that make it so, and failing closed on each:
|
||||
//
|
||||
// 1. **composed**: the user list the mesh composes now grants the controller itself to publish its ask
|
||||
// (controllerMayAsk) — the controller's own grant, not whether any user of the list changed;
|
||||
// 2. **in force**: the bus's own module, on the machine holding the bus, says the user list there allows it
|
||||
// and the server reloaded after that list was written (the nats module's `nats_user_can`, `in_force`) — a
|
||||
// list sent is not a list the bus enforces until it reloads.
|
||||
//
|
||||
// Anything that cannot be worked out is not held, with why, so the conditions that need the operator are said
|
||||
// as undelivered rather than asked into a refusal.
|
||||
func grantHeldIn(open *stores, conn *nats.Conn) func(ctx context.Context) (bool, string, error) {
|
||||
return grantHeldBy(composedGrantIn(open), busGrantIn(open, conn), time.Now)
|
||||
}
|
||||
|
||||
// grantJudge is one of the two facts: whether it holds, and why not.
|
||||
type grantJudge func(ctx context.Context) (bool, string, error)
|
||||
|
||||
// grantHeldBy is the judgement over the two facts, remembered for grantLookEvery: composing the list resolves
|
||||
// the bus's machine whole, and asking the bus's module is a call.
|
||||
func grantHeldBy(composed, inForce grantJudge, now func() time.Time) func(ctx context.Context) (bool, string, error) {
|
||||
var mu sync.Mutex
|
||||
var at time.Time
|
||||
var held bool
|
||||
var why string
|
||||
return func(ctx context.Context) (bool, string, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if !at.IsZero() && now().Sub(at) < grantLookEvery {
|
||||
return held, why, nil
|
||||
}
|
||||
held, why = judgeGrant(ctx, composed, inForce)
|
||||
at = now()
|
||||
return held, why, nil
|
||||
}
|
||||
}
|
||||
|
||||
// judgeGrant is held only when both facts hold; an error is a fact not known, never a yes. A fact that holds
|
||||
// with a caveat (a reload not confirmed) gives that caveat as the why of a yes.
|
||||
func judgeGrant(ctx context.Context, composed, inForce grantJudge) (bool, string) {
|
||||
var caveat []string
|
||||
for _, fact := range []struct {
|
||||
judge grantJudge
|
||||
what string
|
||||
}{{composed, "whether the mesh composes the controller a grant to ask"}, {inForce, "whether the bus enforces it"}} {
|
||||
if fact.judge == nil {
|
||||
return false, fact.what + " is not judged here"
|
||||
}
|
||||
ok, why, err := fact.judge(ctx)
|
||||
if err != nil {
|
||||
return false, fact.what + " could not be worked out: " + err.Error()
|
||||
}
|
||||
if !ok {
|
||||
return false, why
|
||||
}
|
||||
if why != "" {
|
||||
caveat = append(caveat, why)
|
||||
}
|
||||
}
|
||||
return true, strings.Join(caveat, "; ")
|
||||
}
|
||||
|
||||
// controllerMayAsk says whether the controller's own grant, as composed from these records, lets it publish its
|
||||
// ask (ADR 0259 §3: composed while a router that takes asks under its asker's name is assigned).
|
||||
func controllerMayAsk(records broker.Records) (bool, error) {
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, u := range users {
|
||||
if u.Kind != broker.KindController {
|
||||
continue
|
||||
}
|
||||
perms, err := broker.PermissionsFor(u)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return broker.MayPublish(perms, asks.AskSubject(askerName)), nil
|
||||
}
|
||||
return false, errors.New("the composition holds no controller")
|
||||
}
|
||||
|
||||
// composedGrantIn is the first fact, from the mesh's records.
|
||||
func composedGrantIn(open *stores) grantJudge {
|
||||
return func(ctx context.Context) (bool, string, error) {
|
||||
if open == nil || open.inventory == nil {
|
||||
return false, "", errors.New("the mesh's records are not open")
|
||||
}
|
||||
records, err := open.inventory.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
may, err := controllerMayAsk(records)
|
||||
if err != nil || may {
|
||||
return may, "", err
|
||||
}
|
||||
return false, "the mesh composes the controller no grant to ask: no router that takes asks under its " +
|
||||
"asker's name is assigned", nil
|
||||
}
|
||||
}
|
||||
|
||||
// busGrantIn is the second fact, asked of the bus's own module on the machine holding the bus, with the check
|
||||
// main made before it (novox/hq issue 353, PR 186: is the user list sent to the bus's machine the one composed
|
||||
// now) kept for the cases the module's word does not settle.
|
||||
func busGrantIn(open *stores, conn *nats.Conn) grantJudge {
|
||||
ask := func(ctx context.Context) (string, json.RawMessage, error) {
|
||||
if open == nil || open.inventory == nil || conn == nil {
|
||||
return "", nil, errors.New("the mesh's records or the bus are not open")
|
||||
}
|
||||
holders, err := seatHolders(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
h, held := holders[theBrokerSeat]
|
||||
if !held || h.Node == "" || h.Module == "" {
|
||||
return "", nil, nil
|
||||
}
|
||||
a, err := link.AskModuleToolOn(ctx, conn, h.Module, busUserCanTool, h.Node, map[string]any{
|
||||
"user": broker.ControllerName, "action": "publish", "subject": asks.AskSubject(askerName)}, 15*time.Second)
|
||||
if err != nil {
|
||||
return h.Node, nil, err
|
||||
}
|
||||
if a.Error != "" {
|
||||
return h.Node, nil, errors.New(a.Error)
|
||||
}
|
||||
return h.Node, a.Result, nil
|
||||
}
|
||||
list := func(ctx context.Context) (busListState, error) {
|
||||
if open == nil || open.inventory == nil {
|
||||
return busListState{}, errors.New("the mesh's records are not open")
|
||||
}
|
||||
s, err := brokerListState(ctx, open, nil)
|
||||
if err != nil || s.Machine == "" || !s.Behind {
|
||||
return s, err
|
||||
}
|
||||
step, err := busHeld(ctx, open.inventory, []string{s.Machine})
|
||||
if err != nil {
|
||||
return busListState{}, err
|
||||
}
|
||||
s.BusStep = step[s.Machine]
|
||||
return s, nil
|
||||
}
|
||||
return busGrantBy(ask, list)
|
||||
}
|
||||
|
||||
// busListState is the user list on the bus's machine as the controller's record knows it: whether the list
|
||||
// sent there is behind the one composed now, why it could not be composed, and the bus step a send there waits
|
||||
// on (that machine cannot be pushed until a person's `bus upgrade`).
|
||||
type busListState struct {
|
||||
Machine string
|
||||
Behind bool
|
||||
Uncomposed error
|
||||
BusStep string
|
||||
}
|
||||
|
||||
// remedy is the one thing that sends the bus's machine the list composed now: the plan's own reason when it
|
||||
// cannot be composed, the bus upgrade while sends there wait on it, and otherwise a push of that machine.
|
||||
func (s busListState) remedy(node string) string {
|
||||
if s.Machine != "" {
|
||||
node = s.Machine
|
||||
}
|
||||
switch {
|
||||
case s.Uncomposed != nil:
|
||||
return fmt.Sprintf("the user list for %s cannot be composed (%v); `plan %s` says what to fix first",
|
||||
node, s.Uncomposed, node)
|
||||
case s.BusStep != "":
|
||||
return fmt.Sprintf("sends to %s wait on the bus's planned step, so a push will not carry it: %s", node, s.BusStep)
|
||||
}
|
||||
return fmt.Sprintf("`push %s` carries it", node)
|
||||
}
|
||||
|
||||
// busGrantBy judges the bus's word, and names for each way it is not held the one thing that makes it held:
|
||||
//
|
||||
// - **in force**: held;
|
||||
// - **allowed, from a module older than `in_force`**: held, with the caveat that the reload is not confirmed.
|
||||
// The file allows the publish, and the bus reloads within about 5 s of a change to it, so the worst case is
|
||||
// one ask refused in those seconds, which is asked again (issue 369). The list check is not consulted:
|
||||
// while a bus step is pending the bus's machine cannot be pushed, so any change to the composed list would
|
||||
// read as behind for good and silence every ask until a person's bus upgrade (the review of PR 188);
|
||||
// - **allowed, written, not reloaded** (the module gives both times): wait about 5 s;
|
||||
// - **allowed, in force not known** (the module could not read a time): what is not known, said;
|
||||
// - **not allowed, or the module cannot answer**: the list check decides — the remedy that sends the list
|
||||
// when it is behind (busListState.remedy), and, for a module that cannot answer, held while the list sent
|
||||
// is current, as main judged it.
|
||||
//
|
||||
// ask answers the machine holding the bus and its module's answer ("" when nothing holds the bus); list is the
|
||||
// record's word on the list sent to that machine.
|
||||
func busGrantBy(ask func(ctx context.Context) (string, json.RawMessage, error),
|
||||
list func(ctx context.Context) (busListState, error)) grantJudge {
|
||||
return func(ctx context.Context) (bool, string, error) {
|
||||
node, raw, askErr := ask(ctx)
|
||||
if askErr == nil && node == "" {
|
||||
return false, "no module holds the bus, so nothing says what it enforces", nil
|
||||
}
|
||||
var answer busAnswer
|
||||
if askErr == nil {
|
||||
if answer, askErr = readBusAnswer(raw); askErr != nil {
|
||||
askErr = fmt.Errorf("its answer could not be read: %w", askErr)
|
||||
}
|
||||
}
|
||||
if askErr != nil {
|
||||
if node == "" {
|
||||
node = "the bus's machine"
|
||||
}
|
||||
s, err := list(ctx)
|
||||
if err != nil {
|
||||
return false, "", fmt.Errorf("the bus module on %s could not answer (%v), and whether the list sent "+
|
||||
"there is the one composed now could not be worked out: %w", node, askErr, err)
|
||||
}
|
||||
if s.Behind {
|
||||
return false, fmt.Sprintf("the bus module on %s could not answer (%v), and the user list sent there is "+
|
||||
"behind what the mesh composes; %s", node, askErr, s.remedy(node)), nil
|
||||
}
|
||||
return true, fmt.Sprintf("the bus module on %s could not answer (%v); judged by the user list sent there, "+
|
||||
"which is the one composed now", node, askErr), nil
|
||||
}
|
||||
notAllowed := func(older string) (bool, string, error) {
|
||||
s, err := list(ctx)
|
||||
if err != nil {
|
||||
return false, "", fmt.Errorf("the user list on %s does not allow the controller to ask%s, and whether "+
|
||||
"the list sent there is the one composed now could not be worked out: %w", node, older, err)
|
||||
}
|
||||
if s.Behind {
|
||||
return false, fmt.Sprintf("the user list on %s does not allow the controller to ask%s: it is behind "+
|
||||
"what the mesh composes; %s", node, older, s.remedy(node)), nil
|
||||
}
|
||||
return false, fmt.Sprintf("the user list on %s does not allow the controller to ask%s, though the list "+
|
||||
"sent there is the one composed now; %s again", node, older, s.remedy(node)), nil
|
||||
}
|
||||
switch {
|
||||
case answer.InForce == nil && answer.Allowed:
|
||||
return true, fmt.Sprintf("the user list on %s allows the controller to ask, but its reload is not "+
|
||||
"confirmed: the bus module there is older than the answer whether a grant is in force, and the bus "+
|
||||
"upgrade (`bus upgrade`) carries one that confirms it", node), nil
|
||||
case answer.InForce == nil:
|
||||
return notAllowed(" (the bus module there is older than the answer whether a grant is in force)")
|
||||
case !answer.Allowed:
|
||||
return notAllowed("")
|
||||
case *answer.InForce:
|
||||
return true, "", nil
|
||||
case answer.FileWritten != "" && answer.ServerLoaded != "":
|
||||
return false, fmt.Sprintf("the bus on %s has the controller's grant to ask written (%s) but has not "+
|
||||
"reloaded since (last load %s); it reloads in place within about 5 s, so nothing is to be done but wait",
|
||||
node, answer.FileWritten, answer.ServerLoaded), nil
|
||||
}
|
||||
why := answer.Why
|
||||
if why == "" {
|
||||
why = "it gives no reason"
|
||||
}
|
||||
return false, fmt.Sprintf("the user list on %s allows the controller to ask, but whether the bus enforces it "+
|
||||
"is not known: %s", node, why), nil
|
||||
}
|
||||
}
|
||||
|
||||
// busUserCanTool is the bus module's answer to whether a user may do something, and whether it is in force.
|
||||
const busUserCanTool = "nats_user_can"
|
||||
|
||||
// busAnswer is the bus module's answer to busUserCanTool; InForce is nil from a module older than that field,
|
||||
// and FileWritten and ServerLoaded are given only when it could read both times (mesh-catalog's nats InForce).
|
||||
type busAnswer struct {
|
||||
Allowed bool `json:"allowed"`
|
||||
InForce *bool `json:"in_force"`
|
||||
Why string `json:"why"`
|
||||
FileWritten string `json:"file_written"`
|
||||
ServerLoaded string `json:"server_loaded"`
|
||||
}
|
||||
|
||||
func readBusAnswer(raw json.RawMessage) (busAnswer, error) {
|
||||
var answer busAnswer
|
||||
err := json.Unmarshal(raw, &answer)
|
||||
return answer, err
|
||||
}
|
||||
|
||||
// grantLookEvery is how often the bus's user list is judged against the one its machine was last sent.
|
||||
const grantLookEvery = 30 * time.Second
|
||||
|
||||
// channelsIn is what the channels are now, as a fingerprint: each module claiming a kind of the channel
|
||||
// bench, where, promising what, and whether of its own account. An ask the router refused is asked again
|
||||
// once this changes.
|
||||
func channelsIn(inv *inventory.Inventory) func(ctx context.Context) string {
|
||||
return func(ctx context.Context) string {
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
var parts []string
|
||||
for _, e := range entries {
|
||||
for _, c := range e.Manifest.Claims {
|
||||
if c.Kind == "" || !catalogue.KindedBenches[c.Name] {
|
||||
continue
|
||||
}
|
||||
on := append([]string(nil), e.On...)
|
||||
sort.Strings(on)
|
||||
caps := append([]string(nil), c.Capabilities...)
|
||||
sort.Strings(caps)
|
||||
parts = append(parts, fmt.Sprintf("%s/%s=%s@%s[%s]own:%t", c.Name, c.Kind, e.Manifest.Module,
|
||||
strings.Join(on, ","), strings.Join(caps, ","), e.Manifest.RunsAs != ""))
|
||||
}
|
||||
}
|
||||
sort.Strings(parts)
|
||||
return strings.Join(parts, ";")
|
||||
}
|
||||
}
|
||||
|
||||
// startAsking makes the serving controller's asker and hands it the router's words.
|
||||
func startAsking(ctx context.Context, open *stores, server *link.Server, conn *nats.Conn, keeper *conditions.Keeper) {
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
fmt.Printf("the operator cannot be asked: %v\n", err)
|
||||
return
|
||||
}
|
||||
a := servingAsker(open, conn, js, keeper)
|
||||
if err := server.Decides(a); err != nil {
|
||||
fmt.Printf("the operator's answers cannot be heard, so nothing is asked: %v\n", err)
|
||||
return
|
||||
}
|
||||
askerFrom = a
|
||||
go a.keep(ctx)
|
||||
}
|
||||
|
||||
// servingAsker is the serving controller's asker, wired to the mesh: what it reads, publishes, performs and
|
||||
// records, and the judgements it asks by — the bus's grant to ask among them (novox/hq issue 353).
|
||||
func servingAsker(open *stores, conn *nats.Conn, js jetstream.JetStream, keeper *conditions.Keeper) *asker {
|
||||
return &asker{
|
||||
open: keeper.Open,
|
||||
silence: func(ctx context.Context, key string, d time.Duration, by, why string) error {
|
||||
_, err := keeper.Silence(ctx, key, d, by, why)
|
||||
return err
|
||||
},
|
||||
store: busAsked{conn: conn},
|
||||
publish: func(ctx context.Context, subject string, body []byte, id string) error {
|
||||
_, err := js.Publish(ctx, subject, body, jetstream.WithMsgID(id))
|
||||
return err
|
||||
},
|
||||
call: callAction(conn),
|
||||
// A proposed settings layer is set by this controller itself on the warrant (novox/hq ADR 0277).
|
||||
setLayer: setLayerIn(open),
|
||||
record: func(ctx context.Context, act link.HandAct) error {
|
||||
_, err := link.RecordHandAct(ctx, conn, act)
|
||||
return err
|
||||
},
|
||||
routerRecord: routerRecordOf(conn, open.inventory),
|
||||
routerHere: routerHereIn(open.inventory),
|
||||
grantHeld: grantHeldIn(open, conn),
|
||||
channels: channelsIn(open.inventory),
|
||||
raise: func(ctx context.Context, obs []conditions.Observation) error {
|
||||
return keeper.Reconcile(ctx, sourceAsker, obs)
|
||||
},
|
||||
now: time.Now,
|
||||
logf: func(format string, args ...any) { fmt.Printf(format+"\n", args...) },
|
||||
}
|
||||
}
|
||||
@@ -1,182 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
func TestAModuleAssignedAndNotPushedIsAwaitingAPushNotUncomposable(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop") // the machine was pushed before; then the module is assigned
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The read the rest of the mesh asks still refuses it, with the composer's typed error: nothing
|
||||
// can be compared about a secret that does not exist (status), and nothing here string-matches.
|
||||
plan, settings, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = declarationWith(ctx, open, "laptop", plan, settings, gens, Reading)
|
||||
var notMade *catalogue.NotMadeError
|
||||
if !errors.As(err, ¬Made) || notMade.Module != "keeper" || notMade.Name != "repository" {
|
||||
t.Fatalf("a read composition did not say which secret is not made, typed: %v", err)
|
||||
}
|
||||
// Foreseen, it composes, names what the push makes, and made nothing.
|
||||
foreseen, err := declarationWith(ctx, open, "laptop", plan, settings, gens, Foreseeing)
|
||||
if err != nil || len(foreseen.foreseen) != 1 || foreseen.foreseen[0] != "keeper/repository" {
|
||||
t.Fatalf("foreseen: %v %v", foreseen.foreseen, err)
|
||||
}
|
||||
if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "keeper", "repository"); err != nil || held {
|
||||
t.Fatalf("asking ahead of the push made the secret (held %v, %v)", held, err)
|
||||
}
|
||||
|
||||
// Within the bound: nothing at all — no urgent, no warning, nobody notified.
|
||||
if got := d1(t, open); len(got) != 0 {
|
||||
t.Fatalf("a machine waiting for a push raised %+v", got)
|
||||
}
|
||||
|
||||
// Past the bound: a warning, not urgent, saying what the push will make.
|
||||
before := awaitingPushBound
|
||||
awaitingPushBound = -time.Minute
|
||||
t.Cleanup(func() { awaitingPushBound = before })
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.awaiting-push" || got[0].Severity != conditions.Warning ||
|
||||
!strings.Contains(got[0].Summary, "keeper/repository") || !strings.Contains(got[0].Summary, "push laptop") {
|
||||
t.Fatalf("a machine left un-pushed past the bound: %+v", got)
|
||||
}
|
||||
|
||||
// Pushed: the secret is made and D1 says nothing.
|
||||
pushedBy(t, open, "laptop")
|
||||
if got := d1(t, open); len(got) != 0 {
|
||||
t.Fatalf("a pushed machine still raised %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASecretThePushCannotMakeIsStillUncomposable(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aTalker())
|
||||
if _, err := assign(ctx, open, "laptop", "talker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || got[0].Severity != conditions.Urgent ||
|
||||
!strings.Contains(got[0].Summary, "module issue talker --node laptop") {
|
||||
t.Fatalf("a push that will be refused was not said urgently: %+v", got)
|
||||
}
|
||||
|
||||
// And a machine whose composition fails for anything else, with a secret waiting beside it, is
|
||||
// uncomposable for that — the stand-in hides nothing.
|
||||
register(t, open, aKeeper())
|
||||
if _, err := assign(ctx, open, "anchor", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
one, two := rivals()
|
||||
register(t, open, one)
|
||||
register(t, open, two)
|
||||
_, _ = assign(ctx, open, "anchor", "rival-one")
|
||||
_, _ = assign(ctx, open, "anchor", "rival-two")
|
||||
keys := map[string]conditions.Severity{}
|
||||
for _, o := range d1(t, open) {
|
||||
keys[o.Key()] = o.Severity
|
||||
}
|
||||
if keys["machine.anchor.uncomposable"] != conditions.Urgent {
|
||||
t.Fatalf("a real failure beside a waiting secret: %v", keys)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAGivenSecretUnderAnOldKeyIsNotForeseen(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "keeper", "repository", "given"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, err := open.inventory.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "issue it again") {
|
||||
t.Fatalf("a given secret under an old key: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMachineAwaitsAPushSinceItsOldestAssignmentSinceTheLastSend(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop")
|
||||
sent := time.Now()
|
||||
since, err := open.inventory.AwaitingSince(ctx, "laptop")
|
||||
if err != nil || since.After(sent) {
|
||||
t.Fatalf("nothing assigned since the send: waiting since %v (%v), the send was before %v", since, err, sent)
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
since, err = open.inventory.AwaitingSince(ctx, "laptop")
|
||||
if err != nil || since.Before(sent.Add(-time.Second)) {
|
||||
t.Fatalf("assigned after the send: waiting since %v (%v), not from the assignment", since, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestALastSendIsReadFromTheSendAndTheReport(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop")
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sends, err := readDeliveries(ctx, open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatal("a holder assigned and not pushed is expected to answer")
|
||||
}
|
||||
if !sends["laptop"].settled(overlay.Name, time.Now().Add(time.Hour)) {
|
||||
t.Fatal("a module the machine was sent long ago is not expected to answer")
|
||||
}
|
||||
pushedBy(t, open, "laptop")
|
||||
sends, err = readDeliveries(ctx, open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatal("pushed a moment ago and not reported, a holder is already expected")
|
||||
}
|
||||
if !sends["laptop"].settled("keeper", time.Now().Add(reportGrace+time.Minute)) {
|
||||
t.Fatal("pushed past the grace, a holder is not expected")
|
||||
}
|
||||
if _, ok := sends["laptop"].carried["keeper"]; !ok {
|
||||
t.Fatalf("the send's builds do not carry keeper: %v", sends["laptop"].carried)
|
||||
}
|
||||
pushedAndApplied(t, open, "laptop")
|
||||
if sends, err = readDeliveries(ctx, open.inventory); err != nil || !sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatalf("pushed and reported applied, a holder is not expected to answer (%v)", err)
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,15 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// Between `assign` and `push` a module's own secrets are not made yet: the push makes them. D1 composed
|
||||
@@ -78,14 +81,140 @@ func d1(t *testing.T, open *stores) []conditions.Observation {
|
||||
|
||||
// **THE WINDOW, REPRODUCED**: assigned and not pushed, a module whose own secret the push makes is
|
||||
// waiting, not uncomposable; past the bound it is a warning naming what the push makes; pushed, nothing.
|
||||
func TestAModuleAssignedAndNotPushedIsAwaitingAPushNotUncomposable(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop") // the machine was pushed before; then the module is assigned
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The read the rest of the mesh asks still refuses it, with the composer's typed error: nothing
|
||||
// can be compared about a secret that does not exist (status), and nothing here string-matches.
|
||||
plan, settings, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = declarationWith(ctx, open, "laptop", plan, settings, gens, Reading)
|
||||
var notMade *catalogue.NotMadeError
|
||||
if !errors.As(err, ¬Made) || notMade.Module != "keeper" || notMade.Name != "repository" {
|
||||
t.Fatalf("a read composition did not say which secret is not made, typed: %v", err)
|
||||
}
|
||||
// Foreseen, it composes, names what the push makes, and made nothing.
|
||||
foreseen, err := declarationWith(ctx, open, "laptop", plan, settings, gens, Foreseeing)
|
||||
if err != nil || len(foreseen.foreseen) != 1 || foreseen.foreseen[0] != "keeper/repository" {
|
||||
t.Fatalf("foreseen: %v %v", foreseen.foreseen, err)
|
||||
}
|
||||
if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "keeper", "repository"); err != nil || held {
|
||||
t.Fatalf("asking ahead of the push made the secret (held %v, %v)", held, err)
|
||||
}
|
||||
|
||||
// Within the bound: nothing at all — no urgent, no warning, nobody notified.
|
||||
if got := d1(t, open); len(got) != 0 {
|
||||
t.Fatalf("a machine waiting for a push raised %+v", got)
|
||||
}
|
||||
|
||||
// Past the bound: a warning, not urgent, saying what the push will make.
|
||||
before := awaitingPushBound
|
||||
awaitingPushBound = -time.Minute
|
||||
t.Cleanup(func() { awaitingPushBound = before })
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.awaiting-push" || got[0].Severity != conditions.Warning ||
|
||||
!strings.Contains(got[0].Summary, "keeper/repository") || !strings.Contains(got[0].Summary, "push laptop") {
|
||||
t.Fatalf("a machine left un-pushed past the bound: %+v", got)
|
||||
}
|
||||
|
||||
// Pushed: the secret is made and D1 says nothing.
|
||||
pushedBy(t, open, "laptop")
|
||||
if got := d1(t, open); len(got) != 0 {
|
||||
t.Fatalf("a pushed machine still raised %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **A REAL FAILURE IS STILL URGENT**: a secret the push would be refused on is not one it will make.
|
||||
// A bus credential nobody issued (issue 203) fails the push, so D1 says it — urgent, in the push's words.
|
||||
func TestASecretThePushCannotMakeIsStillUncomposable(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aTalker())
|
||||
if _, err := assign(ctx, open, "laptop", "talker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || got[0].Severity != conditions.Urgent ||
|
||||
!strings.Contains(got[0].Summary, "module issue talker --node laptop") {
|
||||
t.Fatalf("a push that will be refused was not said urgently: %+v", got)
|
||||
}
|
||||
|
||||
// And a machine whose composition fails for anything else, with a secret waiting beside it, is
|
||||
// uncomposable for that — the stand-in hides nothing.
|
||||
register(t, open, aKeeper())
|
||||
if _, err := assign(ctx, open, "anchor", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
one, two := rivals()
|
||||
register(t, open, one)
|
||||
register(t, open, two)
|
||||
_, _ = assign(ctx, open, "anchor", "rival-one")
|
||||
_, _ = assign(ctx, open, "anchor", "rival-two")
|
||||
keys := map[string]conditions.Severity{}
|
||||
for _, o := range d1(t, open) {
|
||||
keys[o.Key()] = o.Severity
|
||||
}
|
||||
if keys["machine.anchor.uncomposable"] != conditions.Urgent {
|
||||
t.Fatalf("a real failure beside a waiting secret: %v", keys)
|
||||
}
|
||||
}
|
||||
|
||||
// A given secret sealed to a key the machine no longer has is not the mesh's to make again: the push is
|
||||
// refused on it, so D1 is too.
|
||||
func TestAGivenSecretUnderAnOldKeyIsNotForeseen(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "keeper", "repository", "given"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, err := open.inventory.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "issue it again") {
|
||||
t.Fatalf("a given secret under an old key: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The bound is read from when the machine began waiting: the oldest assignment since its last send.
|
||||
func TestAMachineAwaitsAPushSinceItsOldestAssignmentSinceTheLastSend(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop")
|
||||
sent := time.Now()
|
||||
since, err := open.inventory.AwaitingSince(ctx, "laptop")
|
||||
if err != nil || since.After(sent) {
|
||||
t.Fatalf("nothing assigned since the send: waiting since %v (%v), the send was before %v", since, err, sent)
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
since, err = open.inventory.AwaitingSince(ctx, "laptop")
|
||||
if err != nil || since.Before(sent.Add(-time.Second)) {
|
||||
t.Fatalf("assigned after the send: waiting since %v (%v), not from the assignment", since, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **D3 AND D13 WAIT FOR THE SEND**: a holder is expected to answer on a machine once the machine was sent
|
||||
// it and had time to report — never between assign and push.
|
||||
@@ -113,3 +242,40 @@ func TestAHolderIsExpectedOnlyOnceSentAndReported(t *testing.T) {
|
||||
}
|
||||
|
||||
// And through the stores: assigned, not in the last send; pushed and reported, carried and settled.
|
||||
func TestALastSendIsReadFromTheSendAndTheReport(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop")
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sends, err := readDeliveries(ctx, open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatal("a holder assigned and not pushed is expected to answer")
|
||||
}
|
||||
if !sends["laptop"].settled(overlay.Name, time.Now().Add(time.Hour)) {
|
||||
t.Fatal("a module the machine was sent long ago is not expected to answer")
|
||||
}
|
||||
pushedBy(t, open, "laptop")
|
||||
sends, err = readDeliveries(ctx, open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatal("pushed a moment ago and not reported, a holder is already expected")
|
||||
}
|
||||
if !sends["laptop"].settled("keeper", time.Now().Add(reportGrace+time.Minute)) {
|
||||
t.Fatal("pushed past the grace, a holder is not expected")
|
||||
}
|
||||
if _, ok := sends["laptop"].carried["keeper"]; !ok {
|
||||
t.Fatalf("the send's builds do not carry keeper: %v", sends["laptop"].carried)
|
||||
}
|
||||
pushedAndApplied(t, open, "laptop")
|
||||
if sends, err = readDeliveries(ctx, open.inventory); err != nil || !sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatalf("pushed and reported applied, a holder is not expected to answer (%v)", err)
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,698 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func windowed(t *testing.T) *stores {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
settings := t.TempDir() + "/merge-window.json"
|
||||
if err := os.WriteFile(settings, []byte(`{"merge-window": "90s", "merge-window-at-most": "10m"}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv(mergeWindowFileVar, settings)
|
||||
noGroups(t)
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Ref: "main", BuiltFrom: "c0",
|
||||
Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range []struct{ module, repository, path string }{
|
||||
{"app", "novox/mesh-catalog", "modules/app"},
|
||||
{"notes", "novox/mesh-catalog", "modules/notes"},
|
||||
{"one", "novox/one", ""},
|
||||
{"two", "novox/two", ""},
|
||||
{"three", "novox/three", ""},
|
||||
} {
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1"},
|
||||
inventory.Source{Repository: m.repository, Seat: "git", Path: m.path, Ref: "main", BuiltFrom: "c0",
|
||||
Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return open
|
||||
}
|
||||
|
||||
func hear(t *testing.T, open *stores, m link.SourceMoved, now time.Time) {
|
||||
t.Helper()
|
||||
if err := (following{open: open}).hearMerge(t.Context(), m, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func cutAt(t *testing.T, open *stores, now time.Time) {
|
||||
t.Helper()
|
||||
if err := cutBatchesHeld(t.Context(), open, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTwoMergesSecondsApartAreOneWalkAtTheLaterCommit(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asked := asksWithPaths(t)
|
||||
claude := catalogueMerge("553b7191claude", "app", t0)
|
||||
dunst := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second))
|
||||
hear(t, open, claude, t0.Add(time.Second))
|
||||
hear(t, open, dunst, t0.Add(18*time.Second))
|
||||
if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 1 {
|
||||
t.Fatalf("within the window: %d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
cutAt(t, open, t0.Add(18*time.Second+89*time.Second))
|
||||
if ws, _ := walks(t, open); len(ws) != 0 {
|
||||
t.Fatalf("cut before the window closed: %+v", ws)
|
||||
}
|
||||
cutAt(t, open, t0.Add(18*time.Second+90*time.Second))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 0 {
|
||||
t.Fatalf("after the window: %d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
w := ws[0]
|
||||
if w.Commit != dunst.Commit || len(w.Commits) != 1 || w.Commits[0].Commit != dunst.Commit {
|
||||
t.Fatalf("the walk is at %s %+v, not the later commit", w.Commit, w.Commits)
|
||||
}
|
||||
for _, m := range []string{"app", "notes"} {
|
||||
if _, in := w.Modules[m]; !in {
|
||||
t.Fatalf("the walk does not build %s, which one of its merges moved: %v", m, w.Modules)
|
||||
}
|
||||
}
|
||||
want := []inventory.PlanMerge{{Repository: "novox/mesh-catalog", Commit: claude.Commit, Carried: dunst.Commit},
|
||||
{Repository: "novox/mesh-catalog", Commit: dunst.Commit}}
|
||||
if w.Delivery == nil || !slices.EqualFunc(w.Delivery.Merges, want, sameMerge) {
|
||||
t.Fatalf("the walk answers %+v, want %+v", w.Delivery, want)
|
||||
}
|
||||
if len(*asked) != 2 || (*asked)[0][2] != "main" || (*asked)[1][2] != "main" {
|
||||
t.Fatalf("the walk did not ask its modules at the branch, which holds the commit it carries: %v", *asked)
|
||||
}
|
||||
// Heard again, as the bus may hand it over twice: never a second merge, nor a second walk.
|
||||
hear(t, open, claude, t0.Add(5*time.Minute))
|
||||
if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 {
|
||||
t.Fatalf("a merge heard twice made %d walk(s) and %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestThreeRepositoriesInOneWindowAreOneWalk(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "feat/three")
|
||||
asksRecorded(t)
|
||||
for i, r := range []string{"one", "two", "three"} {
|
||||
hear(t, open, fromHead(repoMerge(r, "c-"+r, t0.Add(time.Duration(i)*20*time.Second)), "feat/three"),
|
||||
t0.Add(time.Duration(i)*20*time.Second+time.Second))
|
||||
}
|
||||
cutAt(t, open, t0.Add(41*time.Second+90*time.Second))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 0 {
|
||||
t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
w := ws[0]
|
||||
if len(w.Commits) != 3 || len(w.Delivery.Merges) != 3 {
|
||||
t.Fatalf("the walk carries %+v and answers %+v", w.Commits, w.Delivery.Merges)
|
||||
}
|
||||
for _, r := range []string{"one", "two", "three"} {
|
||||
if w.CommitOf("novox/"+r) != "c-"+r {
|
||||
t.Fatalf("the walk carries %s at %q", r, w.CommitOf("novox/"+r))
|
||||
}
|
||||
if _, in := w.Modules[r]; !in {
|
||||
t.Fatalf("the walk does not build %s: %v", r, w.Modules)
|
||||
}
|
||||
}
|
||||
// Each module's ask is recorded at its own repository's commit.
|
||||
for _, r := range []string{"one", "two", "three"} {
|
||||
q, found, err := open.inventory.BuildRequestByID(t.Context(), w.Modules[r].Build)
|
||||
if err != nil || !found || q.Commit != "c-"+r {
|
||||
t.Fatalf("%s's build was not recorded at its own commit: %+v %v %v", r, q, found, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMergeRestartsTheWindowAndTheMaximumClosesIt(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "feat/both", "feat/busy")
|
||||
asksWithPaths(t)
|
||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0)
|
||||
hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(80*time.Second)), "feat/both"), t0.Add(80*time.Second))
|
||||
cutAt(t, open, t0.Add(100*time.Second))
|
||||
if ws, _ := walks(t, open); len(ws) != 0 {
|
||||
t.Fatal("the window closed 90 seconds after its first merge, not after its last")
|
||||
}
|
||||
cutAt(t, open, t0.Add(170*time.Second))
|
||||
if ws, _ := walks(t, open); len(ws) != 1 || len(ws[0].Commits) != 2 {
|
||||
t.Fatalf("the window did not close 90 seconds after its last merge: %+v", ws)
|
||||
}
|
||||
|
||||
busy := windowed(t)
|
||||
groupsWaiting(t, "feat/both", "feat/busy")
|
||||
asksWithPaths(t)
|
||||
var last time.Time
|
||||
for i := 0; i < 12; i++ {
|
||||
last = t0.Add(time.Duration(i) * time.Minute)
|
||||
repo := []string{"one", "two", "three"}[i%3]
|
||||
hear(t, busy, fromHead(repoMerge(repo, "c"+string(rune('a'+i)), last), "feat/busy"), last)
|
||||
if ws, _ := walks(t, busy); len(ws) > 0 {
|
||||
if i != 10 {
|
||||
t.Fatalf("a busy window closed at merge %d (minute %d), not at its maximum", i, i)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
ws, _ := walks(t, busy)
|
||||
if len(ws) != 1 {
|
||||
t.Fatalf("ten minutes of merges a minute apart were never cut: %d walk(s)", len(ws))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheAssemblingBatchIsShown(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "pr/553b7191claude")
|
||||
asksWithPaths(t)
|
||||
var said []inventory.Plan
|
||||
was := inventory.PlanSaved
|
||||
inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { said = append(said, p) }
|
||||
t.Cleanup(func() { inventory.PlanSaved = was })
|
||||
now := time.Now().UTC()
|
||||
claude := catalogueMerge("553b7191claude", "app", now.Add(-20*time.Second))
|
||||
claude.Number, claude.Title = 174, "claude-code: an agent proposes a section"
|
||||
dunst := catalogueMerge("48bda475dunst", "notes", now.Add(-2*time.Second))
|
||||
dunst.Number, dunst.Title = 175, "dunst: the font the operator chose"
|
||||
hear(t, open, claude, now.Add(-19*time.Second))
|
||||
hear(t, open, dunst, now.Add(-time.Second))
|
||||
hear(t, open, fromHead(repoMerge("one", "a6bc0931one", now), dunst.Head), now)
|
||||
out := captured(t, func() error { return plansCommand(t.Context(), nil) })
|
||||
lines := strings.Split(out, "\n")
|
||||
first := lines[0]
|
||||
for _, want := range []string{"assembling: ", " s left (at the latest ", "grouped: novox/mesh-catalog@48bda475 " +
|
||||
"(answers 553b7191), novox/one@a6bc0931; plan not yet calculated"} {
|
||||
if !strings.Contains(first, want) {
|
||||
t.Fatalf("plans' first line %q does not say %q", first, want)
|
||||
}
|
||||
}
|
||||
// Under it, one line per repository: the pull request, what it answers, what it moves.
|
||||
if len(lines) < 3 || strings.TrimSpace(lines[1]) != "mesh-catalog #175 dunst: the font the operator chose (answers "+
|
||||
"#174 claude-code: an agent proposes a section) · app, notes" ||
|
||||
strings.TrimSpace(lines[2]) != "one a6bc0931 · one" {
|
||||
t.Fatalf("the grouped list reads %q", lines[1:4])
|
||||
}
|
||||
if len(said) == 0 || said[len(said)-1].State != inventory.PlanAssembling || said[len(said)-1].Delivery.Batch == nil ||
|
||||
len(said[len(said)-1].Delivery.Merges) != 3 {
|
||||
t.Fatalf("the batch was not said as assembling with its merges: %+v", said)
|
||||
}
|
||||
}
|
||||
|
||||
func captured(t *testing.T, run func() error) string {
|
||||
t.Helper()
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
was := os.Stdout
|
||||
os.Stdout = w
|
||||
runErr := run()
|
||||
os.Stdout = was
|
||||
_ = w.Close()
|
||||
var b bytes.Buffer
|
||||
_, _ = io.Copy(&b, r)
|
||||
if runErr != nil {
|
||||
t.Fatal(runErr)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func TestAMergeDuringAWalkJoinsTheNextBatch(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 || !ws[0].Open() {
|
||||
t.Fatalf("the first batch was not cut: %+v", ws)
|
||||
}
|
||||
first := ws[0]
|
||||
hear(t, open, repoMerge("one", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
|
||||
cutAt(t, open, t0.Add(5*time.Minute))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != first.ID {
|
||||
t.Fatalf("the merge during the walk: %d walk(s), batches %+v", len(ws), bs)
|
||||
}
|
||||
if got, _ := open.inventory.PlanByID(t.Context(), first.ID); !got.Open() {
|
||||
t.Fatalf("the started walk was %s by the next merge", got.State)
|
||||
}
|
||||
if !strings.HasPrefix(batchWords(bs[0], t0.Add(5*time.Minute)), "queued behind "+first.ID) {
|
||||
t.Fatalf("a queued batch reads %q", batchWords(bs[0], t0.Add(5*time.Minute)))
|
||||
}
|
||||
first.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(t.Context(), &first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(6*time.Minute))
|
||||
ws, bs = walks(t, open)
|
||||
if len(ws) != 2 || len(bs) != 0 || ws[0].Commit != "c2" {
|
||||
t.Fatalf("the queued batch was not cut when the walk ended: %+v %+v", ws, bs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWaitingWalkIsFoldedIntoTheNextBatch(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 || !ws[0].Waiting() {
|
||||
t.Fatalf("the walk does not wait for its word: %+v", ws)
|
||||
}
|
||||
waiting := ws[0]
|
||||
hear(t, open, repoMerge("one", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
|
||||
cutAt(t, open, t0.Add(2*time.Minute+90*time.Second))
|
||||
ws, bs := walks(t, open)
|
||||
if len(bs) != 0 || len(ws) != 2 {
|
||||
t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
folded, err := open.inventory.PlanByID(ctx, waiting.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
newer := ws[0]
|
||||
if folded.State != inventory.PlanSuperseded || folded.Delivery.TakenOverBy != newer.ID {
|
||||
t.Fatalf("the waiting walk is %s, taken over by %q", folded.State, folded.Delivery.TakenOverBy)
|
||||
}
|
||||
if len(newer.Delivery.Merges) != 2 || newer.CommitOf("novox/one") != "c2" {
|
||||
t.Fatalf("the newer walk answers %+v", newer.Delivery.Merges)
|
||||
}
|
||||
if _, in := newer.Modules["one"]; !in {
|
||||
t.Fatalf("what the folded walk was to build is not built: %v", newer.Modules)
|
||||
}
|
||||
}
|
||||
|
||||
func TestALateMergeIsAnsweredByTheWalkOfTheLaterOne(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
later := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second))
|
||||
hear(t, open, later, t0.Add(18*time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
w := ws[0]
|
||||
w.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Moves only notes, which the done walk built from the branch after it: answered there, at once.
|
||||
hear(t, open, catalogueMerge("553b7191early", "notes", t0), t0.Add(15*time.Minute))
|
||||
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
||||
if got.State != inventory.PlanDone || len(got.Delivery.Merges) != 2 ||
|
||||
!sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: "553b7191early",
|
||||
Carried: later.Commit}) {
|
||||
t.Fatalf("the late merge is not named by the walk that carried it: %+v", got.Delivery.Merges)
|
||||
}
|
||||
// Moves app, which that walk never built: the next batch walks it.
|
||||
hear(t, open, catalogueMerge("1111aaaaapp", "app", t0.Add(time.Second)), t0.Add(16*time.Minute))
|
||||
_, bs := walks(t, open)
|
||||
if len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "1111aaaaapp" {
|
||||
t.Fatalf("a late merge moving what the walk never built did not join the next batch: %+v", bs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailedWalkWalksItsEarlierMergesAlone(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asked := asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
// Made before the cut marks their modules seen, as merges are: walked again, they are news all the same.
|
||||
oldest := catalogueMerge("aaaa0001", "app", t0.Add(-3*time.Minute))
|
||||
middle := catalogueMerge("bbbb0002", "app", t0.Add(-3*time.Minute+10*time.Second))
|
||||
newest := catalogueMerge("cccc0003", "notes", t0.Add(-3*time.Minute+20*time.Second))
|
||||
for i, m := range []link.SourceMoved{oldest, middle, newest} {
|
||||
hear(t, open, m, t0.Add(time.Duration(i)*10*time.Second+time.Second))
|
||||
}
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
failed := ws[0]
|
||||
failed.State, failed.Note = inventory.PlanFailed, "notes failed to build in tier 0"
|
||||
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(3*time.Minute))
|
||||
got, _ := open.inventory.PlanByID(ctx, failed.ID)
|
||||
if got.State != inventory.PlanFailed || len(got.Delivery.Merges) != 3 {
|
||||
t.Fatalf("the failed walk's record changed: %s %+v", got.State, got.Delivery.Merges)
|
||||
}
|
||||
ws, _ = walks(t, open)
|
||||
alone := ws[0]
|
||||
if alone.ID == failed.ID || alone.Commit != middle.Commit || len(alone.Delivery.Merges) != 1 ||
|
||||
!sameMerge(alone.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: middle.Commit}) {
|
||||
t.Fatalf("the newest earlier merge was not walked alone on its own commit: %+v", alone)
|
||||
}
|
||||
if _, in := alone.Modules["app"]; !in || (*asked)[len(*asked)-1] != [3]string{"novox/mesh-catalog", "modules/app",
|
||||
middle.Commit} {
|
||||
t.Fatalf("the merge walked alone does not build app at its own commit: %v, asked %v", alone.Modules, *asked)
|
||||
}
|
||||
// Retried, the failed walk would name merges the search answers now.
|
||||
if _, err := retryPlan(ctx, open, failed.ID); err == nil || !strings.Contains(err.Error(), "walked alone") {
|
||||
t.Fatalf("a failed walk whose merges are searched was retried: %v", err)
|
||||
}
|
||||
alone.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &alone); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(4*time.Minute))
|
||||
ws, bs := walks(t, open)
|
||||
if ws[0].ID != alone.ID || len(bs) != 0 {
|
||||
t.Fatalf("the search went on after a merge was delivered: %+v", ws[0])
|
||||
}
|
||||
done, _ := open.inventory.PlanByID(ctx, alone.ID)
|
||||
if len(done.Delivery.Merges) != 2 || !sameMerge(done.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog",
|
||||
Commit: oldest.Commit, Carried: middle.Commit}) {
|
||||
t.Fatalf("the oldest merge is not answered by the walk that delivered the one after it: %+v", done.Delivery.Merges)
|
||||
}
|
||||
// A stopped walk starts no search: a person ended it.
|
||||
hear(t, open, catalogueMerge("dddd0004", "app", t0.Add(5*time.Minute)), t0.Add(5*time.Minute))
|
||||
hear(t, open, catalogueMerge("eeee0005", "notes", t0.Add(5*time.Minute+time.Second)), t0.Add(5*time.Minute+time.Second))
|
||||
cutAt(t, open, t0.Add(8*time.Minute))
|
||||
ws, _ = walks(t, open)
|
||||
if _, err := stopWalk(ctx, open.inventory, ws[0].ID, "mesh-delivery for jochen", "not now"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(9*time.Minute))
|
||||
if again, _ := walks(t, open); again[0].ID != ws[0].ID {
|
||||
t.Fatalf("a stopped walk's earlier merge was walked alone: %+v", again[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAGroupsOrderBecomesTiersInsideTheWalk(t *testing.T) {
|
||||
for _, grouped := range []bool{true, false} {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
for _, m := range []string{"mesh-host"} {
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"},
|
||||
inventory.Source{Repository: "novox/" + m, Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
host := repoMerge("mesh-host", "h1", t0)
|
||||
ctl := repoMerge("mesh-controller", "k1", t0.Add(time.Second))
|
||||
host.Head, ctl.Head = "feat/together", "feat/together"
|
||||
if !grouped {
|
||||
ctl.Head = "feat/alone"
|
||||
}
|
||||
hear(t, open, ctl, t0.Add(2*time.Second))
|
||||
hear(t, open, host, t0.Add(3*time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 {
|
||||
t.Fatalf("%d walks", len(ws))
|
||||
}
|
||||
tiers := ws[0].Tiers
|
||||
if grouped {
|
||||
if len(tiers) != 2 || !slices.Equal(tiers[0], []string{"mesh-host"}) ||
|
||||
!slices.Equal(tiers[1], []string{"mesh-controller"}) {
|
||||
t.Fatalf("the group's order is not the walk's tiers: %v", tiers)
|
||||
}
|
||||
} else if len(tiers) != 1 {
|
||||
t.Fatalf("two merges of no group were ordered: %v", tiers)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestABatchSurvivesARestart(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "feat/both")
|
||||
asksWithPaths(t)
|
||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0)
|
||||
hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(30*time.Second)), "feat/both"), t0.Add(30*time.Second))
|
||||
|
||||
// A new controller: nothing of the first one's but the store.
|
||||
again, err := openStores(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(again.Close)
|
||||
hear(t, again, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0.Add(60*time.Second)) // the bus hands it over again
|
||||
cutAt(t, again, t0.Add(119*time.Second))
|
||||
if ws, bs := walks(t, again); len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 {
|
||||
t.Fatalf("the restarted controller's batch: %d walk(s), %+v", len(ws), bs)
|
||||
}
|
||||
cutAt(t, again, t0.Add(120*time.Second))
|
||||
ws, bs := walks(t, again)
|
||||
if len(ws) != 1 || len(bs) != 0 || len(ws[0].Delivery.Merges) != 2 {
|
||||
t.Fatalf("the restarted controller did not cut the batch at its window: %+v %+v", ws, bs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestALateMergeOfAOneModuleRepositoryIsNamed(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
c2 := repoMerge("one", "c2", t0.Add(-50*time.Second))
|
||||
hear(t, open, c2, t0)
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
w := ws[0]
|
||||
w.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, repoMerge("one", "c1", t0.Add(-60*time.Second)), t0.Add(15*time.Minute))
|
||||
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
||||
if len(got.Delivery.Merges) != 2 || !sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/one",
|
||||
Commit: "c1", Carried: "c2"}) {
|
||||
t.Fatalf("the late merge was not named by the walk that carried it: %+v", got.Delivery.Merges)
|
||||
}
|
||||
|
||||
// A walk of two that built only two: a late merge of one is not its to name.
|
||||
hear(t, open, repoMerge("two", "d2", t0.Add(16*time.Minute)), t0.Add(16*time.Minute))
|
||||
cutAt(t, open, t0.Add(18*time.Minute))
|
||||
ws, _ = walks(t, open)
|
||||
w = ws[0]
|
||||
delete(w.Modules, "two")
|
||||
w.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, repoMerge("two", "d1", t0.Add(15*time.Minute)), t0.Add(20*time.Minute))
|
||||
if got, _ := open.inventory.PlanByID(ctx, w.ID); len(got.Delivery.Merges) != 1 {
|
||||
t.Fatalf("a walk that never built what the late merge moves named it: %+v", got.Delivery.Merges)
|
||||
}
|
||||
if _, bs := walks(t, open); len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "d1" {
|
||||
t.Fatalf("the late merge did not join the next batch: %+v", bs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWalkLetGoBesideAStartedOneStartsOnceItEnded(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asked := asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
started, _ := walks(t, open)
|
||||
waiting := inventory.Plan{ID: "plan-waiting", Repository: "novox/one", Branch: "main", Commit: "d1", Created: t0,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"one"}}, Modules: map[string]*inventory.PlanModule{"one": {}},
|
||||
Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}}
|
||||
if err := open.inventory.SavePlan(ctx, &waiting); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := letGo(ctx, open.inventory, waiting.ID, catalogue.DeliverySeat, "its turn"); err != nil {
|
||||
t.Fatalf("the word was refused beside %s: %v", started[0].ID, err)
|
||||
}
|
||||
before := len(*asked)
|
||||
advanceHeld(ctx, open)
|
||||
got, _ := open.inventory.PlanByID(ctx, waiting.ID)
|
||||
if len(*asked) != before || !strings.Contains(got.Note, "starts once "+started[0].ID) {
|
||||
t.Fatalf("a walk let go beside a started one asked (%d → %d) or does not say it waits: %q", before, len(*asked), got.Note)
|
||||
}
|
||||
// Read as a wait, an hour on: its note on the line, never LATE, and no tier of it late for S3.
|
||||
later := time.Now().Add(time.Hour)
|
||||
if line := planLine(got, later); !strings.Contains(line, "starts once "+started[0].ID) || strings.Contains(line, "LATE") {
|
||||
t.Fatalf("a deferred walk reads %q", line)
|
||||
}
|
||||
facts, _, err := gatherPlans(ctx, open.inventory, later, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range facts {
|
||||
if f.id == got.ID {
|
||||
t.Fatalf("a deferred walk is watched as a tier running late: %+v", f)
|
||||
}
|
||||
}
|
||||
done := started[0]
|
||||
done.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) != before+1 {
|
||||
t.Fatalf("the walk did not start once the started one ended: asked %v", *asked)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMergeOnTheControllersPathNeverSharesABatch(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asked := asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, catalogueMerge("aaaa0001", "app", t0), t0)
|
||||
hear(t, open, repoMerge("mesh-controller", "k1", t0.Add(10*time.Second)), t0.Add(10*time.Second))
|
||||
hear(t, open, catalogueMerge("bbbb0002", "notes", t0.Add(20*time.Second)), t0.Add(20*time.Second))
|
||||
_, bs := walks(t, open)
|
||||
var ownBatch, catalogueBatch inventory.Plan
|
||||
for _, b := range bs {
|
||||
if b.OwnPath() {
|
||||
ownBatch = b
|
||||
} else {
|
||||
catalogueBatch = b
|
||||
}
|
||||
}
|
||||
if len(bs) != 2 || ownBatch.ID == "" || catalogueBatch.ID == "" || len(catalogueBatch.Delivery.Merges) != 2 ||
|
||||
len(ownBatch.Delivery.Merges) != 1 {
|
||||
t.Fatalf("a controller merge and two catalogue merges in one window: %+v", bs)
|
||||
}
|
||||
if !strings.Contains(batchWords(ownBatch, t0.Add(30*time.Second)), "own path") {
|
||||
t.Fatalf("the own-path batch does not say so: %q", batchWords(ownBatch, t0.Add(30*time.Second)))
|
||||
}
|
||||
// Both windows closed, the controller's batch is cut first and starts; the catalogue batch queues behind it,
|
||||
// is cut when the controller's walk ended, and waits for its word with both merges.
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || ws[0].Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k1" {
|
||||
t.Fatalf("the controller's batch was not cut first into a started walk: %+v", ws)
|
||||
}
|
||||
ownWalk := ws[0]
|
||||
for _, m := range []string{"app", "notes"} {
|
||||
if _, in := ownWalk.Modules[m]; in {
|
||||
t.Fatalf("the controller's walk builds %s, a catalogue module: it skipped its turn", m)
|
||||
}
|
||||
}
|
||||
if len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != ownWalk.ID || bs[0].OwnPath() {
|
||||
t.Fatalf("the catalogue batch does not queue behind the controller's walk: %+v", bs)
|
||||
}
|
||||
ownWalk.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &ownWalk); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(3*time.Minute))
|
||||
ws, bs = walks(t, open)
|
||||
if len(bs) != 0 || ws[0].ID != catalogueBatch.ID || !ws[0].Waiting() || len(ws[0].Delivery.Merges) != 2 {
|
||||
t.Fatalf("the catalogue batch was not cut into a waiting walk once the controller's ended: %+v %+v", ws, bs)
|
||||
}
|
||||
for _, m := range []string{"app", "notes"} {
|
||||
if _, in := ws[0].Modules[m]; !in {
|
||||
t.Fatalf("the catalogue walk does not build %s: %v", m, ws[0].Modules)
|
||||
}
|
||||
}
|
||||
for _, a := range *asked {
|
||||
if a[1] == "modules/app" || a[1] == "modules/notes" {
|
||||
t.Fatalf("a catalogue module was asked without the word: %v", *asked)
|
||||
}
|
||||
}
|
||||
|
||||
// A catalogue walk waiting for its word when an own-path batch is cut keeps waiting beside the own-path
|
||||
// walk, is not folded into it, and its word is taken meanwhile: it starts once the own-path walk ended.
|
||||
catalogueWalk := ws[0]
|
||||
hear(t, open, repoMerge("mesh-controller", "k2", t0.Add(4*time.Minute)), t0.Add(4*time.Minute))
|
||||
cutAt(t, open, t0.Add(6*time.Minute))
|
||||
ws, _ = walks(t, open)
|
||||
kept, _ := open.inventory.PlanByID(ctx, catalogueWalk.ID)
|
||||
if !kept.Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k2" || ws[0].Waiting() {
|
||||
t.Fatalf("the waiting catalogue walk was not kept waiting beside the controller's walk: %s %+v", kept.State, ws)
|
||||
}
|
||||
if _, in := ws[0].Modules["app"]; in {
|
||||
t.Fatalf("the controller's walk folded the waiting catalogue walk in: %v", ws[0].Modules)
|
||||
}
|
||||
if _, err := letGo(ctx, open.inventory, catalogueWalk.ID, catalogue.DeliverySeat, "its turn"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := len(*asked)
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) != before {
|
||||
t.Fatalf("the catalogue walk started beside the controller's: asked %v", (*asked)[before:])
|
||||
}
|
||||
own2 := ws[0]
|
||||
own2.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &own2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) < before+1 || (*asked)[before][1] != "modules/app" {
|
||||
t.Fatalf("the catalogue walk did not start once the controller's ended: %v", (*asked)[before:])
|
||||
}
|
||||
}
|
||||
|
||||
func TestALateCatalogueMergeIsNotAnsweredByAnOwnPathWalk(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
for _, m := range []struct {
|
||||
module string
|
||||
claims []catalogue.Claim
|
||||
}{
|
||||
{"nats", nil},
|
||||
{"mesh-delivery", []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
||||
} {
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1", Claims: m.claims},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + m.module, Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mixed := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "m1xed", MergedAt: t0.Format(time.RFC3339Nano),
|
||||
Paths: []string{"modules/nats/module.json", "modules/app/module.json"}, ModuleDirs: []string{"modules/nats", "modules/app"},
|
||||
ModuleDirsSaid: true}
|
||||
hear(t, open, mixed, t0.Add(time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 || ws[0].Waiting() {
|
||||
t.Fatalf("the mixed merge's walk waited, or was not cut: %+v", ws)
|
||||
}
|
||||
done := ws[0]
|
||||
done.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, catalogueMerge("ear1ier", "app", t0.Add(-30*time.Second)), t0.Add(3*time.Minute))
|
||||
got, _ := open.inventory.PlanByID(ctx, done.ID)
|
||||
_, bs := walks(t, open)
|
||||
if len(got.Delivery.Merges) != 1 || len(bs) != 1 || bs[0].OwnPath() || bs[0].Delivery.Merges[0].Commit != "ear1ier" {
|
||||
t.Fatalf("the late catalogue merge was answered by the own-path walk (%+v) rather than the next catalogue batch (%+v)",
|
||||
got.Delivery.Merges, bs)
|
||||
}
|
||||
}
|
||||
@@ -1,293 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The suite's tests of a merge, but for the merge window's own (this file's), plan the merge as it is heard: a
|
||||
// window of nothing closes at once, so a merge with no walk open is cut into its walk at once, as every merge
|
||||
// was planned before novox/hq ADR 0276. The window's tests set it through the controller's settings.
|
||||
func init() {
|
||||
mergeWindowDefault = 0
|
||||
// No forge in a test: no manifest is found at any commit, so a merge draws no edge of the manifests' change
|
||||
// (delivery_edges.go). A test that wants some replaces these (forgeHolding).
|
||||
manifestAt = func(context.Context, string, string, string, string) (catalogue.Manifest, bool, error) {
|
||||
return catalogue.Manifest{}, false, nil
|
||||
}
|
||||
parentOf = func(context.Context, string, string, string) (string, error) { return "", nil }
|
||||
}
|
||||
|
||||
// windowed is a mesh whose controller's settings give a merge window of 90 seconds and at most 10 minutes, as its
|
||||
// settings file says them, and
|
||||
// modules built from the catalogue (app, notes) and from three repositories of their own.
|
||||
|
||||
// t0 is the moment a test's clock starts: merges are dated by it, and heard after it.
|
||||
var t0 = time.Now().UTC().Truncate(time.Second)
|
||||
|
||||
// catalogueMerge is a merge of the catalogue changing one module's directory, made at a moment, from a pull
|
||||
// request's head branch of its own, as the forge announces every merge. Whether its head names a delivery group
|
||||
// with a member still to merge is mesh-delivery's word, which windowed stubs as no group at all (noGroups); a test
|
||||
// whose merges wait for the window as a group's says so (groupsWaiting).
|
||||
func catalogueMerge(commit, module string, made time.Time) link.SourceMoved {
|
||||
return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Head: "pr/" + commit, Commit: commit,
|
||||
MergedAt: made.Format(time.RFC3339Nano), Paths: []string{"modules/" + module + "/module.json"},
|
||||
ModuleDirs: []string{"modules/" + module}, ModuleDirsSaid: true}
|
||||
}
|
||||
|
||||
// repoMerge is a merge of a repository of one module's own, from a head branch of its own (catalogueMerge).
|
||||
func repoMerge(repo, commit string, made time.Time) link.SourceMoved {
|
||||
return link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Head: "pr/" + commit, Commit: commit,
|
||||
MergedAt: made.Format(time.RFC3339Nano), Paths: []string{"main.go"}}
|
||||
}
|
||||
|
||||
// noGroups stubs mesh-delivery's word on groups for a test: no head names a group.
|
||||
func noGroups(t *testing.T) {
|
||||
t.Helper()
|
||||
was := deliveryGroupOf
|
||||
deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) { return nil, false, nil }
|
||||
t.Cleanup(func() { deliveryGroupOf = was })
|
||||
}
|
||||
|
||||
// groupsWaiting stubs mesh-delivery's word on groups for a test of the merge window's own timing: each head named
|
||||
// is one whose group mesh-delivery cannot say, so a merge from it waits for the window (ADR 0306 decision 2) and
|
||||
// nothing holds the batch open; every other head names no group. A group with a member still to merge, which holds
|
||||
// its batch open however long it takes (ADR 0418 clause 3), is groupStates'.
|
||||
func groupsWaiting(t *testing.T, heads ...string) {
|
||||
t.Helper()
|
||||
was := deliveryGroupOf
|
||||
deliveryGroupOf = func(_ context.Context, _ bool, head string) ([]groupMember, bool, error) {
|
||||
if slices.Contains(heads, head) {
|
||||
return nil, false, errors.New("the test's mesh-delivery does not say")
|
||||
}
|
||||
return nil, false, nil
|
||||
}
|
||||
t.Cleanup(func() { deliveryGroupOf = was })
|
||||
}
|
||||
|
||||
// everyHeadWaits stubs mesh-delivery's word on groups for a test whose merges each wait for the window as they did
|
||||
// before novox/hq ADR 0304: mesh-delivery cannot say any head's group.
|
||||
func everyHeadWaits(t *testing.T) {
|
||||
t.Helper()
|
||||
was := deliveryGroupOf
|
||||
deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) {
|
||||
return nil, false, errors.New("the test's mesh-delivery does not say")
|
||||
}
|
||||
t.Cleanup(func() { deliveryGroupOf = was })
|
||||
}
|
||||
|
||||
// groupStates stubs mesh-delivery's word on one group: its members and their states, which a test changes as the
|
||||
// members merge.
|
||||
func groupStates(t *testing.T, head string, states map[string]string) {
|
||||
t.Helper()
|
||||
groupStatesFrom(t, head, states, true)
|
||||
}
|
||||
|
||||
// groupStatesFrom is groupStates for a mesh-delivery that does (composed) or does not yet say the group's gate is the
|
||||
// composed check's: only the first holds the group's batch open for its last member.
|
||||
func groupStatesFrom(t *testing.T, head string, states map[string]string, composed bool) {
|
||||
t.Helper()
|
||||
was := deliveryGroupOf
|
||||
deliveryGroupOf = func(_ context.Context, _ bool, asked string) ([]groupMember, bool, error) {
|
||||
if asked != head {
|
||||
return nil, false, nil
|
||||
}
|
||||
var members []groupMember
|
||||
for repository, state := range states {
|
||||
members = append(members, groupMember{Repository: repository, State: state, Composed: composed})
|
||||
}
|
||||
return members, true, nil
|
||||
}
|
||||
t.Cleanup(func() { deliveryGroupOf = was })
|
||||
}
|
||||
|
||||
// fromHead is a merge from a head branch a test names: merges from one head are members of one delivery group,
|
||||
// related by it (novox/hq ADR 0304 decision 1), so they share a batch whatever else they share.
|
||||
func fromHead(m link.SourceMoved, head string) link.SourceMoved {
|
||||
m.Head = head
|
||||
return m
|
||||
}
|
||||
|
||||
// lone is a merge that names no head: known to be no delivery group's, it waits for no window unless it is
|
||||
// related to a batch assembling (ADR 0304 decision 1).
|
||||
func lone(m link.SourceMoved) link.SourceMoved {
|
||||
m.Head = ""
|
||||
return m
|
||||
}
|
||||
|
||||
// hear is a merge heard at a moment.
|
||||
|
||||
// cutAt is the cutter looking at a moment.
|
||||
|
||||
// walks is every plan record that is a walk, newest first, and the batches not yet cut.
|
||||
func walks(t *testing.T, open *stores) (walks, batches []inventory.Plan) {
|
||||
t.Helper()
|
||||
recent, err := open.inventory.RecentPlans(t.Context(), 50)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, p := range recent {
|
||||
if p.Batch() {
|
||||
batches = append(batches, p)
|
||||
} else {
|
||||
walks = append(walks, p)
|
||||
}
|
||||
}
|
||||
return walks, batches
|
||||
}
|
||||
|
||||
// The replay of issue 362: two catalogue merges 18 seconds apart are one batch, cut once into one walk at the
|
||||
// later commit, which names both merges — the earlier carried by the later.
|
||||
|
||||
// Merges of three repositories in one window are one walk, one commit for each.
|
||||
|
||||
// Each merge restarts the window: a merge at second 80 keeps the batch open until second 170; the window
|
||||
// closes at most ten minutes after its first merge however busy.
|
||||
|
||||
// `plans` lists the batch being assembled first, in the operator's words: how long is left, when at the latest,
|
||||
// what is grouped, and that the plan is not yet calculated. It says so on the bus as plan-moved, too.
|
||||
|
||||
// captured is what a command printed.
|
||||
|
||||
// A merge heard during a walk it is related to (the same repository, issue 973) joins the next batch, which is
|
||||
// cut when the walk ends; the walk that started is never superseded (ADR 0276 decision 4, ADR 0407 decision 4).
|
||||
|
||||
// A walk waiting for its delivery's word is folded into the next batch's walk, which answers its merges; it
|
||||
// names the walk that took it over.
|
||||
|
||||
// A merge heard after a later merge of its branch was walked is answered by that walk when it builds all it
|
||||
// moves — named at once on a walk done — and joins the next batch when it does not.
|
||||
|
||||
// A failed walk marks nothing delivered: its earlier merges are walked alone, on their own commit, the newest
|
||||
// first; the first delivered answers the older ones, and the search stops.
|
||||
|
||||
// A delivery group's order holds inside the walk (ADR 0249 unchanged): merges of two repositories from one head
|
||||
// branch name, the node-engine's before the controller's, are tiered so; without the group, one tier.
|
||||
|
||||
// A restarted controller resumes the window where it stood: the batch, its merges and their times are in the
|
||||
// store, a merge handed over again is not doubled, and the batch is cut when its window closes.
|
||||
|
||||
// The window's settings reach the controller in its settings file, read at every look; one that says no
|
||||
// duration, or no file, is the default.
|
||||
func TestTheWindowIsTheControllersSetting(t *testing.T) {
|
||||
file := t.TempDir() + "/merge-window.json"
|
||||
t.Setenv(mergeWindowFileVar, file)
|
||||
for _, c := range []struct {
|
||||
content string
|
||||
window, atMost time.Duration
|
||||
}{
|
||||
{`{"merge-window": "60s", "merge-window-at-most": "5m"}`, time.Minute, 5 * time.Minute},
|
||||
{`{"merge-window": 30, "merge-window-at-most": ""}`, 30 * time.Second, mergeWindowAtMostDefault},
|
||||
{`{"merge-window": "soon"}`, mergeWindowDefault, mergeWindowAtMostDefault},
|
||||
} {
|
||||
if err := os.WriteFile(file, []byte(c.content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if w, m := mergeWindowOf(); w != c.window || m != c.atMost {
|
||||
t.Errorf("%s reads as %s and %s, want %s and %s", c.content, w, m, c.window, c.atMost)
|
||||
}
|
||||
}
|
||||
t.Setenv(mergeWindowFileVar, file+".gone")
|
||||
if w, m := mergeWindowOf(); w != mergeWindowDefault || m != mergeWindowAtMostDefault {
|
||||
t.Errorf("no file reads as %s and %s", w, m)
|
||||
}
|
||||
}
|
||||
|
||||
// S16 names every merge a waiting walk answers, never one commit a supersession may have ended (issue 362).
|
||||
func TestAWaitingWalkNamesTheMergesItAnswers(t *testing.T) {
|
||||
f := calm(t0)
|
||||
f.waits = []waitFacts{{id: "plan-2", repository: "novox/mesh-catalog", commit: "48bda475dunst", awaits: "mesh-delivery",
|
||||
since: t0.Add(-31 * time.Minute), merges: []inventory.PlanMerge{
|
||||
{Repository: "novox/mesh-catalog", Commit: "553b7191claude", Carried: "48bda475dunst"},
|
||||
{Repository: "novox/mesh-catalog", Commit: "48bda475dunst"}}}}
|
||||
got := watchWaits(f)
|
||||
if len(got) != 1 || !strings.Contains(got[0].Summary, "novox/mesh-catalog@553b7191") ||
|
||||
!strings.Contains(got[0].Summary, "novox/mesh-catalog@48bda475") {
|
||||
t.Fatalf("the waiting walk does not name both merges: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge heard after a later merge of its branch was cut is named by that walk however its modules read since
|
||||
// the cut, in a repository of one module (review of this change); a walk that never built what it moves names
|
||||
// nothing, and the merge joins the next batch.
|
||||
|
||||
// A related walk waits for the delivery's word too: a waiting walk let go beside a started one it is related to
|
||||
// (the same repository, issue 973) takes the word and starts once that one ended, asking nothing before.
|
||||
|
||||
// A file a merge of the batch removed and a later one brought back is not removed; one removed last is.
|
||||
func TestARemovedFileIsTheLastMergesWord(t *testing.T) {
|
||||
ev := func(commit string, paths, removed []string) inventory.BatchedMerge {
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "one", Base: "main", Commit: commit, Paths: paths, Removed: removed}
|
||||
b, _ := json.Marshal(m)
|
||||
return inventory.BatchedMerge{Repository: "novox/one", Branch: "main", Commit: commit, Event: b,
|
||||
Merged: t0.Add(time.Duration(len(commit)) * time.Second)}
|
||||
}
|
||||
got := combinedMerges([]inventory.BatchedMerge{
|
||||
ev("a", []string{"x/module.json", "y/module.json"}, []string{"x/module.json", "y/module.json"}),
|
||||
ev("bb", []string{"x/module.json"}, nil),
|
||||
ev("ccc", []string{"z.go"}, nil)})
|
||||
if len(got) != 1 || got[0].Commit != "ccc" || !slices.Equal(got[0].Removed, []string{"y/module.json"}) ||
|
||||
len(got[0].Paths) != 3 {
|
||||
t.Fatalf("combined as %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge on the controller's own path never shares a batch with one that waits for mesh-delivery's word
|
||||
// (decided during the build, 2026-10-10): the two are batches of their own, cut one after the other; a walk on
|
||||
// the own path folds no waiting catalogue walk but batches its merges again behind it; the catalogue's walk
|
||||
// still waits for the word, so no catalogue delivery skips its turn.
|
||||
|
||||
// `plans` names a walk by what it moves (asked by the operator, 2026-10-10): the repository's pull request and
|
||||
// title, the modules it moves and the machines running them, then the state words; a record naming no pull
|
||||
// request is named by its commit, and a title is cut at fifty runes.
|
||||
func TestAPlanLineNamesWhatItMoves(t *testing.T) {
|
||||
now := time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)
|
||||
p := inventory.Plan{ID: "plan-1", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1c1c1c1c1",
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"messenger", "telegram"}}, TierEntered: now.Add(-2 * time.Minute),
|
||||
Modules: map[string]*inventory.PlanModule{"messenger": {State: "asked"}, "telegram": {State: "asked"}},
|
||||
Commits: []inventory.PlanCommit{{Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1c1c1c1c1"}},
|
||||
Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{{Repository: "novox/mesh-catalog",
|
||||
Commit: "c1c1c1c1c1", Number: 175, Title: "a tap shows its outcome", Moves: []string{"telegram", "messenger"}}}}}
|
||||
running := func(module string) []string {
|
||||
if module == "messenger" || module == "telegram" {
|
||||
return []string{"novox"}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if got, want := planLineOn(p, now, pauseView{}, tierAtLeast, running),
|
||||
"mesh-catalog #175 a tap shows its outcome · messenger, telegram → novox · tier 1 of 1, building for 2m0s"; got != want {
|
||||
t.Fatalf("the line reads %q, want %q", got, want)
|
||||
}
|
||||
if got := planLine(p, now); !strings.HasPrefix(got, "mesh-catalog #175 a tap shows its outcome · messenger, telegram · tier") {
|
||||
t.Fatalf("without the machines the line reads %q", got)
|
||||
}
|
||||
old := p
|
||||
old.Commits, old.Delivery = nil, nil
|
||||
if got := planLine(old, now); !strings.HasPrefix(got, "mesh-catalog c1c1c1c1 · tier 1 of 1") {
|
||||
t.Fatalf("a record naming no pull request reads %q", got)
|
||||
}
|
||||
long := p
|
||||
long.Delivery.Merges[0].Title = strings.Repeat("abcdefghij", 6)
|
||||
if got := planHeadline(long, nil); !strings.Contains(got, "#175 "+strings.Repeat("abcdefghij", 4)+"abcdefghi…") {
|
||||
t.Fatalf("a long title is not cut at fifty runes: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// sameMerge compares what a walk names of a merge: its repository, commit and the commit carrying it.
|
||||
func sameMerge(a, b inventory.PlanMerge) bool {
|
||||
return a.Repository == b.Repository && a.Commit == b.Commit && a.Carried == b.Carried
|
||||
}
|
||||
|
||||
// A catalogue merge heard after a later merge of its branch was walked without the word (a merge that touched
|
||||
// the bus too, on the controller's own path) is not answered by that walk while the delivery seat has a holder:
|
||||
// its delivery would skip its turn. It joins the next catalogue batch.
|
||||
@@ -1,153 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **Every wait of a batch has a bound and a condition** (novox/hq ADR 0276 decision 9): a batch still
|
||||
// assembling a minute past its maximum while no walk is open is a cut the controller failed to make (S18),
|
||||
// and a batch waiting behind an open walk longer than that walk's bound waits on a walk gone wrong (S19).
|
||||
|
||||
// The kinds S18 and S19 raise.
|
||||
const (
|
||||
kindBatchNotCut = "batch-not-cut"
|
||||
kindBatchBehindWalk = "batch-behind-walk"
|
||||
// kindGroupNotMerged is raised for a group whose last member has not merged by the merge window's maximum after
|
||||
// its first (novox/hq ADR 0418 clause 3).
|
||||
kindGroupNotMerged = "group-not-merged"
|
||||
)
|
||||
|
||||
// batchFacts is one batch not yet cut, as the watchdogs read it.
|
||||
type batchFacts struct {
|
||||
id, state, grouped string
|
||||
// atMost is when its window closes at the latest; closed when it closed.
|
||||
atMost, closed time.Time
|
||||
// behind is the open walk it waits behind, and walkBound that walk's bound: a tier's bound for each of its
|
||||
// tiers.
|
||||
behind string
|
||||
walkBound time.Duration
|
||||
// holds are the groups the batch is held for, by head, with why; created is when the batch was made, the first
|
||||
// merge of the group heard; atMostWindow is the maximum a window may take (the controller's setting).
|
||||
holds map[string]string
|
||||
created time.Time
|
||||
atMostWindow time.Duration
|
||||
}
|
||||
|
||||
// gatherBatches is every batch not yet cut, with the bound of the walk it waits behind.
|
||||
func gatherBatches(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]batchFacts, error) {
|
||||
batches, err := inv.Batches(ctx)
|
||||
if err != nil || len(batches) == 0 {
|
||||
return nil, err
|
||||
}
|
||||
bounds, err := measuredTierBounds(ctx, inv, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []batchFacts
|
||||
for _, b := range batches {
|
||||
_, atMostWindow := mergeWindowOf()
|
||||
f := batchFacts{id: b.ID, state: b.State, grouped: groupedWords(b), created: b.Created, atMostWindow: atMostWindow}
|
||||
if w := b.Delivery; w != nil && w.Batch != nil {
|
||||
f.holds = w.Batch.Holds
|
||||
f.atMost, f.closed, f.behind = w.Batch.AtMost, w.Batch.ClosesAt, w.Batch.Behind
|
||||
if f.atMost.Before(f.closed) {
|
||||
f.closed = f.atMost
|
||||
}
|
||||
if w.Batch.CutAtOnce && !b.Created.IsZero() {
|
||||
// Its window closed when it was made, which is its latest too (ADR 0304 decision 1).
|
||||
f.atMost, f.closed = b.Created, b.Created
|
||||
}
|
||||
}
|
||||
if f.behind != "" {
|
||||
if walk, err := inv.PlanByID(ctx, f.behind); err == nil {
|
||||
f.walkBound = bounds.of(walk.Repository) * time.Duration(max(1, len(walk.Tiers)))
|
||||
}
|
||||
}
|
||||
out = append(out, f)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// watchBatchesNotCut is S18: a batch still assembling a minute past its maximum, while no walk is open.
|
||||
func watchBatchesNotCut(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, b := range f.batches {
|
||||
if b.state != inventory.PlanAssembling || b.atMost.IsZero() || f.now.Sub(b.atMost) <= batchLateAfter ||
|
||||
len(b.holds) > 0 { // a batch held for a group waits on a member, which has its own condition
|
||||
continue
|
||||
}
|
||||
late := f.now.Sub(b.atMost)
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id, Kind: kindBatchNotCut,
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the batch %s is still assembling %s past its latest close (%s), with no walk open: "+
|
||||
"the controller did not cut it; grouped: %s", b.id, ago(late), b.atMost.UTC().Format(time.RFC3339), b.grouped),
|
||||
Said: fmt.Sprintf("assembling since %s past its maximum", ago(late)),
|
||||
Headline: "Merged changes are not being delivered",
|
||||
Explanation: fmt.Sprintf("Merges collected for one delivery should have been planned %s ago and were not. "+
|
||||
"Nothing is lost; the mesh keeps them until it plans them.", humanDuration(late)),
|
||||
Resolved: "Merged changes are being delivered again"})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// watchBatchesBehind is S19: a batch waiting behind an open walk longer than that walk's bound, naming it.
|
||||
func watchBatchesBehind(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, b := range f.batches {
|
||||
if b.state != inventory.PlanQueued || b.behind == "" || b.walkBound <= 0 || f.now.Sub(b.closed) <= b.walkBound {
|
||||
continue
|
||||
}
|
||||
in := f.now.Sub(b.closed)
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id, Kind: kindBatchBehindWalk,
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the batch %s has waited %s behind the walk %s, longer than that walk's bound (%s); "+
|
||||
"grouped: %s; `plans %s` says where that walk stands", b.id, ago(in), b.behind, ago(b.walkBound),
|
||||
b.grouped, b.behind),
|
||||
Said: fmt.Sprintf("queued behind %s for %s", b.behind, ago(in)),
|
||||
Headline: "Merged changes wait behind a slow delivery",
|
||||
Explanation: fmt.Sprintf("Merges collected for the next delivery have waited %s for the delivery before "+
|
||||
"them, which is taking longer than it should. Nothing is lost.", humanDuration(in)),
|
||||
Resolved: "Merged changes no longer wait behind a slow delivery"})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// watchGroupsNotMerged is the condition of a batch held for a group whose last member has not merged by the merge
|
||||
// window's maximum after the first member merged, naming the member and why (novox/hq ADR 0418 clause 3): nothing of
|
||||
// the group is delivered while it waits.
|
||||
func watchGroupsNotMerged(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, b := range f.batches {
|
||||
if len(b.holds) == 0 || b.created.IsZero() || f.now.Sub(b.created) <= b.atMostWindow {
|
||||
continue
|
||||
}
|
||||
heads := make([]string, 0, len(b.holds))
|
||||
for h := range b.holds {
|
||||
heads = append(heads, h)
|
||||
}
|
||||
sort.Strings(heads)
|
||||
var why []string
|
||||
for _, h := range heads {
|
||||
why = append(why, b.holds[h])
|
||||
}
|
||||
waited := f.now.Sub(b.created)
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id,
|
||||
Kind: kindGroupNotMerged, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s; its first member merged %s ago, past the merge window's maximum (%s), and nothing "+
|
||||
"of the group is delivered while it waits (batch %s)", strings.Join(why, "; "), ago(waited), ago(b.atMostWindow), b.id),
|
||||
Said: "a group waits for a member that has not merged",
|
||||
Headline: "A change waits for the rest of its group to merge",
|
||||
Explanation: fmt.Sprintf("The changes of a group are delivered together, once the last one is merged. "+
|
||||
"One has not merged in %s. Nothing is lost; the merged ones are kept and delivered with it.",
|
||||
humanDuration(waited)),
|
||||
Resolved: "The group's last change merged"})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1,108 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// An unrelated closed batch is cut into a walk beside an open one, and that walk asks its first tier at once
|
||||
// (issue 973, novox/hq ADR 0407 decision 4): a hotfix is never held by an unrelated delivery.
|
||||
func TestAnUnrelatedBatchWalksBesideAnOpenWalk(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
if ws, _ := walks(t, open); len(ws) != 1 || !ws[0].Open() {
|
||||
t.Fatalf("the first walk is not open: %+v", ws)
|
||||
}
|
||||
heard := t0.Add(100 * time.Second)
|
||||
hear(t, open, lone(repoMerge("two", "c2", heard)), heard)
|
||||
advanceHeld(ctx, open)
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 2 || len(bs) != 0 {
|
||||
t.Fatalf("%d walk(s) and %d batch(es); want the unrelated batch cut beside the open walk", len(ws), len(bs))
|
||||
}
|
||||
for _, w := range ws {
|
||||
if !w.Open() || w.Waiting() || deferred(w) || !askedAny(w) {
|
||||
t.Errorf("%s (%s) is not walking: open %v, deferred %v, asked %v, note %q", w.ID, w.Named(), w.Open(),
|
||||
deferred(w), askedAny(w), w.Note)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A closed batch related to an open walk is queued behind it with the reason, while one that shares nothing is
|
||||
// cut beside it; the walk that started is never superseded (ADR 0276 decision 4, issue 973).
|
||||
func TestARelatedBatchWaitsBehindTheOpenWalk(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
first := func() inventory.Plan { ws, _ := walks(t, open); return ws[len(ws)-1] }()
|
||||
heard := t0.Add(100 * time.Second)
|
||||
hear(t, open, lone(repoMerge("one", "c2", heard)), heard)
|
||||
hear(t, open, lone(repoMerge("three", "c3", heard.Add(time.Second))), heard.Add(time.Second))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 2 || len(bs) != 1 {
|
||||
t.Fatalf("%d walk(s) and %d batch(es); want the unrelated merge walking and the related one queued", len(ws), len(bs))
|
||||
}
|
||||
b := bs[0]
|
||||
words := batchWords(b, heard)
|
||||
if b.State != inventory.PlanQueued || b.Delivery.Batch.Behind != first.ID || b.CommitOf("novox/one") != "c2" ||
|
||||
!strings.Contains(words, "queued behind "+first.ID) || !strings.Contains(words, "both change novox/one") {
|
||||
t.Fatalf("the related batch is %s, behind %q, reading %q", b.State, b.Delivery.Batch.Behind, words)
|
||||
}
|
||||
if got, _ := open.inventory.PlanByID(ctx, first.ID); !got.Open() {
|
||||
t.Fatalf("the started walk was %s by the related merge", got.State)
|
||||
}
|
||||
first.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, heard.Add(2*time.Second))
|
||||
if _, bs := walks(t, open); len(bs) != 0 {
|
||||
t.Fatalf("the queued batch was not cut once the walk it waited for ended: %+v", bs)
|
||||
}
|
||||
}
|
||||
|
||||
// A batch on the controller's own path never walks beside another walk, and nothing walks beside it, whatever
|
||||
// the two share (issue 973): it replaces the controller, so it goes alone.
|
||||
func TestAnOwnPathBatchNeverWalksBeside(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
ws, _ := walks(t, open)
|
||||
first := ws[0]
|
||||
heard := t0.Add(100 * time.Second)
|
||||
hear(t, open, lone(repoMerge("mesh-controller", "k1", heard)), heard)
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 1 || !bs[0].OwnPath() || bs[0].State != inventory.PlanQueued ||
|
||||
bs[0].Delivery.Batch.Behind != first.ID || !strings.Contains(batchWords(bs[0], heard), "own path") {
|
||||
t.Fatalf("an own-path batch beside an open walk: %d walk(s), batches %+v", len(ws), bs)
|
||||
}
|
||||
first.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, heard.Add(time.Second))
|
||||
ws, _ = walks(t, open)
|
||||
if len(ws) != 2 || !ws[0].Open() || ws[0].CommitOf("novox/mesh-controller") != "k1" {
|
||||
t.Fatalf("the own-path batch was not cut once the walk ended: %+v", ws)
|
||||
}
|
||||
own := ws[0]
|
||||
// Unrelated to the own-path walk by every footprint, a merge still waits for it.
|
||||
later := heard.Add(time.Minute)
|
||||
hear(t, open, lone(repoMerge("two", "c2", later)), later)
|
||||
ws, bs = walks(t, open)
|
||||
if len(ws) != 2 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != own.ID {
|
||||
t.Fatalf("a merge beside the own-path walk: %d walk(s), batches %+v", len(ws), bs)
|
||||
}
|
||||
}
|
||||
@@ -1,102 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
func TestTheIncidentAConsumerStaysBesideItsDataUntilAPersonPinsIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range storeManifests() {
|
||||
register(t, open, m)
|
||||
}
|
||||
assignAll := func(pairs ...[2]string) {
|
||||
for _, a := range pairs {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
}
|
||||
// The anchor's store and resolver hold the mesh's seats, on record; the laptop runs its own of each.
|
||||
assignAll([2]string{"anchor", "store"}, [2]string{"anchor", "resolver"})
|
||||
for _, seat := range [][2]string{{"mesh-store", "store"}, {"mesh-dns-resolver", "resolver"}} {
|
||||
if err := inv.HoldSeat(ctx, seat[0], catalogue.ScopeMesh, "anchor", seat[1]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
assignAll([2]string{"laptop", "store"}, [2]string{"laptop", "resolver"}, [2]string{"laptop", "network"},
|
||||
[2]string{"laptop", "board"})
|
||||
|
||||
plan, _, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := need(t, plan, "board", "postgres-database"); n.From != "laptop" || n.Module != "store" || !n.KeepsData {
|
||||
t.Fatalf("the consumer was bound to %s/%s (keeps data: %v); its data is beside it", n.From, n.Module, n.KeepsData)
|
||||
}
|
||||
if n := need(t, plan, "network", "wildcard-resolution"); n.From != "anchor" || n.KeepsData {
|
||||
t.Fatalf("the resolver was bound to %s (keeps data: %v); its seat is held on anchor (issue 258)", n.From, n.KeepsData)
|
||||
}
|
||||
|
||||
// Sent, and recorded: only the binding to data.
|
||||
bindings := boundToData(plan, nil)
|
||||
if len(bindings) != 1 || bindings[0].Provider != (catalogue.Chosen{Node: "laptop", Module: "store"}) {
|
||||
t.Fatalf("recorded %+v", bindings)
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, "laptop", bindings); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 {
|
||||
t.Fatalf("a mesh bound where it was sent: %+v, %v", found, err)
|
||||
}
|
||||
|
||||
// The laptop's store taken away: refused, not moved to the anchor's empty one.
|
||||
if err := inv.Unassign(ctx, "laptop", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := planFor(ctx, open, "laptop"); err == nil || !unresolvable(err) ||
|
||||
!strings.Contains(err.Error(), "board on laptop is bound to laptop/store") ||
|
||||
!strings.Contains(err.Error(), "pin laptop postgres-database anchor store") {
|
||||
t.Fatalf("the consumer's store went and it was answered elsewhere: %v", err)
|
||||
}
|
||||
|
||||
// A person pins the anchor's: it moves, said before it is sent, and the record keeps where it was.
|
||||
if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, _, err = planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := need(t, plan, "board", "postgres-database"); n.From != "anchor" {
|
||||
t.Fatalf("pinned to the anchor and bound to %s", n.From)
|
||||
}
|
||||
found, err := probeBindings(ctx, &doctor{open: open})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(found) != 1 || found[0].Kind != kindBindingMoving || found[0].Severity != conditions.Warning ||
|
||||
!strings.Contains(found[0].Summary, "board's postgres-database moves from laptop/store to anchor/store") {
|
||||
t.Fatalf("a pinned move is not said before it is sent: %+v", found)
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
all, err := inv.Bindings(ctx)
|
||||
if err != nil || len(all) != 1 || all[0].MovedFrom != "laptop/store" {
|
||||
t.Fatalf("%+v, %v", all, err)
|
||||
}
|
||||
if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 {
|
||||
t.Fatalf("a move sent is still said: %+v, %v", found, err)
|
||||
}
|
||||
}
|
||||
@@ -39,6 +39,96 @@ func need(t *testing.T, plan catalogue.Resolution, consumer, provision string) c
|
||||
// The incident through the stores: the laptop runs its own store and a consumer of it, the anchor's
|
||||
// store holds the mesh's seat. The consumer stays beside its data, the resolver follows its seat, the
|
||||
// binding is recorded as sent, and a pin — only a pin — moves it, said before the push that carries it.
|
||||
func TestTheIncidentAConsumerStaysBesideItsDataUntilAPersonPinsIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range storeManifests() {
|
||||
register(t, open, m)
|
||||
}
|
||||
assignAll := func(pairs ...[2]string) {
|
||||
for _, a := range pairs {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
}
|
||||
// The anchor's store and resolver hold the mesh's seats, on record; the laptop runs its own of each.
|
||||
assignAll([2]string{"anchor", "store"}, [2]string{"anchor", "resolver"})
|
||||
for _, seat := range [][2]string{{"mesh-store", "store"}, {"mesh-dns-resolver", "resolver"}} {
|
||||
if err := inv.HoldSeat(ctx, seat[0], catalogue.ScopeMesh, "anchor", seat[1]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
assignAll([2]string{"laptop", "store"}, [2]string{"laptop", "resolver"}, [2]string{"laptop", "network"},
|
||||
[2]string{"laptop", "board"})
|
||||
|
||||
plan, _, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := need(t, plan, "board", "postgres-database"); n.From != "laptop" || n.Module != "store" || !n.KeepsData {
|
||||
t.Fatalf("the consumer was bound to %s/%s (keeps data: %v); its data is beside it", n.From, n.Module, n.KeepsData)
|
||||
}
|
||||
if n := need(t, plan, "network", "wildcard-resolution"); n.From != "anchor" || n.KeepsData {
|
||||
t.Fatalf("the resolver was bound to %s (keeps data: %v); its seat is held on anchor (issue 258)", n.From, n.KeepsData)
|
||||
}
|
||||
|
||||
// Sent, and recorded: only the binding to data.
|
||||
bindings := boundToData(plan, nil)
|
||||
if len(bindings) != 1 || bindings[0].Provider != (catalogue.Chosen{Node: "laptop", Module: "store"}) {
|
||||
t.Fatalf("recorded %+v", bindings)
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, "laptop", bindings); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 {
|
||||
t.Fatalf("a mesh bound where it was sent: %+v, %v", found, err)
|
||||
}
|
||||
|
||||
// The laptop's store taken away: refused, not moved to the anchor's empty one.
|
||||
if err := inv.Unassign(ctx, "laptop", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := planFor(ctx, open, "laptop"); err == nil || !unresolvable(err) ||
|
||||
!strings.Contains(err.Error(), "board on laptop is bound to laptop/store") ||
|
||||
!strings.Contains(err.Error(), "pin laptop postgres-database anchor store") {
|
||||
t.Fatalf("the consumer's store went and it was answered elsewhere: %v", err)
|
||||
}
|
||||
|
||||
// A person pins the anchor's: it moves, said before it is sent, and the record keeps where it was.
|
||||
if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, _, err = planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := need(t, plan, "board", "postgres-database"); n.From != "anchor" {
|
||||
t.Fatalf("pinned to the anchor and bound to %s", n.From)
|
||||
}
|
||||
found, err := probeBindings(ctx, &doctor{open: open})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(found) != 1 || found[0].Kind != kindBindingMoving || found[0].Severity != conditions.Warning ||
|
||||
!strings.Contains(found[0].Summary, "board's postgres-database moves from laptop/store to anchor/store") {
|
||||
t.Fatalf("a pinned move is not said before it is sent: %+v", found)
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
all, err := inv.Bindings(ctx)
|
||||
if err != nil || len(all) != 1 || all[0].MovedFrom != "laptop/store" {
|
||||
t.Fatalf("%+v, %v", all, err)
|
||||
}
|
||||
if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 {
|
||||
t.Fatalf("a move sent is still said: %+v, %v", found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// What one machine's resolution says against its record.
|
||||
func TestBindingFindingsSayAKeptMoveAndAMoveNothingAskedFor(t *testing.T) {
|
||||
|
||||
@@ -1,207 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// **A filling bucket or log is said before it is full** (novox/hq ADR 0297 §6, issue 501).
|
||||
//
|
||||
// A module's bucket and its log each have a cap on the bus, and when either is full the bus refuses
|
||||
// the next write: the module stops doing what it writes for, and nothing said so before. On
|
||||
// 2026-10-11 the issue tracker's bucket held a sixth of its cap after two days, growing toward a stop
|
||||
// nobody would have heard coming. So the self-check reads every module's bucket and log, and one at
|
||||
// fillRaiseAt of its cap or more raises a condition naming the module, the bucket or log, and how full
|
||||
// it is.
|
||||
//
|
||||
// **Cleared by observation below fillClearBelow, not below fillRaiseAt**: a bucket or log hovering at
|
||||
// its threshold would otherwise be raised and cleared on every run. Between the two, an open condition
|
||||
// is kept and none is raised.
|
||||
//
|
||||
// **One that cannot be read is said, and the rest are still judged.** An unanswered question about one
|
||||
// stream is no reason to know nothing of the others: it is a finding of its own, naming the bucket or
|
||||
// log and why, and a fill condition already open for it is kept, because not knowing is not a pass.
|
||||
|
||||
// The fill thresholds, in percent of a bucket's or log's cap.
|
||||
const (
|
||||
fillRaiseAt = 75
|
||||
fillClearBelow = 70
|
||||
)
|
||||
|
||||
// The conditions the fill probe raises: one filling, and one that could not be read.
|
||||
const (
|
||||
kindBucketOrLogFilling = "bucket-or-log-filling"
|
||||
kindBucketOrLogUnread = "bucket-or-log-unread"
|
||||
)
|
||||
|
||||
// probeFillID is the probe's id in the registry.
|
||||
const probeFillID = "D-fill"
|
||||
|
||||
// bucketOrLog is one module's bucket or log as the fill probe reads it.
|
||||
type bucketOrLog struct {
|
||||
Module string
|
||||
// Kind is `bucket` or `log`; Name its local name; Stream the stream it is on the bus.
|
||||
Kind, Name, Stream string
|
||||
}
|
||||
|
||||
// filled is how full one bucket or log is, read from the bus.
|
||||
type filled struct {
|
||||
Bytes, Max uint64
|
||||
}
|
||||
|
||||
// percent is how full, in whole percent, rounded down.
|
||||
func (f filled) percent() uint64 {
|
||||
if f.Max == 0 {
|
||||
return 0
|
||||
}
|
||||
return f.Bytes * 100 / f.Max
|
||||
}
|
||||
|
||||
// fillKey is where a bucket's or log's fill condition is kept.
|
||||
func fillKey(s bucketOrLog) string { return conditions.Key(conditions.ScopeBus, s.Stream, "filling") }
|
||||
|
||||
// fillObservation is what one bucket's or log's fill says: a finding at fillRaiseAt or above, and,
|
||||
// while its condition is open, at fillClearBelow or above too; nothing otherwise, which is what clears
|
||||
// it. The operator's words are the kind's (plain_words.go); the summary and the evidence name the
|
||||
// module, the bucket or log, and the fill.
|
||||
func fillObservation(s bucketOrLog, f filled, open bool) (conditions.Observation, bool) {
|
||||
if f.Max == 0 {
|
||||
return conditions.Observation{}, false // one with no cap cannot fill
|
||||
}
|
||||
// Compared in bytes, not rounded percent: 74.9% is not 75%.
|
||||
atRaise := f.Bytes*100 >= f.Max*fillRaiseAt
|
||||
aboveClear := f.Bytes*100 >= f.Max*fillClearBelow
|
||||
if !atRaise && !(open && aboveClear) {
|
||||
return conditions.Observation{}, false
|
||||
}
|
||||
pct := f.percent()
|
||||
return conditions.Observation{
|
||||
Scope: conditions.ScopeBus, ID: s.Stream, Token: "filling", Kind: kindBucketOrLogFilling,
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s's %s %s holds %s of %s, %d%%: at its cap the bus refuses the module's next write",
|
||||
s.Module, s.Kind, s.Name, mibWords(f.Bytes), mibWords(f.Max), pct),
|
||||
Said: fmt.Sprintf("module %s, %s %s (stream %s): %d of %d bytes, %d%%", s.Module, s.Kind, s.Name,
|
||||
s.Stream, f.Bytes, f.Max, pct),
|
||||
}, true
|
||||
}
|
||||
|
||||
// unreadObservation says one bucket or log could not be read, and why. Asked over the network, so one
|
||||
// look can be wrong: raised on the second look in a row (confirm.go).
|
||||
func unreadObservation(s bucketOrLog, why error) conditions.Observation {
|
||||
return conditions.Observation{
|
||||
Scope: conditions.ScopeBus, ID: s.Stream, Token: "unread", Kind: kindBucketOrLogUnread,
|
||||
Severity: conditions.Warning, Confirm: true,
|
||||
Summary: fmt.Sprintf("%s's %s %s could not be read, so how full it is is not known", s.Module, s.Kind, s.Name),
|
||||
Said: fmt.Sprintf("module %s, %s %s (stream %s): %v", s.Module, s.Kind, s.Name, s.Stream, why),
|
||||
}
|
||||
}
|
||||
|
||||
// keptFilling is an open fill condition said again for a bucket or log that could not be read this
|
||||
// time: not knowing how full it is now is no reason to say it has room. Only ever made from a condition
|
||||
// read back as open — its own summary and severity, never words made up for it.
|
||||
func keptFilling(s bucketOrLog, open conditions.Condition, why error) conditions.Observation {
|
||||
return conditions.Observation{
|
||||
Scope: conditions.ScopeBus, ID: s.Stream, Token: "filling", Kind: kindBucketOrLogFilling,
|
||||
Severity: open.Severity, Summary: open.Summary,
|
||||
Said: fmt.Sprintf("module %s, %s %s (stream %s): not read this time (%v); kept open as it was",
|
||||
s.Module, s.Kind, s.Name, s.Stream, why),
|
||||
}
|
||||
}
|
||||
|
||||
// mibWords is a size as a person reads it, in MiB with one decimal.
|
||||
func mibWords(b uint64) string {
|
||||
return fmt.Sprintf("%.1f MiB", float64(b)/(1024*1024))
|
||||
}
|
||||
|
||||
// readFill is how full one bucket or log is on the bus; found false when it is not on the bus.
|
||||
type readFill func(ctx context.Context, s bucketOrLog) (f filled, found bool, err error)
|
||||
|
||||
// openFill is the fill condition open under a key, if one is, or why it could not be read.
|
||||
type openFill func(ctx context.Context, key string) (conditions.Condition, bool, error)
|
||||
|
||||
// judgeFills judges every bucket and log on its own: one that cannot be read is said as unread, with
|
||||
// its fill condition kept only when that condition was read back and is open — when it cannot be read
|
||||
// either, nothing is said of its fill, which the unread finding already covers. Every other is judged
|
||||
// by its fill; for one of those, a condition that cannot be read is taken as open, so an unknown never
|
||||
// clears a fill measured between fillClearBelow and fillRaiseAt.
|
||||
func judgeFills(ctx context.Context, all []bucketOrLog, read readFill, open openFill) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, s := range all {
|
||||
f, found, err := read(ctx, s)
|
||||
if err != nil {
|
||||
out = append(out, unreadObservation(s, err))
|
||||
if c, isOpen, cerr := open(ctx, fillKey(s)); cerr == nil && isOpen {
|
||||
out = append(out, keptFilling(s, c, err))
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !found {
|
||||
continue // not on the bus: created on the controller's next raise, and nothing there can fill
|
||||
}
|
||||
_, isOpen, cerr := open(ctx, fillKey(s))
|
||||
isOpen = isOpen || cerr != nil
|
||||
if o, said := fillObservation(s, f, isOpen); said {
|
||||
out = append(out, o)
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].Key() < out[j].Key() })
|
||||
return out
|
||||
}
|
||||
|
||||
// declaredBucketsAndLogs is every module's bucket and log the catalogue declares, by its stream.
|
||||
func declaredBucketsAndLogs(ctx context.Context, d *doctor) ([]bucketOrLog, error) {
|
||||
buckets, err := d.open.inventory.DeclaredBuckets(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logs, err := d.open.inventory.DeclaredLogs(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []bucketOrLog
|
||||
for _, b := range buckets {
|
||||
out = append(out, bucketOrLog{Module: b.Module, Kind: "bucket", Name: b.Name, Stream: "KV_" + b.Bucket()})
|
||||
}
|
||||
for _, l := range logs {
|
||||
out = append(out, bucketOrLog{Module: l.Module, Kind: "log", Name: l.Name, Stream: l.Stream()})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// probeFill reads every module's bucket and log on the bus and says each one filling toward its cap,
|
||||
// and each one that could not be read.
|
||||
func probeFill(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
all, err := declaredBucketsAndLogs(ctx, d)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js := d.js.Context()
|
||||
read := func(ctx context.Context, s bucketOrLog) (filled, bool, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return filled{}, false, err
|
||||
}
|
||||
info, err := js.StreamInfo(s.Stream, nats.Context(ctx))
|
||||
switch {
|
||||
case errors.Is(err, nats.ErrStreamNotFound):
|
||||
return filled{}, false, nil
|
||||
case err != nil:
|
||||
return filled{}, false, err
|
||||
case info.Config.MaxBytes <= 0:
|
||||
return filled{}, true, nil
|
||||
}
|
||||
return filled{Bytes: info.State.Bytes, Max: uint64(info.Config.MaxBytes)}, true, nil
|
||||
}
|
||||
open := func(ctx context.Context, key string) (conditions.Condition, bool, error) {
|
||||
if d.keeper == nil {
|
||||
return conditions.Condition{}, false, nil
|
||||
}
|
||||
return d.keeper.Get(ctx, key)
|
||||
}
|
||||
return judgeFills(ctx, all, read, open), nil
|
||||
}
|
||||
@@ -1,165 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// **A filling log or bucket is said at three quarters of its cap and cleared below seven tenths**
|
||||
// (novox/hq ADR 0297 §6): raised at 75% naming the module, the bucket or log and its fill; not at 74.9%;
|
||||
// kept between 70% and 75% while it is open, and not raised there when it is not; and cleared — said no
|
||||
// more — once it reads below 70%, open or not.
|
||||
func TestAFillingBucketOrLogIsRaisedAtThreeQuartersAndClearedBelowSevenTenths(t *testing.T) {
|
||||
const mib = 1024 * 1024
|
||||
log := bucketOrLog{Module: "mesh-issues", Kind: "log", Name: "changes", Stream: "LOG_mesh-issues_changes"}
|
||||
bucket := bucketOrLog{Module: "mesh-issues", Kind: "bucket", Name: "issues", Stream: "KV_mesh-issues_issues"}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
of bucketOrLog
|
||||
bytes uint64
|
||||
max uint64
|
||||
open bool
|
||||
said bool
|
||||
}{
|
||||
{"a log at exactly 75%", log, 768 * mib, 1024 * mib, false, true},
|
||||
{"a bucket at exactly 75%", bucket, 48 * mib, 64 * mib, false, true},
|
||||
{"a bucket full", bucket, 64 * mib, 64 * mib, false, true},
|
||||
{"a log just under 75%", log, 768*mib - 1, 1024 * mib, false, false},
|
||||
{"a bucket at 72%, not open", bucket, 64 * mib * 72 / 100, 64 * mib, false, false},
|
||||
{"a bucket at 72%, open: kept", bucket, 64 * mib * 72 / 100, 64 * mib, true, true},
|
||||
{"a log at exactly 70%, open: kept", log, 700 * mib, 1000 * mib, true, true},
|
||||
{"a log just under 70%, open: cleared", log, 700*mib - 1, 1000 * mib, true, false},
|
||||
{"a bucket at 10%, open: cleared", bucket, 64 * mib / 10, 64 * mib, true, false},
|
||||
{"a bucket with no cap", bucket, 100, 0, true, false},
|
||||
} {
|
||||
o, said := fillObservation(c.of, filled{Bytes: c.bytes, Max: c.max}, c.open)
|
||||
if said != c.said {
|
||||
t.Errorf("%s: said %v, want %v", c.name, said, c.said)
|
||||
continue
|
||||
}
|
||||
if !said {
|
||||
continue
|
||||
}
|
||||
if o.Key() != fillKey(c.of) || o.Kind != kindBucketOrLogFilling || o.Severity != conditions.Warning {
|
||||
t.Errorf("%s: raised as %s (%s, %s)", c.name, o.Key(), o.Kind, o.Severity)
|
||||
}
|
||||
for _, part := range []string{c.of.Module, c.of.Kind + " " + c.of.Name, "%", " of "} {
|
||||
if !strings.Contains(o.Summary, part) {
|
||||
t.Errorf("%s: does not name %q: %q", c.name, part, o.Summary)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(o.Said, "bytes") {
|
||||
t.Errorf("%s: the evidence does not say the fill in bytes: %q", c.name, o.Said)
|
||||
}
|
||||
}
|
||||
o, _ := fillObservation(log, filled{Bytes: 800 * mib, Max: 1024 * mib}, false)
|
||||
if !strings.Contains(o.Summary, "800.0 MiB of 1024.0 MiB, 78%") {
|
||||
t.Errorf("the fill is said as %q", o.Summary)
|
||||
}
|
||||
}
|
||||
|
||||
// **One bucket or log that cannot be read does not stop the others being judged** (review of #231): it is
|
||||
// said as unread with its reason, a fill condition open for it is kept, and every other bucket and log
|
||||
// is judged by its own fill.
|
||||
func TestAnUnreadableBucketOrLogIsSaidAndTheRestAreStillJudged(t *testing.T) {
|
||||
const mib = 1024 * 1024
|
||||
broken := bucketOrLog{Module: "a", Kind: "bucket", Name: "broken", Stream: "KV_a_broken"}
|
||||
brokenOpen := bucketOrLog{Module: "a", Kind: "log", Name: "was-filling", Stream: "LOG_a_was-filling"}
|
||||
full := bucketOrLog{Module: "b", Kind: "log", Name: "changes", Stream: "LOG_b_changes"}
|
||||
empty := bucketOrLog{Module: "c", Kind: "bucket", Name: "quiet", Stream: "KV_c_quiet"}
|
||||
absent := bucketOrLog{Module: "d", Kind: "bucket", Name: "not-yet", Stream: "KV_d_not-yet"}
|
||||
refusal := errors.New("the bus did not answer")
|
||||
read := func(_ context.Context, s bucketOrLog) (filled, bool, error) {
|
||||
switch s {
|
||||
case broken, brokenOpen:
|
||||
return filled{}, false, refusal
|
||||
case full:
|
||||
return filled{Bytes: 900 * mib, Max: 1000 * mib}, true, nil
|
||||
case empty:
|
||||
return filled{Bytes: 1, Max: 64 * mib}, true, nil
|
||||
}
|
||||
return filled{}, false, nil
|
||||
}
|
||||
open := func(_ context.Context, key string) (conditions.Condition, bool, error) {
|
||||
if key == fillKey(brokenOpen) {
|
||||
return conditions.Condition{Key: key, Severity: conditions.Warning,
|
||||
Summary: "a's log was-filling holds 800.0 MiB of 1024.0 MiB, 78%"}, true, nil
|
||||
}
|
||||
return conditions.Condition{}, false, nil
|
||||
}
|
||||
got := map[string]conditions.Observation{}
|
||||
for _, o := range judgeFills(context.Background(), []bucketOrLog{broken, brokenOpen, full, empty, absent}, read, open) {
|
||||
got[o.Key()] = o
|
||||
}
|
||||
for _, s := range []bucketOrLog{broken, brokenOpen} {
|
||||
o, said := got[conditions.Key(conditions.ScopeBus, s.Stream, "unread")]
|
||||
if !said || o.Kind != kindBucketOrLogUnread || !o.Confirm || !strings.Contains(o.Said, refusal.Error()) {
|
||||
t.Errorf("%s could not be read and was said as %+v", s.Stream, o)
|
||||
}
|
||||
}
|
||||
if o, kept := got[fillKey(brokenOpen)]; !kept || !strings.Contains(o.Said, "kept open") {
|
||||
t.Errorf("an open fill condition of a log not read this time was not kept: %+v", o)
|
||||
}
|
||||
if _, said := got[fillKey(broken)]; said {
|
||||
t.Error("a bucket not read and not filling was said filling")
|
||||
}
|
||||
if o, said := got[fillKey(full)]; !said || o.Kind != kindBucketOrLogFilling {
|
||||
t.Errorf("a log at 90%% beside an unreadable one was not judged: %+v", got)
|
||||
}
|
||||
if len(got) != 4 {
|
||||
t.Errorf("said %d findings, want 4 (two unread, one kept, one filling): %v", len(got), got)
|
||||
}
|
||||
}
|
||||
|
||||
// Both kinds have plain words of their own, which hold to the plain rule (novox/hq ADR 0253).
|
||||
func TestAFillingOrUnreadBucketOrLogIsSaidInPlainWords(t *testing.T) {
|
||||
for _, kind := range []string{kindBucketOrLogFilling, kindBucketOrLogUnread} {
|
||||
wording, has := plainWordings[kind]
|
||||
if !has {
|
||||
t.Errorf("%s has no plain words", kind)
|
||||
continue
|
||||
}
|
||||
if why, ok := conditions.PlainWords(wording(conditions.Observation{Kind: kind})); !ok {
|
||||
t.Errorf("%s: %s", kind, why)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **When neither the bucket or log nor its condition can be read, nothing is said of its fill** (second
|
||||
// review of #231): the unread finding covers it, and no fill is invented for it. For one that is read and
|
||||
// measured between seven tenths and three quarters, a condition that cannot be read is taken as open, so
|
||||
// an unknown never clears it.
|
||||
func TestNothingIsSaidOfAFillWhenNeitherItNorItsConditionCanBeRead(t *testing.T) {
|
||||
const mib = 1024 * 1024
|
||||
broken := bucketOrLog{Module: "a", Kind: "log", Name: "changes", Stream: "LOG_a_changes"}
|
||||
between := bucketOrLog{Module: "b", Kind: "bucket", Name: "issues", Stream: "KV_b_issues"}
|
||||
read := func(_ context.Context, s bucketOrLog) (filled, bool, error) {
|
||||
if s == broken {
|
||||
return filled{}, false, errors.New("the bus did not answer")
|
||||
}
|
||||
return filled{Bytes: 72 * mib, Max: 100 * mib}, true, nil
|
||||
}
|
||||
open := func(context.Context, string) (conditions.Condition, bool, error) {
|
||||
return conditions.Condition{}, false, errors.New("the conditions bucket did not answer")
|
||||
}
|
||||
got := map[string]conditions.Observation{}
|
||||
for _, o := range judgeFills(context.Background(), []bucketOrLog{broken, between}, read, open) {
|
||||
got[o.Key()] = o
|
||||
}
|
||||
if o, said := got[fillKey(broken)]; said {
|
||||
t.Fatalf("a fill was said for a log whose fill and condition could not be read: %+v", o)
|
||||
}
|
||||
if _, said := got[conditions.Key(conditions.ScopeBus, broken.Stream, "unread")]; !said {
|
||||
t.Error("the log that could not be read was not said as unread")
|
||||
}
|
||||
if _, kept := got[fillKey(between)]; !kept {
|
||||
t.Error("a bucket at 72% whose condition could not be read was cleared")
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Errorf("said %d findings, want 2: %v", len(got), got)
|
||||
}
|
||||
}
|
||||
@@ -15,7 +15,6 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
@@ -167,10 +166,6 @@ func buildFrom(result link.BuildResult) inventory.Build {
|
||||
for _, r := range result.Read {
|
||||
kept.Read = append(kept.Read, inventory.ReadRepository{Repository: r.Repository, Ref: r.Ref})
|
||||
}
|
||||
// What it was made from, as files (novox/hq ADR 0267): the planner maps the next merge onto it.
|
||||
for _, s := range result.Sources {
|
||||
kept.Sources = append(kept.Sources, inventory.BuildSource{Repository: s.Repository, Ref: s.Ref, Paths: s.Paths})
|
||||
}
|
||||
var announced []inventory.Artifact
|
||||
for _, made := range result.Made {
|
||||
announced = append(announced, inventory.Artifact{
|
||||
@@ -183,10 +178,8 @@ func buildFrom(result link.BuildResult) inventory.Build {
|
||||
kept.Manifest = recordedManifest(result.Manifest, announced)
|
||||
// The module name comes from the manifest, which only exists when the build got that far.
|
||||
if len(kept.Manifest) > 0 {
|
||||
if list, err := judgedWith(result); err == nil {
|
||||
if m, err := catalogue.ParseManifestWith(kept.Manifest, list); err == nil {
|
||||
kept.Module = m.Module
|
||||
}
|
||||
if m, err := catalogue.ParseManifest(kept.Manifest); err == nil {
|
||||
kept.Module = m.Module
|
||||
}
|
||||
}
|
||||
return kept
|
||||
@@ -458,10 +451,6 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Through a verb, only a repository the catalogue builds from (novox/hq ADR 0266).
|
||||
if err := verbMayAsk(ctx, source, repository); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
ident, err := openIdentity(ctx)
|
||||
if err != nil {
|
||||
@@ -486,7 +475,6 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
|
||||
Seats: seatBases(ctx),
|
||||
DryRun: dryRun,
|
||||
}
|
||||
request.Reuse = reuseAsked(ctx, dryRun)
|
||||
fmt.Printf("asked for %s", source)
|
||||
if source.Seat != "" {
|
||||
fmt.Printf(" (%s)", repository)
|
||||
@@ -512,10 +500,8 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
|
||||
// never before, so an ask that failed never reads as a build in flight. A dry run registers nothing, and
|
||||
// is not kept.
|
||||
keep := !dryRun && asker != ""
|
||||
// Asked at the terminal is what lets its outcome register a module from a repository the catalogue does
|
||||
// not build it from (novox/hq ADR 0266); never through a verb.
|
||||
asked := inventory.BuildRequest{ID: request.ID, Repository: source.Repository, Seat: source.Seat, Path: path,
|
||||
Ref: ref, For: asker, AtTerminal: startedAtTheTerminal()}
|
||||
Ref: ref, For: asker}
|
||||
|
||||
if wait == 0 {
|
||||
// Asked and not waited for (novox/hq issue 176): the outcome is the role's event, and the
|
||||
@@ -605,11 +591,6 @@ func saysWhenThePolicyActs(ctx context.Context, inv *inventory.Inventory, module
|
||||
// written with the same values.
|
||||
func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResult) (
|
||||
catalogue.Manifest, inventory.Build, error) {
|
||||
// The list of predefined keys as the store keeps it now: another controller process may have kept a newer one.
|
||||
if err := loadPredefined(ctx, inv); err != nil {
|
||||
return catalogue.Manifest{}, inventory.Build{}, fmt.Errorf("the list of predefined keys kept cannot be "+
|
||||
"read: %w", err)
|
||||
}
|
||||
kept := buildFrom(result)
|
||||
if err := inv.RecordBuild(ctx, kept); err != nil {
|
||||
return catalogue.Manifest{}, kept, err
|
||||
@@ -620,15 +601,7 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
||||
return catalogue.Manifest{}, kept, fmt.Errorf("%s could not build %s:\n%s",
|
||||
result.On, result.Repository, result.Failed)
|
||||
}
|
||||
// Judged first against the list the build was judged against: the catalogue's at the commit built (novox/hq
|
||||
// ADR 0348 §7), so a module naming a key its own commit adds is read; and again below against the list the
|
||||
// mesh publishes, before it is registered.
|
||||
list, err := judgedWith(result)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On,
|
||||
result.Repository, short(result.Commit), err)
|
||||
}
|
||||
manifest, err := catalogue.ParseManifestWith(kept.Manifest, list)
|
||||
manifest, err := catalogue.ParseManifest(kept.Manifest)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, kept, fmt.Errorf("%s built %s and what came back is not a manifest: %w",
|
||||
result.On, result.Repository, err)
|
||||
@@ -658,21 +631,6 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), and the mesh does not register it: %w",
|
||||
result.On, result.Repository, short(result.Commit), err)
|
||||
}
|
||||
// **Only from the repository the catalogue builds the module from** (novox/hq ADR 0266): else the trunk
|
||||
// below is the trunk of whatever repository was built, which may be one an agent made — and a module named
|
||||
// `sudo` from it would be what the next push sends. Another repository is the operator's, at the terminal.
|
||||
trunk := result.Trunk
|
||||
if was, err := inv.SourceOf(ctx, manifest.Module); err == nil && followedBranch(was.Ref) != "" {
|
||||
trunk = followedBranch(was.Ref)
|
||||
}
|
||||
if trunk == "" {
|
||||
trunk = "main"
|
||||
}
|
||||
repoID, err := mayRegisterFrom(ctx, inv, manifest.Module, recorded, result.ID, result.Path, trunk)
|
||||
if err != nil {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On,
|
||||
manifest.Module, short(result.Commit), err)
|
||||
}
|
||||
// **Only a commit on the trunk is published** (novox/hq ADR 0238): a commit off its repository's
|
||||
// default branch — a pull request's head, a feature branch built by hand, a `rebuild` or `replay
|
||||
// --register` of one — is for checking, and is never a module's version; nothing could then send it.
|
||||
@@ -686,12 +644,6 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On,
|
||||
manifest.Module, short(result.Commit), err)
|
||||
}
|
||||
// **Never a build in the scratch namespace** (novox/hq ADR 0239 §9, ADR 0350 §2): what a check builds is
|
||||
// registered only as the walk's copy in the module's own namespace.
|
||||
if scratch := inScratch(result.Made); scratch != "" {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %s is in the scratch "+
|
||||
"namespace, which only a walk's copy leaves (ADR 0350)", result.On, manifest.Module, short(result.Commit), scratch)
|
||||
}
|
||||
// **A build that failed its gate is never registered again** (novox/hq ADR 0236): an outcome heard
|
||||
// twice, or replayed, would otherwise make the build a rollback put back what the module is again,
|
||||
// and the next push would send it.
|
||||
@@ -702,23 +654,6 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
||||
"back: it is recorded and not registered again — a newer build is", result.On, manifest.Module,
|
||||
short(result.Commit))
|
||||
}
|
||||
// **The newest trunk build's list of predefined keys is published** (novox/hq ADR 0348 §7): a build of the
|
||||
// catalogue that may be registered keeps its list when its commit is newer than the one kept. Then the module
|
||||
// is judged against the list the mesh publishes, as every manifest read again is: one naming a key that list
|
||||
// does not hold, or of another type, is refused here and never enters the catalogue.
|
||||
if carried, err := carriedList(result); err != nil {
|
||||
return manifest, kept, err
|
||||
} else if carried != nil {
|
||||
if err := publishPredefined(ctx, inv, result, carried); err != nil {
|
||||
fmt.Printf("%s: the list of predefined keys it carried is not published: %v\n", result.ID, err)
|
||||
}
|
||||
}
|
||||
if manifest.NamesPredefined() {
|
||||
if _, err := catalogue.ParseManifest(kept.Manifest); err != nil {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: judged against the "+
|
||||
"list of predefined keys the mesh publishes, %w", result.On, manifest.Module, short(result.Commit), err)
|
||||
}
|
||||
}
|
||||
// **A build whose source is unchanged is never a move** (novox/hq issue 280): a rebuild made from
|
||||
// what the build the mesh stands on was made from registers that build's artifacts at the new
|
||||
// commit, so no machine is sent a new digest for a source nobody changed — an image is not
|
||||
@@ -743,10 +678,6 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
||||
}
|
||||
return manifest, kept, err
|
||||
}
|
||||
// Which repository it is registered from, by the forge's own id (novox/hq ADR 0266).
|
||||
if err := inv.SetSourceIdentity(ctx, manifest.Module, repoID); err != nil {
|
||||
return manifest, kept, err
|
||||
}
|
||||
// The keep set just moved, and new bytes just landed (novox/hq ADR 0189). Asked here rather
|
||||
// than on a timer of its own: this is the only moment either is true. Never fatal — the build
|
||||
// worked and the module is registered.
|
||||
@@ -754,16 +685,6 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
||||
return manifest, kept, nil
|
||||
}
|
||||
|
||||
// inScratch is the first artifact a build made into the scratch namespace; empty when none is.
|
||||
func inScratch(made []link.MadeArtifact) string {
|
||||
for _, m := range made {
|
||||
if path, kept := catalogue.InArtifactStore(catalogue.Recorded(m.Reference)); kept && strings.HasPrefix(path, builder.ScratchPrefix) {
|
||||
return m.Reference
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// errOffTheTrunk is a build of a commit off its repository's trunk, which is never published.
|
||||
var errOffTheTrunk = errors.New("the commit is not on its repository's trunk: a commit off the trunk is checked, " +
|
||||
"never published (ADR 0238) — merge it, and the merge builds it")
|
||||
@@ -798,9 +719,6 @@ func buildAndShow(ctx context.Context, source buildSource, path, ref string, wai
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := verbMayAsk(ctx, source, repository); err != nil {
|
||||
return err
|
||||
}
|
||||
ident, err := openIdentity(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -852,9 +770,6 @@ type answers struct {
|
||||
sources map[string]inventory.Source
|
||||
// waiting is every machine not running what the mesh would send it.
|
||||
waiting []inventory.Machine
|
||||
// recordedWaits is, per machine, every move of a recorded module that waits there for a person (novox/hq ADR
|
||||
// 0377 §6): the machine is behind for these, and `push <node>` sends them.
|
||||
recordedWaits map[string][]string
|
||||
// reported is every machine's last word beside when it was last sent a declaration — the
|
||||
// pair that answers "has it caught up", which waiting alone cannot (the sent digest is
|
||||
// recorded at send, not at apply).
|
||||
@@ -888,9 +803,6 @@ type answers struct {
|
||||
// public name on the machine went dark. The holds were correct; they were recorded only in the
|
||||
// machine's own state file, and the one visible symptom was a count that did not add up.
|
||||
untaken map[string]map[string]int
|
||||
// leftOut is, per machine, every module of its set its composition leaves out, and why (novox/hq issue
|
||||
// 380): assigned and not applied, which every push said only in passing. Not well while there is any.
|
||||
leftOut map[string][]leftOutModule
|
||||
// unheld is every module on a machine whose resources are applied through a seat nothing on
|
||||
// that machine holds (novox/hq ADR 0207), with the modules that could hold it. Reported, not
|
||||
// refused, until the switch — and while there is any, the mesh is not all well: the order the
|
||||
@@ -919,10 +831,6 @@ type answers struct {
|
||||
// applied, is not well: an assignment somebody made is not made yet, or will not be.
|
||||
pending []inventory.PendingAssignment
|
||||
pendingUnread string
|
||||
// lastSent is every machine's last recorded send, by name, with what the machine answered of it (novox/hq
|
||||
// issue 485); lastSentUnread why it could not be read.
|
||||
lastSent map[string]inventory.Send
|
||||
lastSentUnread string
|
||||
}
|
||||
|
||||
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
|
||||
@@ -934,34 +842,6 @@ type answers struct {
|
||||
//
|
||||
// Routed through the artifact store as the network reaches it now (novox/hq 04-ISSUES/102): a
|
||||
// base is recorded by digest and path, and a build machine needs something it can pull.
|
||||
// runningBy is, keyed "<module>/<artifact>", each artifact of the build the mesh has registered for its module, as a
|
||||
// builder reaches the artifact store (novox/hq issue 721): what a check compares its builds with. Empty when it cannot
|
||||
// be read, said, and then nothing is said unchanged.
|
||||
func runningBy(ctx context.Context) map[string]string {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read what this mesh runs, so the check compares nothing: %v\n", err)
|
||||
// empty-on-error: said above; a check given nothing to compare with says no module unchanged, and all move
|
||||
return nil
|
||||
}
|
||||
defer open.Close()
|
||||
running, err := open.inventory.RegisteredHeld(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read what this mesh runs, so the check compares nothing: %v\n", err)
|
||||
// empty-on-error: said above; a check given nothing to compare with says no module unchanged, and all move
|
||||
return nil
|
||||
}
|
||||
address, err := whereABuilderReachesTheStore(ctx, open.inventory)
|
||||
if err != nil || address == "" {
|
||||
return running
|
||||
}
|
||||
routed := make(map[string]string, len(running))
|
||||
for key, reference := range running {
|
||||
routed[key] = catalogue.Rerouted(reference, address)
|
||||
}
|
||||
return routed
|
||||
}
|
||||
|
||||
func heldBy(ctx context.Context) map[string]string {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
|
||||
@@ -1,131 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func TestABuildHeardIsRecordedAndRegistered(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
manifest, _ := json.Marshal(map[string]any{"module": "shop", "version": "3"})
|
||||
m, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, link.BuildResult{
|
||||
ID: "b-1", Repository: "http://forge.internal:20000/novox/shop.git", Path: "modules/shop",
|
||||
Ref: "main", On: "anchor", Commit: "abcdef0123", Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/shop"},
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m.Module != "shop" {
|
||||
t.Fatalf("registered %q", m.Module)
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, held := shelf["shop"]; !held {
|
||||
t.Fatal("the module a heard build produced is not in the catalogue")
|
||||
}
|
||||
src, err := open.inventory.SourceOf(ctx, "shop")
|
||||
if err != nil || src.Seat != "git" || src.Repository != "novox/shop" || src.BuiltFrom != "abcdef0123" {
|
||||
t.Fatalf("the source is the seat and the path, never the URL: %+v %v", src, err)
|
||||
}
|
||||
builds, err := open.inventory.Builds(ctx, "shop", 5)
|
||||
if err != nil || len(builds) != 1 || builds[0].ID != "b-1" {
|
||||
t.Fatalf("the build is not recorded once: %v %v", builds, err)
|
||||
}
|
||||
|
||||
named, _ := json.Marshal(map[string]any{"module": "idp", "version": "1", "resources": []any{
|
||||
map[string]any{"id": "server", "type": "container", "image": "x@sha256:aa",
|
||||
"env": map[string]any{"KC_HOSTNAME": "https://login.mesh-one.be"}}}})
|
||||
_, _, err = takeIn(ctx, open.inventory, link.BuildResult{
|
||||
ID: "b-2", Repository: "/r", On: "anchor", Commit: "0123456789", Manifest: named})
|
||||
if err == nil || !strings.Contains(err.Error(), "does not register it") {
|
||||
t.Fatalf("a definition naming an installation was taken in: %v", err)
|
||||
}
|
||||
if shelf, _ := open.inventory.Catalogue(ctx); shelf["idp"].Module != "" {
|
||||
t.Fatal("the refused module was registered anyway")
|
||||
}
|
||||
if builds, _ := open.inventory.Builds(ctx, "idp", 5); len(builds) != 1 {
|
||||
t.Fatalf("the refused build was not recorded: %v", builds)
|
||||
}
|
||||
|
||||
_, _, err = takeIn(ctx, open.inventory, link.BuildResult{ID: "b-3", Repository: "/r", On: "anchor", Failed: "no compiler"})
|
||||
if err == nil || !strings.Contains(err.Error(), "no compiler") {
|
||||
t.Fatalf("a failure is said in the builder's words: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABuildAtACommitKeepsTheBranchTheModuleFollows(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
manifest, _ := json.Marshal(map[string]any{"module": "unifi", "version": "1"})
|
||||
result := func(id, ref, commit string) link.BuildResult {
|
||||
return link.BuildResult{ID: id, Repository: "http://forge.internal:20000/novox/mesh-catalog.git",
|
||||
Path: "modules/unifi", Ref: ref, On: "anchor", Commit: commit, Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, result("b-1", "main", "1111111aaaa"))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, result("b-2", "9c97a8a", "9c97a8a1d2c3")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
src, err := open.inventory.SourceOf(ctx, "unifi")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if src.Ref != "main" || src.BuiltFrom != "9c97a8a1d2c3" {
|
||||
t.Errorf("after a build at a commit the module follows %q, built from %q; want main, 9c97a8a1d2c3", src.Ref, src.BuiltFrom)
|
||||
}
|
||||
|
||||
// One new to the catalogue, first built at a commit, follows the default branch.
|
||||
other, _ := json.Marshal(map[string]any{"module": "letta", "version": "1"})
|
||||
r := result("b-3", "deadbeef", "deadbeefcafe")
|
||||
r.Manifest, r.Path = other, "modules/letta"
|
||||
if _, _, err := takeIn(ctx, open.inventory, r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if src, _ := open.inventory.SourceOf(ctx, "letta"); src.Ref != "" {
|
||||
t.Errorf("a module first built at a commit follows %q, want the default branch", src.Ref)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOlderBuildHeardLaterDoesNotReplaceTheNewer(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
older := time.Date(2026, 10, 3, 21, 33, 45, 0, time.UTC)
|
||||
newer := time.Date(2026, 10, 3, 21, 51, 57, 0, time.UTC)
|
||||
result := func(asked time.Time, image string) link.BuildResult {
|
||||
manifest, _ := json.Marshal(map[string]any{"module": "postgres", "version": image})
|
||||
return link.BuildResult{ID: link.NewBuildID(asked), Repository: "http://forge.internal:20000/novox/mesh-catalog.git",
|
||||
Path: "modules/postgres", Ref: "main", On: "anchor", Commit: "efff5415", Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, result(newer, "4bcd5f73"))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, err := takeIn(ctx, open.inventory, result(older, "0ab07fa9"))
|
||||
if !errors.Is(err, inventory.ErrSuperseded) {
|
||||
t.Fatalf("the older request's outcome was taken in as current: %v", err)
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := shelf["postgres"].Version; got != "4bcd5f73" {
|
||||
t.Errorf("postgres is %q; want the newer request's 4bcd5f73", got)
|
||||
}
|
||||
if builds, _ := open.inventory.Builds(ctx, "postgres", 5); len(builds) != 2 {
|
||||
t.Errorf("the late build was not recorded: %v", builds)
|
||||
}
|
||||
}
|
||||
@@ -1,566 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// Where a build may register a module from (novox/hq ADR 0266).
|
||||
//
|
||||
// A build's outcome registers its module, and a registered module is what the next push sends. Before this,
|
||||
// an outcome registered whatever its manifest named, from whichever repository it was built from: an agent
|
||||
// that made a repository of its own, committed `modules/sudo/module.json` granting itself a rule without a
|
||||
// password, and asked the `build` verb for it had its repository registered as the module `sudo` — whose next
|
||||
// push made the agent root on every node. A fork of the node-engine did the same everywhere. The trunk rule
|
||||
// (ADR 0238) did not stop it: the trunk it checked was the trunk of the repository built, which was the
|
||||
// agent's own.
|
||||
//
|
||||
// So **an outcome registers a module only from the repository the catalogue already builds that module
|
||||
// from**; and a module new to the catalogue only from a repository the catalogue already builds another
|
||||
// module from — whose trunk takes a reviewed, approved merge, which is how a merge adds a module (novox/hq
|
||||
// issue 300). Anything else — a module moved to another repository, a module from a repository the
|
||||
// catalogue has never built — is the operator's, at the controller's terminal: allowed only when the build
|
||||
// request was kept as asked there. Judged at the take-in, which every outcome reaches whoever hears it and
|
||||
// whichever verb asked it (`build`, `rebuild`, `replay --register`, `assign` with build), and before the ask
|
||||
// for a call through a verb, so an agent cannot have a build node run a repository the catalogue does not
|
||||
// build from at all.
|
||||
|
||||
// errNotItsSource is an outcome refused for where it was built from.
|
||||
var errNotItsSource = errors.New("not built from the repository the catalogue builds it from")
|
||||
|
||||
// servedVar marks every process the serving controller starts — each verb's command, each child — and the serving
|
||||
// process itself, so none of them can read as the operator at the terminal (novox/hq ADR 0266). Set by serve
|
||||
// before it answers anything, inherited by every child through os.Environ.
|
||||
const servedVar = "MESH_SERVED_BY_THE_CONTROLLER"
|
||||
|
||||
// startedAtTheTerminal says this process was started at the controller's terminal: not the serving controller,
|
||||
// not anything it started, not a verb's command, not a seat call's. The serving controller marks its own
|
||||
// environment (servedVar), so a build asked in it, or by any process it starts, never reads as the terminal's;
|
||||
// runVerb also names the verb and the caller.
|
||||
//
|
||||
// **And a line mesh-cli asked as the controller's terminal** (novox/hq ADR 0272 §4): the serving controller runs it
|
||||
// without the served mark and without a verb, names its caller, and marks it with cliTerminalVar — a mark only the
|
||||
// mesh-cli path sets and every other command line the controller runs is stripped of (commandEnvironment).
|
||||
func startedAtTheTerminal() bool {
|
||||
if os.Getenv(servedVar) != "" || os.Getenv(verbVar) != "" {
|
||||
return false
|
||||
}
|
||||
return os.Getenv(link.CallerVar) == "" || os.Getenv(cliTerminalVar) != ""
|
||||
}
|
||||
|
||||
// cliTerminalVar marks a command line the serving controller runs as the controller's terminal for mesh-cli.
|
||||
const cliTerminalVar = "MESH_CLI_TERMINAL"
|
||||
|
||||
// markServed marks this process, and so everything it starts, as the serving controller's.
|
||||
func markServed() {
|
||||
if err := os.Setenv(servedVar, "1"); err != nil {
|
||||
panic("the serving controller could not mark its environment: " + err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// sourceForms compares sources however each is spelled: a path on a seat's holder (ADR 0111) or a URL — a
|
||||
// build asked of a seat's path is registered with the URL composed from it when its outcome does not echo
|
||||
// the seat. A seat's path is composed into its URL where the seat's holder is known, so both spellings of
|
||||
// one repository are one; where it is not, a seat's path matches only the same seat's same path.
|
||||
type sourceForms struct {
|
||||
bases map[string]string // the seat's clone base, `scheme://host:port`, by seat; "" where it is not known
|
||||
base func(seat string) string
|
||||
}
|
||||
|
||||
// newSourceForms reads the seats' bases from the mesh once, when first needed.
|
||||
func newSourceForms(ctx context.Context, inv *inventory.Inventory) *sourceForms {
|
||||
f := &sourceForms{bases: map[string]string{}}
|
||||
var world *catalogue.World
|
||||
f.base = func(seat string) string {
|
||||
if b, known := f.bases[seat]; known {
|
||||
return b
|
||||
}
|
||||
if world == nil {
|
||||
w := catalogue.World{}
|
||||
if shelf, err := inv.Catalogue(ctx); err == nil {
|
||||
if read, err := theRestOfTheMesh(ctx, inv, shelf, ""); err == nil {
|
||||
w = read
|
||||
}
|
||||
}
|
||||
world = &w
|
||||
}
|
||||
b, err := seatBase(*world, seat)
|
||||
if err != nil {
|
||||
b = ""
|
||||
}
|
||||
f.bases[seat] = b
|
||||
return b
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// canonical is one spelling of a repository: lower case, no `.git`, no trailing slash, and a seat's path as
|
||||
// the URL its holder serves it at where that is known.
|
||||
func (f *sourceForms) canonical(repository, seat string) string {
|
||||
trim := func(s string) string {
|
||||
s = strings.TrimSpace(strings.ToLower(s))
|
||||
s = strings.TrimRight(s, "/")
|
||||
return strings.TrimRight(strings.TrimSuffix(s, ".git"), "/")
|
||||
}
|
||||
if seat == "" {
|
||||
return trim(repository)
|
||||
}
|
||||
if b := f.base(seat); b != "" {
|
||||
return trim(b + "/" + strings.Trim(repository, "/"))
|
||||
}
|
||||
return "seat:" + seat + ":" + trim(strings.Trim(repository, "/"))
|
||||
}
|
||||
|
||||
// same says two sources are one repository.
|
||||
func (f *sourceForms) same(aRepository, aSeat, bRepository, bSeat string) bool {
|
||||
if aRepository == "" || bRepository == "" {
|
||||
return false
|
||||
}
|
||||
return f.canonical(aRepository, aSeat) == f.canonical(bRepository, bSeat)
|
||||
}
|
||||
|
||||
// buildsFrom says the catalogue builds some module from this repository.
|
||||
func (f *sourceForms) buildsFrom(entries []inventory.Entry, repository, seat string) bool {
|
||||
for _, e := range entries {
|
||||
if e.Provided || e.Source.Repository == "" {
|
||||
continue
|
||||
}
|
||||
if f.same(e.Source.Repository, e.Source.Seat, repository, seat) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// forgeFacts is what the mesh's forge says of a repository a module is registered from (novox/hq ADR 0266).
|
||||
type forgeFacts struct {
|
||||
// ID is the forge's own id of the repository: what tells it from one deleted and made again by its name.
|
||||
ID int64
|
||||
// Guarded is whether the branch is protected as a module's trunk must be: no direct push, at least one
|
||||
// required status, and no administrator merging past one. Why says what is missing when it is not.
|
||||
Guarded bool
|
||||
Why string
|
||||
}
|
||||
|
||||
// askTheForge asks the forge, through its module's tools on the bus, for a repository's id and its branch's
|
||||
// protection, waiting out a forge not served for a moment (novox/hq issue 486). A variable so a test needs no
|
||||
// forge.
|
||||
var askTheForge = func(ctx context.Context, owner, repo, branch string) (forgeFacts, error) {
|
||||
return forgeWaitedOut(ctx, forgeWait, owner+"/"+repo, func(ctx context.Context) (forgeFacts, error) {
|
||||
return askTheForgeOnce(ctx, owner, repo, branch)
|
||||
})
|
||||
}
|
||||
|
||||
// waitOut bounds how long a question is asked again while its answer is "not now": a pause of first, doubling
|
||||
// to most, for at most bound in all. now and sleep are the clock, so a test waits in no real time.
|
||||
type waitOut struct {
|
||||
bound, first, most time.Duration
|
||||
now func() time.Time
|
||||
sleep func(context.Context, time.Duration) error
|
||||
}
|
||||
|
||||
// forgeWait is the registration's wait for the forge (novox/hq issue 486). On 2026-10-11 a build's registration
|
||||
// asked the forge while node-tools on the control-node re-served after a rollout of the tool runner, heard
|
||||
// "nothing serves gitea.gitea_api", and failed its plan; the forge answered again seconds later. A re-serve takes
|
||||
// about 10 s, so 60 s waits one out with room to spare. It is a cap on the whole wait, an ask in flight included:
|
||||
// the registration holds the controller's one receive loop while it waits, and a loop held two minutes reads as
|
||||
// deaf. The pauses are those the build machine waits the registry with (novox/hq issue 457).
|
||||
var forgeWait = waitOut{bound: 60 * time.Second, first: time.Second, most: 10 * time.Second,
|
||||
now: time.Now, sleep: sleepOrDone}
|
||||
|
||||
// sleepOrDone pauses for d, or less when ctx ends first.
|
||||
func sleepOrDone(ctx context.Context, d time.Duration) error {
|
||||
t := time.NewTimer(d)
|
||||
defer t.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-t.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// forgeNotNow is an answer that says the forge's tool is not served now — the tool runner re-serving — not that
|
||||
// the forge refused. Only that is asked again: a refusal, an unreadable answer or a repository with no id is the
|
||||
// forge's word, and a tool that did not answer in time has already held the receive loop its whole wait.
|
||||
func forgeNotNow(err error) bool { return errors.Is(err, link.ErrNothingServes) }
|
||||
|
||||
// forgeWaitedOut asks the forge about repository, asking again while it answers "not served now", all of it
|
||||
// within w's bound, an ask in flight included (novox/hq issue 486). Past the bound it fails, saying it waited,
|
||||
// for how long, and what the forge last said; when the caller's own context ends, it says that instead.
|
||||
func forgeWaitedOut(ctx context.Context, w waitOut, repository string,
|
||||
ask func(context.Context) (forgeFacts, error)) (forgeFacts, error) {
|
||||
capped, cancel := context.WithTimeout(ctx, w.bound)
|
||||
defer cancel()
|
||||
began := w.now()
|
||||
pause := w.first
|
||||
for asked := 1; ; asked++ {
|
||||
facts, err := ask(capped)
|
||||
if err == nil {
|
||||
if asked > 1 {
|
||||
fmt.Printf("the forge answered about %s again, after %s (novox/hq issue 486)\n", repository,
|
||||
w.now().Sub(began).Round(time.Second))
|
||||
}
|
||||
return facts, nil
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return forgeFacts{}, err // the caller's own context ended: no wait of ours to speak of
|
||||
}
|
||||
gaveUp := func() (forgeFacts, error) {
|
||||
return forgeFacts{}, fmt.Errorf("waited %s for the forge to answer about %s, asking %d times, and it "+
|
||||
"did not (novox/hq issue 486): %w", w.now().Sub(began).Round(time.Second), repository, asked, err)
|
||||
}
|
||||
if capped.Err() != nil {
|
||||
return gaveUp() // the bound ended an ask in flight
|
||||
}
|
||||
if !forgeNotNow(err) {
|
||||
return forgeFacts{}, err
|
||||
}
|
||||
if w.now().Sub(began)+pause > w.bound {
|
||||
return gaveUp()
|
||||
}
|
||||
fmt.Printf("the forge cannot be asked about %s now (%v); asking again in %s, for at most %s in all "+
|
||||
"(novox/hq issue 486)\n", repository, err, pause, w.bound)
|
||||
if serr := w.sleep(capped, pause); serr != nil {
|
||||
if ctx.Err() != nil {
|
||||
return forgeFacts{}, fmt.Errorf("stopped waiting for the forge to answer about %s: %w", repository,
|
||||
ctx.Err())
|
||||
}
|
||||
return gaveUp()
|
||||
}
|
||||
pause = min(2*pause, w.most)
|
||||
}
|
||||
}
|
||||
|
||||
// askTheForgeOnce asks the forge once, through its module's tools on the bus.
|
||||
func askTheForgeOnce(ctx context.Context, owner, repo, branch string) (forgeFacts, error) {
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
return forgeFacts{}, err
|
||||
}
|
||||
defer js.Close()
|
||||
bus := link.OverNATS{Conn: js.Conn()}
|
||||
ask := func(tool string, args map[string]any, into any) error {
|
||||
raw, _ := json.Marshal(args)
|
||||
answer, err := link.Ask(ctx, bus, "gitea", tool, raw, 30*time.Second)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return fmt.Errorf("gitea.%s: %s", tool, answer.Error)
|
||||
}
|
||||
return json.Unmarshal(answer.Result, into)
|
||||
}
|
||||
var found struct {
|
||||
Result struct {
|
||||
ID int64 `json:"id"`
|
||||
FullName string `json:"full_name"`
|
||||
} `json:"result"`
|
||||
}
|
||||
if err := ask("gitea_api", map[string]any{"path": "/repos/" + owner + "/" + repo}, &found); err != nil {
|
||||
return forgeFacts{}, err
|
||||
}
|
||||
if found.Result.ID == 0 {
|
||||
return forgeFacts{}, fmt.Errorf("the forge named no id for %s/%s", owner, repo)
|
||||
}
|
||||
var rules struct {
|
||||
Rules []protectionRule `json:"rules"`
|
||||
}
|
||||
if err := ask("gitea_branch_protection_get", map[string]any{"owner": owner, "repo": repo}, &rules); err != nil {
|
||||
return forgeFacts{}, err
|
||||
}
|
||||
facts := judgedRule(rules.Rules, branch)
|
||||
facts.ID = found.Result.ID
|
||||
return facts, nil
|
||||
}
|
||||
|
||||
// protectionRule is a branch protection rule as the forge's tool summarises it.
|
||||
type protectionRule struct {
|
||||
Rule string `json:"rule"`
|
||||
Push bool `json:"push"`
|
||||
RequiredStatuses []string `json:"required_statuses"`
|
||||
AdminMayOverride bool `json:"admin_may_override"`
|
||||
}
|
||||
|
||||
// judgedRule judges the one rule the forge applies to a branch, as the forge picks it: the rule named for the
|
||||
// branch, else the first glob rule, in the forge's order, that covers it (gitea's first matching rule). Never a
|
||||
// later rule that happens to be stronger: the forge does not read it.
|
||||
func judgedRule(rules []protectionRule, branch string) forgeFacts {
|
||||
var applied *protectionRule
|
||||
for i := range rules {
|
||||
if rules[i].Rule == branch {
|
||||
applied = &rules[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if applied == nil {
|
||||
for i := range rules {
|
||||
if ruleCovers(rules[i].Rule, branch) {
|
||||
applied = &rules[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
switch r := applied; {
|
||||
case r == nil:
|
||||
return forgeFacts{Why: fmt.Sprintf("no protection rule covers %s", branch)}
|
||||
case r.Push:
|
||||
return forgeFacts{Why: fmt.Sprintf("the rule %s lets a person push to %s directly", r.Rule, branch)}
|
||||
case len(r.RequiredStatuses) == 0:
|
||||
return forgeFacts{Why: fmt.Sprintf("the rule %s requires no status before a merge into %s", r.Rule, branch)}
|
||||
case r.AdminMayOverride:
|
||||
return forgeFacts{Why: fmt.Sprintf("the rule %s lets an administrator merge into %s past a status", r.Rule, branch)}
|
||||
}
|
||||
return forgeFacts{Guarded: true}
|
||||
}
|
||||
|
||||
// ruleCovers says a protection rule's name — a branch, or a glob of them — covers a branch, as the forge reads it.
|
||||
func ruleCovers(rule, branch string) bool {
|
||||
if rule == branch {
|
||||
return true
|
||||
}
|
||||
if !strings.ContainsAny(rule, "*?[") {
|
||||
return false
|
||||
}
|
||||
var re strings.Builder
|
||||
re.WriteString("^")
|
||||
for i := 0; i < len(rule); i++ {
|
||||
switch c := rule[i]; {
|
||||
case c == '*' && i+1 < len(rule) && rule[i+1] == '*':
|
||||
re.WriteString(".*")
|
||||
i++
|
||||
case c == '*':
|
||||
re.WriteString("[^/]*")
|
||||
case c == '?':
|
||||
re.WriteString("[^/]")
|
||||
default:
|
||||
re.WriteString(regexp.QuoteMeta(string(c)))
|
||||
}
|
||||
}
|
||||
re.WriteString("$")
|
||||
ok, _ := regexp.MatchString(re.String(), branch)
|
||||
return ok
|
||||
}
|
||||
|
||||
// onTheForge is a source's owner and name on the mesh's own forge (the git seat's holder), and whether it is
|
||||
// there at all.
|
||||
func (f *sourceForms) onTheForge(repository, seat string) (owner, name string, ok bool) {
|
||||
var rest string
|
||||
switch {
|
||||
case seat == gitSeat:
|
||||
rest = strings.Trim(repository, "/")
|
||||
case seat == "":
|
||||
base := f.base(gitSeat)
|
||||
if base == "" {
|
||||
return "", "", false
|
||||
}
|
||||
url, prefix := f.canonical(repository, ""), f.canonical(base, "")+"/"
|
||||
if !strings.HasPrefix(url, prefix) {
|
||||
return "", "", false
|
||||
}
|
||||
// The case the forge spells it with: the URL as given, past the base.
|
||||
rest = strings.TrimSuffix(strings.Trim(repository[len(prefix):], "/"), ".git")
|
||||
default:
|
||||
return "", "", false
|
||||
}
|
||||
parts := strings.Split(rest, "/")
|
||||
if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
|
||||
return "", "", false
|
||||
}
|
||||
return parts[0], parts[1], true
|
||||
}
|
||||
|
||||
// mayRegisterFrom says whether a build's outcome may register module from the source it was built from, and the
|
||||
// forge's id of that repository to record (novox/hq ADR 0266). Through any verb, only:
|
||||
//
|
||||
// - from the module's registered repository — the same repository by the forge's own id, not only its name; or,
|
||||
// for a module new to the catalogue, from a repository the catalogue builds another module from;
|
||||
// - and from a repository on the mesh's forge whose trunk is protected as a trunk must be: no direct push, at
|
||||
// least one required status, no administrator merging past one.
|
||||
//
|
||||
// Anything else only when the build request was kept as asked at the controller's terminal, for this very
|
||||
// repository and path. path is the module's directory as built; branch the trunk it is registered from.
|
||||
func mayRegisterFrom(ctx context.Context, inv *inventory.Inventory, module string, built inventory.Source,
|
||||
buildID, path, branch string) (int64, error) {
|
||||
forms := newSourceForms(ctx, inv)
|
||||
was, err := inv.SourceOf(ctx, module)
|
||||
isNew := errors.Is(err, inventory.ErrNoSuchModule)
|
||||
if err != nil && !isNew {
|
||||
return 0, err
|
||||
}
|
||||
terminal, err := askedHereFor(ctx, inv, forms, buildID, built, path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
refuse := func(why string) (int64, error) {
|
||||
return 0, fmt.Errorf("%w: %s. A module is registered from such a source only by a build asked at the "+
|
||||
"controller's terminal, never through a verb: a registered module is what the next push sends, and whoever "+
|
||||
"may call a verb includes agents (novox/hq ADR 0266)", errNotItsSource, why)
|
||||
}
|
||||
|
||||
var why string
|
||||
var alongside []inventory.Entry // the modules a new one's repository already builds
|
||||
switch {
|
||||
case !isNew && forms.same(was.Repository, was.Seat, built.Repository, built.Seat):
|
||||
case !isNew:
|
||||
registered := was.Repository
|
||||
if registered == "" {
|
||||
registered = "no repository (it was handed over by hand)"
|
||||
}
|
||||
why = fmt.Sprintf("%s is built from %s, and this build is of %s", module, sourceWords(registered, was.Seat),
|
||||
sourceWords(built.Repository, built.Seat))
|
||||
default:
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if !e.Provided && e.Source.Repository != "" &&
|
||||
forms.same(e.Source.Repository, e.Source.Seat, built.Repository, built.Seat) {
|
||||
alongside = append(alongside, e)
|
||||
}
|
||||
}
|
||||
if len(alongside) == 0 {
|
||||
why = fmt.Sprintf("%s is new to the catalogue, and %s is no repository the catalogue builds a module from",
|
||||
module, sourceWords(built.Repository, built.Seat))
|
||||
}
|
||||
}
|
||||
// **Or the move the operator approved** (novox/hq ADR 0420 §7, issue 1025): a known module built from another
|
||||
// repository by a merge, whose pull request mesh-delivery merged on the operator's Approve of that head.
|
||||
// A verb alone, or a merge with no such warrant, is refused as before.
|
||||
movedWhy := ""
|
||||
if why != "" && !terminal && !isNew {
|
||||
accepted, err := approvedMove(ctx, inv, forms, buildID, built)
|
||||
switch {
|
||||
case err == nil:
|
||||
movedWhy, why = accepted, ""
|
||||
case !errors.Is(err, errNotAMerge):
|
||||
fmt.Printf("%s: %s is built from another repository, and no approved move covers this build: %v\n",
|
||||
buildID, module, err)
|
||||
}
|
||||
}
|
||||
if why != "" && !terminal {
|
||||
return refuse(why)
|
||||
}
|
||||
|
||||
owner, name, onForge := forms.onTheForge(built.Repository, built.Seat)
|
||||
if !onForge {
|
||||
if terminal {
|
||||
fmt.Printf("%s: %s is not on the mesh's forge — registered, as asked at the controller's terminal\n",
|
||||
buildID, sourceWords(built.Repository, built.Seat))
|
||||
return 0, nil
|
||||
}
|
||||
return refuse(fmt.Sprintf("%s is not on the mesh's forge, so whether its trunk is protected cannot be read",
|
||||
sourceWords(built.Repository, built.Seat)))
|
||||
}
|
||||
facts, err := askTheForge(ctx, owner, name, branch)
|
||||
if err != nil {
|
||||
if terminal {
|
||||
fmt.Printf("%s: the forge could not be asked about %s/%s (%v) — registered, as asked at the controller's "+
|
||||
"terminal\n", buildID, owner, name, err)
|
||||
return 0, nil
|
||||
}
|
||||
return refuse(fmt.Sprintf("the forge could not say whether %s/%s's %s is protected: %v", owner, name, branch, err))
|
||||
}
|
||||
if terminal {
|
||||
if why != "" || !facts.Guarded {
|
||||
fmt.Printf("%s: %s — registered, as asked at the controller's terminal\n", buildID,
|
||||
strings.Trim(why+"; "+facts.Why, "; "))
|
||||
}
|
||||
return facts.ID, nil
|
||||
}
|
||||
if !facts.Guarded {
|
||||
return refuse(fmt.Sprintf("%s/%s's %s is not protected as a module's trunk must be: %s", owner, name, branch,
|
||||
facts.Why))
|
||||
}
|
||||
// The same repository by the forge's id, not only its name: one deleted and made again is another.
|
||||
recorded := map[string]int64{}
|
||||
// A moved module is registered from the repository the operator approved, not the one it was registered from.
|
||||
if !isNew && movedWhy == "" {
|
||||
id, err := inv.SourceIdentity(ctx, module)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
recorded[module] = id
|
||||
}
|
||||
for _, e := range alongside {
|
||||
id, err := inv.SourceIdentity(ctx, e.Manifest.Module)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
recorded[e.Manifest.Module] = id
|
||||
}
|
||||
for m, id := range recorded {
|
||||
if id != 0 && id != facts.ID {
|
||||
return refuse(fmt.Sprintf("%s/%s is not the repository %s was registered from: the forge knows it as "+
|
||||
"repository %d, and %s was registered from repository %d — one of that name deleted and made again",
|
||||
owner, name, m, facts.ID, m, id))
|
||||
}
|
||||
}
|
||||
if movedWhy != "" {
|
||||
fmt.Printf("%s: %s moved from %s to %s/%s — registered at the operator's Approve: %s\n", buildID, module,
|
||||
sourceWords(was.Repository, was.Seat), owner, name, movedWhy)
|
||||
}
|
||||
return facts.ID, nil
|
||||
}
|
||||
|
||||
// askedHereFor says the build was asked at the controller's terminal, for this repository and this path: a kept
|
||||
// request marked so, whose source is the outcome's (novox/hq ADR 0266).
|
||||
func askedHereFor(ctx context.Context, inv *inventory.Inventory, forms *sourceForms, buildID string,
|
||||
built inventory.Source, path string) (bool, error) {
|
||||
r, found, err := inv.BuildRequestByID(ctx, buildID)
|
||||
if err != nil || !found || !r.AtTerminal {
|
||||
return false, err
|
||||
}
|
||||
if !forms.same(r.Repository, r.Seat, built.Repository, built.Seat) ||
|
||||
strings.Trim(r.Path, "/") != strings.Trim(path, "/") {
|
||||
fmt.Printf("%s was asked at the terminal of %s at %q, and its outcome is of %s at %q: not the terminal's\n",
|
||||
buildID, sourceWords(r.Repository, r.Seat), r.Path, sourceWords(built.Repository, built.Seat), path)
|
||||
return false, nil
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// sourceWords is a source as a person reads it.
|
||||
func sourceWords(repository, seat string) string {
|
||||
if seat == "" {
|
||||
return repository
|
||||
}
|
||||
return buildSource{Repository: repository, Seat: seat}.String()
|
||||
}
|
||||
|
||||
// verbMayAsk refuses, for a call through a verb, a build of a repository the catalogue builds no module from
|
||||
// (novox/hq ADR 0266): the build node would run what an agent wrote, and its outcome could never be
|
||||
// registered anyway. url is the repository as it is cloned. At the terminal anything may be asked.
|
||||
func verbMayAsk(ctx context.Context, source buildSource, url string) error {
|
||||
if startedAtTheTerminal() {
|
||||
return nil
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
entries, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
forms := newSourceForms(ctx, open.inventory)
|
||||
if forms.buildsFrom(entries, source.Repository, source.Seat) || forms.buildsFrom(entries, url, "") {
|
||||
return nil
|
||||
}
|
||||
return terminalRefusal("%s is no repository the catalogue builds a module from, and a build of any other is "+
|
||||
"asked at the controller's terminal only, never through a verb: a build node runs what the repository "+
|
||||
"says, and its outcome would register a module the next push sends — whoever may call a verb includes "+
|
||||
"agents (novox/hq ADR 0266). Nothing was asked", source)
|
||||
}
|
||||
@@ -1,353 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func theCatalogue(t *testing.T) *stores {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
for _, b := range []link.BuildResult{
|
||||
onTrunk("build-sudo", "novox/mesh-catalog", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "1"}),
|
||||
onTrunk("build-host", "novox/mesh-host", "git", "", map[string]any{"module": "mesh-host", "version": "1"}),
|
||||
} {
|
||||
keptAsked(t, open.inventory, b.ID, b.Source.Repository, b.Path, true)
|
||||
if _, _, err := takeIn(ctx, open.inventory, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return open
|
||||
}
|
||||
|
||||
func TestABuildFromAnAgentsRepositoryIsNotRegisteredAsAModuleTheMeshHolds(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
for _, c := range []struct {
|
||||
name, repository, path, module string
|
||||
}{
|
||||
{"its own repository naming sudo", "agent/sudo", "modules/sudo", "sudo"},
|
||||
{"a fork of the catalogue", "agent/mesh-catalog", "modules/sudo", "sudo"},
|
||||
{"a fork of the node-engine", "agent/mesh-host", "", "mesh-host"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
id := "build-" + strings.ReplaceAll(c.repository, "/", "-")
|
||||
keptAsked(t, open.inventory, id, c.repository, c.path, false) // through the build verb
|
||||
evil := onTrunk(id, c.repository, "git", c.path, map[string]any{"module": c.module, "version": "evil"})
|
||||
_, _, err := takeIn(ctx, open.inventory, evil)
|
||||
if !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("a build of %s was taken in as %s: %v", c.repository, c.module, err)
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := shelf[c.module].Version; got != "1" {
|
||||
t.Fatalf("%s is now %q, from %s", c.module, got, c.repository)
|
||||
}
|
||||
if _, found, _ := open.inventory.BuildByID(ctx, id); !found {
|
||||
t.Errorf("the refused build %s is not recorded", id)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestANewModuleFromARepositoryTheCatalogueDoesNotBuildFromIsNotRegistered(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
keptAsked(t, open.inventory, "build-new", "agent/tools", "", false)
|
||||
_, _, err := takeIn(ctx, open.inventory, onTrunk("build-new", "agent/tools", "git", "",
|
||||
map[string]any{"module": "agent-tools", "version": "1"}))
|
||||
if !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("a new module from an agent's repository was taken in: %v", err)
|
||||
}
|
||||
// And one asked of nobody here — an outcome on the bus no request was kept for — the same.
|
||||
_, _, err = takeIn(ctx, open.inventory, onTrunk("build-unasked", "agent/tools", "git", "",
|
||||
map[string]any{"module": "agent-tools", "version": "1"}))
|
||||
if !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("an outcome nobody asked for was taken in: %v", err)
|
||||
}
|
||||
if shelf, _ := open.inventory.Catalogue(ctx); shelf["agent-tools"].Module != "" {
|
||||
t.Fatal("the refused module is in the catalogue")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAtTheTerminalAnotherRepositoryIsRegistered(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
keptAsked(t, open.inventory, "build-moved", "novox/sudo", "", true)
|
||||
if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-moved", "novox/sudo", "git", "",
|
||||
map[string]any{"module": "sudo", "version": "2"})); err != nil {
|
||||
t.Fatalf("a move the operator asked for at the terminal was refused: %v", err)
|
||||
}
|
||||
if src, _ := open.inventory.SourceOf(ctx, "sudo"); src.Repository != "novox/sudo" {
|
||||
t.Fatalf("sudo is built from %q", src.Repository)
|
||||
}
|
||||
if err := open.inventory.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-external",
|
||||
Repository: "http://forge.internal:20000/someone/app.git", For: "build", AtTerminal: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-external", "someone/app", "", "",
|
||||
map[string]any{"module": "app", "version": "1"})); err != nil {
|
||||
t.Fatalf("a new module the operator asked for at the terminal was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestADeliveryFromTheRegisteredRepositoryIsRegistered(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
rebuilt := onTrunk("build-plan", "novox/mesh-catalog", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "2"})
|
||||
if err := open.inventory.RecordBuildRequest(ctx, inventory.BuildRequest{ID: rebuilt.ID,
|
||||
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/sudo", For: "plan"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, rebuilt); err != nil {
|
||||
t.Fatalf("a plan's build of the module's own repository was refused: %v", err)
|
||||
}
|
||||
added := onTrunk("build-merge", "novox/mesh-catalog", "git", "modules/zram", map[string]any{"module": "zram", "version": "1"})
|
||||
if _, _, err := takeIn(ctx, open.inventory, added); err != nil {
|
||||
t.Fatalf("a module a merge added to the catalogue repository was refused: %v", err)
|
||||
}
|
||||
shelf, _ := open.inventory.Catalogue(ctx)
|
||||
if shelf["sudo"].Version != "2" || shelf["zram"].Module == "" {
|
||||
t.Fatalf("not registered: sudo %q, zram %q", shelf["sudo"].Version, shelf["zram"].Module)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAVerbAsksNoBuildOfARepositoryTheCatalogueDoesNotBuildFrom(t *testing.T) {
|
||||
theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
t.Setenv(verbVar, "build")
|
||||
t.Setenv(link.CallerVar, "node-tools.anchor, through the mesh-controller seat")
|
||||
err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, "http://forge.internal:20000/agent/sudo.git")
|
||||
var policy *heldAtTheTerminal
|
||||
if !errors.As(err, &policy) {
|
||||
t.Fatalf("a verb's build of an agent's repository was asked: %v", err)
|
||||
}
|
||||
if err := verbMayAsk(ctx, buildSource{Repository: "novox/mesh-catalog", Seat: "git"},
|
||||
"http://forge.internal:20000/novox/mesh-catalog.git"); err != nil {
|
||||
t.Fatalf("a verb's build of the catalogue repository was refused: %v", err)
|
||||
}
|
||||
t.Setenv(verbVar, "")
|
||||
t.Setenv(link.CallerVar, "")
|
||||
if err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, ""); err != nil {
|
||||
t.Fatalf("the terminal was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARollbackNeverPutsBackABuildFromAnotherRepository(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
fork := onTrunk("build-1791500000000000000", "agent/mesh-catalog", "git", "modules/sudo",
|
||||
map[string]any{"module": "sudo", "version": "evil"})
|
||||
fork.Commit = "c0ffee0123456789" // the commit sudo was registered at
|
||||
keptAsked(t, inv, fork.ID, "agent/mesh-catalog", "modules/sudo", false)
|
||||
if _, _, err := takeIn(ctx, inv, fork); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("the fork's build was taken in: %v", err)
|
||||
}
|
||||
// Asked after the registered build, whenever the test runs: an id naming a fixed moment read as older
|
||||
// than the registered build once the clock passed it (2026-10-10 02:40 UTC), and the test failed on main.
|
||||
failed := onTrunk(fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano()), "novox/mesh-catalog", "git", "modules/sudo",
|
||||
map[string]any{"module": "sudo", "version": "2"})
|
||||
failed.Commit = "badbadbad0123456"
|
||||
if _, _, err := takeIn(ctx, inv, failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, _, err := inv.BuildByID(ctx, failed.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
previous, found, err := inv.PreviousBuild(ctx, "sudo", "c0ffee0123456789", record)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if found && previous.ID == fork.ID {
|
||||
t.Fatalf("a rollback would put back the fork's build %s", previous.ID)
|
||||
}
|
||||
if !found || previous.ID != "build-sudo" {
|
||||
t.Fatalf("a rollback puts back %q (found %v), want the registered build-sudo", previous.ID, found)
|
||||
}
|
||||
}
|
||||
|
||||
func TestATrunkTheForgeDoesNotProtectRegistersNothing(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
theForge.Store("novox/unguarded", forgeFacts{ID: 7, Why: "the rule main lets a person push to main directly"})
|
||||
t.Cleanup(func() { theForge.Delete("novox/unguarded") })
|
||||
first := onTrunk("build-unguarded-1", "novox/unguarded", "git", "", map[string]any{"module": "unguarded", "version": "1"})
|
||||
keptAsked(t, open.inventory, first.ID, "novox/unguarded", "", true)
|
||||
if _, _, err := takeIn(ctx, open.inventory, first); err != nil {
|
||||
t.Fatalf("at the terminal: %v", err)
|
||||
}
|
||||
again := onTrunk("build-unguarded-2", "novox/unguarded", "git", "", map[string]any{"module": "unguarded", "version": "2"})
|
||||
if _, _, err := takeIn(ctx, open.inventory, again); !errors.Is(err, errNotItsSource) ||
|
||||
!strings.Contains(err.Error(), "push to main directly") {
|
||||
t.Fatalf("a rebuild from an unprotected trunk was taken in: %v", err)
|
||||
}
|
||||
theForge.Store("novox/unguarded", errors.New("nothing serves gitea.gitea_api"))
|
||||
if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-unguarded-3", "novox/unguarded", "git", "",
|
||||
map[string]any{"module": "unguarded", "version": "3"})); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("a forge that could not say was read as a protected trunk: %v", err)
|
||||
}
|
||||
if shelf, _ := open.inventory.Catalogue(ctx); shelf["unguarded"].Version != "1" {
|
||||
t.Fatalf("unguarded is %q", shelf["unguarded"].Version)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARepositoryMadeAgainUnderItsNameIsNotTheModulesSource(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
theForge.Store("novox/remade", forgeFacts{ID: 100, Guarded: true})
|
||||
t.Cleanup(func() { theForge.Delete("novox/remade") })
|
||||
first := onTrunk("build-remade-1", "novox/remade", "git", "", map[string]any{"module": "remade", "version": "1"})
|
||||
keptAsked(t, open.inventory, first.ID, "novox/remade", "", true)
|
||||
if _, _, err := takeIn(ctx, open.inventory, first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if id, _ := open.inventory.SourceIdentity(ctx, "remade"); id != 100 {
|
||||
t.Fatalf("the forge's id was not recorded: %d", id)
|
||||
}
|
||||
theForge.Store("novox/remade", forgeFacts{ID: 101, Guarded: true}) // deleted, and made again by an agent
|
||||
if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-remade-2", "novox/remade", "git", "",
|
||||
map[string]any{"module": "remade", "version": "evil"})); !errors.Is(err, errNotItsSource) ||
|
||||
!strings.Contains(err.Error(), "made again") {
|
||||
t.Fatalf("a repository made again under the name was taken in: %v", err)
|
||||
}
|
||||
// And a new module from it, beside the one registered from the first, the same.
|
||||
if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-remade-3", "novox/remade", "git", "modules/other",
|
||||
map[string]any{"module": "other", "version": "1"})); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("a new module from a repository made again was taken in: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestATerminalRequestCoversOnlyWhatItAsked(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-asked", Repository: "novox/app",
|
||||
Seat: "git", Path: "modules/app", For: "build", AtTerminal: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
other := onTrunk("build-asked", "agent/sudo", "git", "modules/app", map[string]any{"module": "sudo", "version": "evil"})
|
||||
if _, _, err := takeIn(ctx, open.inventory, other); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("an outcome of another repository under a terminal request's id was taken in: %v", err)
|
||||
}
|
||||
elsewhere := onTrunk("build-asked", "novox/app", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "evil"})
|
||||
if _, _, err := takeIn(ctx, open.inventory, elsewhere); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("an outcome of another path under a terminal request's id was taken in: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheServingControllerIsNeverTheTerminal(t *testing.T) {
|
||||
t.Setenv(verbVar, "")
|
||||
t.Setenv(link.CallerVar, "")
|
||||
t.Setenv(servedVar, "")
|
||||
if !startedAtTheTerminal() {
|
||||
t.Fatal("a process started by hand is not the terminal")
|
||||
}
|
||||
markServed()
|
||||
if startedAtTheTerminal() {
|
||||
t.Fatal("the serving controller reads as the terminal")
|
||||
}
|
||||
child := exec.Command(os.Args[0], "-test.run=^$")
|
||||
child.Env = os.Environ()
|
||||
if !slices.Contains(child.Env, servedVar+"=1") {
|
||||
t.Fatal("what the serving controller starts does not carry its mark")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnknownIdentityKeepsTheRecordedOne(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.SetSourceIdentity(ctx, "sudo", 100); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetSourceIdentity(ctx, "sudo", 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if id, _ := open.inventory.SourceIdentity(ctx, "sudo"); id != 100 {
|
||||
t.Fatalf("the recorded id became %d", id)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModuleMovesOnTheOperatorsApproveOfTheMergeThatMovedIt(t *testing.T) {
|
||||
const commit = "c0ffee0123456789"
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
change func(*anApprovedMerge)
|
||||
unkept bool
|
||||
request string // who asked: "merge", or "build" through a verb
|
||||
moved bool
|
||||
}{
|
||||
{what: "the operator's Approve of the merged head", request: "merge", moved: true},
|
||||
{what: "no merge mesh-delivery announced", unkept: true, request: "merge"},
|
||||
{what: "a warrant for other heads", request: "merge", change: func(a *anApprovedMerge) {
|
||||
a.merge.Heads = map[string]string{"novox/mesh-host@aaaaaaaaaaaa": "e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5"}
|
||||
}},
|
||||
{what: "a Decline", request: "merge", change: func(a *anApprovedMerge) {
|
||||
a.warrant.Option, a.warrant.Label, a.merge.Option = "decline", "Decline", "decline"
|
||||
}},
|
||||
{what: "a person who is not the operator", request: "merge", change: func(a *anApprovedMerge) { a.warrant.By.Who = "agent" }},
|
||||
{what: "a build asked through a verb", request: "build"},
|
||||
} {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
a := approvedMerge(t)
|
||||
a.merge.Owner, a.merge.Repo, a.merge.Number, a.merge.HeadSHA = "novox", "mesh-host", 7, "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1"
|
||||
if c.change != nil {
|
||||
c.change(&a)
|
||||
}
|
||||
wasKept, wasAsk := keptDeliveryMerge, routerAskOf
|
||||
keptDeliveryMerge = func(_ context.Context, owner, repo string, number int, head string) (deliveryMerge, bool, error) {
|
||||
return a.merge, !c.unkept && owner == "novox" && repo == "mesh-host" && number == 7 && head == a.merge.HeadSHA, nil
|
||||
}
|
||||
routerAskOf = func(_ context.Context, _ *inventory.Inventory, id string) (*asks.Ask, *asks.Warrant, error) {
|
||||
if id != a.merge.AskID {
|
||||
t.Errorf("%s: the router was asked for %q", c.what, id)
|
||||
}
|
||||
return &a.ask, &a.warrant, nil
|
||||
}
|
||||
t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk })
|
||||
event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-host", "base": "main", "number": 7,
|
||||
"head_sha": a.merge.HeadSHA, "merge_commit_sha": commit})
|
||||
if _, err := inv.AddMerge(ctx, inventory.BatchedMerge{Repository: "novox/mesh-host", Branch: "main", Commit: commit,
|
||||
Merged: time.Now(), Heard: time.Now(), Event: event}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id := fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano())
|
||||
if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: id, Repository: "novox/mesh-host", Seat: "git",
|
||||
Path: "modules/sudo", Ref: "main", Commit: commit, For: c.request}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
moved := onTrunk(id, "novox/mesh-host", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "2"})
|
||||
moved.Commit = commit
|
||||
_, _, err := takeIn(ctx, inv, moved)
|
||||
src, _ := inv.SourceOf(ctx, "sudo")
|
||||
switch {
|
||||
case c.moved && err != nil:
|
||||
t.Errorf("%s: the move was refused: %v", c.what, err)
|
||||
case c.moved && src.Repository != "novox/mesh-host":
|
||||
t.Errorf("%s: sudo is built from %q", c.what, src.Repository)
|
||||
case !c.moved && !errors.Is(err, errNotItsSource):
|
||||
t.Errorf("%s: the move was not refused as ADR 0266 has it: %v", c.what, err)
|
||||
case !c.moved && src.Repository != "novox/mesh-catalog":
|
||||
t.Errorf("%s: a refused move changed sudo's source to %q", c.what, src.Repository)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,128 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"hash/fnv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The routes to root a review of ADR 0266 found (novox/hq ADR 0266 §7): an agent makes a repository of its
|
||||
// own — or forks one the mesh builds from — commits a module.json naming a module the mesh runs everywhere
|
||||
// (`sudo`, granting itself a rule without a password; `mesh-host`, the node-engine), and asks the `build` verb
|
||||
// for it. Its outcome was registered under that name from the agent's repository, and the next push sent it.
|
||||
|
||||
// onTrunk is an outcome of a commit on its repository's trunk, as the build seat says it.
|
||||
func onTrunk(id, repository, seat, path string, manifest map[string]any) link.BuildResult {
|
||||
raw, _ := json.Marshal(manifest)
|
||||
r := link.BuildResult{ID: id, Repository: "http://forge.internal:20000/" + repository + ".git", Path: path,
|
||||
Ref: "main", On: "anchor", Commit: "c0ffee0123456789", Manifest: raw,
|
||||
Trunk: "main", OnTrunk: true, Branches: []string{"main"}}
|
||||
if seat != "" {
|
||||
r.Source = &link.SourceOnSeat{Seat: seat, Repository: repository}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// keptAsked keeps a build request as the asker would: through a verb, or at the terminal.
|
||||
func keptAsked(t *testing.T, inv *inventory.Inventory, id, repository, path string, atTerminal bool) {
|
||||
t.Helper()
|
||||
if err := inv.RecordBuildRequest(t.Context(), inventory.BuildRequest{ID: id, Repository: repository, Seat: "git",
|
||||
Path: path, For: "build", AtTerminal: atTerminal}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// theCatalogue is a mesh whose sudo is built from the catalogue repository and whose node-engine from its own.
|
||||
|
||||
// The operator at the terminal may still move a module, or add one from a new repository.
|
||||
|
||||
// The delivery's flow is untouched: a merge's rebuild of a module from its own repository, and a merge adding
|
||||
// a module to a repository the catalogue builds from (novox/hq issue 300), are registered with no terminal.
|
||||
|
||||
// Through a verb, a build of a repository the catalogue builds nothing from is not even asked: the build node
|
||||
// would run what the agent wrote.
|
||||
|
||||
// asTheOperator keeps a build as asked at the controller's terminal, as the operator's first build of a module
|
||||
// from a repository the catalogue does not yet build from is (novox/hq ADR 0266), and hands it back.
|
||||
func asTheOperator(t *testing.T, inv *inventory.Inventory, b link.BuildResult) link.BuildResult {
|
||||
t.Helper()
|
||||
repository, seat := b.Repository, ""
|
||||
if b.Source != nil {
|
||||
repository, seat = b.Source.Repository, b.Source.Seat
|
||||
}
|
||||
if err := inv.RecordBuildRequest(t.Context(), inventory.BuildRequest{ID: b.ID, Repository: repository, Seat: seat,
|
||||
Path: b.Path, For: "build", AtTerminal: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// A rollback puts back only a build of the module's own repository: an agent's build of the module's name,
|
||||
// recorded and refused, at the very commit the machine ran before (a fork carries it), is never registered by
|
||||
// the back door of a failed gate.
|
||||
|
||||
// theForge is what the forge says in a test, by owner/name: a repository not named here is protected as a
|
||||
// trunk must be, with an id of its own.
|
||||
var theForge sync.Map
|
||||
|
||||
func init() {
|
||||
askTheForge = func(_ context.Context, owner, repo, _ string) (forgeFacts, error) {
|
||||
if said, ok := theForge.Load(owner + "/" + repo); ok {
|
||||
switch f := said.(type) {
|
||||
case error:
|
||||
return forgeFacts{}, f
|
||||
case forgeFacts:
|
||||
return f, nil
|
||||
}
|
||||
}
|
||||
h := fnv.New32a()
|
||||
_, _ = h.Write([]byte(owner + "/" + repo))
|
||||
return forgeFacts{ID: int64(h.Sum32()), Guarded: true}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// A module's trunk the forge does not protect — direct pushes, no required status — or a forge that cannot say,
|
||||
// registers nothing through a verb: the trunk rule means nothing on a branch anyone pushes to.
|
||||
|
||||
// A repository deleted and made again under the module's repository's name is another repository: the forge's
|
||||
// id, recorded at registration, tells them apart.
|
||||
|
||||
// The terminal's mark is the operator's for the repository and path they asked: an outcome of another, under that
|
||||
// build's id, is not theirs.
|
||||
|
||||
// The serving controller, and everything it starts, is never the terminal: a build asked in it reads as asked
|
||||
// through the mesh even when no verb and no caller is named.
|
||||
|
||||
// The forge applies one rule to a branch: the rule named for it, else the first glob that covers it. A stronger
|
||||
// rule later in the list is not what guards the branch, and is not read as if it were (the confirmation review).
|
||||
func TestTheRuleJudgedIsTheOneTheForgeApplies(t *testing.T) {
|
||||
guarded := protectionRule{Rule: "*", RequiredStatuses: []string{"mesh/merge-gate"}}
|
||||
open := protectionRule{Rule: "main", Push: true, RequiredStatuses: []string{"mesh/merge-gate"}}
|
||||
if f := judgedRule([]protectionRule{guarded, open}, "main"); f.Guarded {
|
||||
t.Error("an exact rule letting pushes was passed over for a glob that guards")
|
||||
}
|
||||
if f := judgedRule([]protectionRule{{Rule: "ma*", RequiredStatuses: nil}, {Rule: "*", RequiredStatuses: []string{"x"}}},
|
||||
"main"); f.Guarded || !strings.Contains(f.Why, "ma*") {
|
||||
t.Errorf("the first glob covering the branch was not the one judged: %+v", f)
|
||||
}
|
||||
if f := judgedRule([]protectionRule{{Rule: "release/*"}, {Rule: "main", RequiredStatuses: []string{"x"}}}, "main"); !f.Guarded {
|
||||
t.Errorf("the rule named for the branch was not judged: %+v", f)
|
||||
}
|
||||
if f := judgedRule(nil, "main"); f.Guarded {
|
||||
t.Error("no rule is no protection")
|
||||
}
|
||||
}
|
||||
|
||||
// An id the forge could not give keeps the id already recorded.
|
||||
|
||||
// **The operator's Approve of a reviewed group registers a moved module from its new repository** (novox/hq ADR 0420 §7,
|
||||
// issue 1025). sudo is built from the catalogue; a merge into the repository of node-engine's module, announced by
|
||||
// mesh-delivery with the ask it merged on, brings its build from there. Accepted with the operator's Approve of the
|
||||
// merged head; refused with no kept merge, a warrant for other heads, a Decline, a person who is not the operator, and
|
||||
// through a verb.
|
||||
@@ -1,9 +1,13 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
@@ -11,12 +15,126 @@ import (
|
||||
// the module registered with its source as the seat and path when the request said so — never the
|
||||
// URL. A definition naming an installation is recorded and not registered; a failure is recorded
|
||||
// and said.
|
||||
func TestABuildHeardIsRecordedAndRegistered(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
manifest, _ := json.Marshal(map[string]any{"module": "shop", "version": "3"})
|
||||
m, _, err := takeIn(ctx, open.inventory, link.BuildResult{
|
||||
ID: "b-1", Repository: "http://forge.internal:20000/novox/shop.git", Path: "modules/shop",
|
||||
Ref: "main", On: "anchor", Commit: "abcdef0123", Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/shop"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m.Module != "shop" {
|
||||
t.Fatalf("registered %q", m.Module)
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, held := shelf["shop"]; !held {
|
||||
t.Fatal("the module a heard build produced is not in the catalogue")
|
||||
}
|
||||
src, err := open.inventory.SourceOf(ctx, "shop")
|
||||
if err != nil || src.Seat != "git" || src.Repository != "novox/shop" || src.BuiltFrom != "abcdef0123" {
|
||||
t.Fatalf("the source is the seat and the path, never the URL: %+v %v", src, err)
|
||||
}
|
||||
builds, err := open.inventory.Builds(ctx, "shop", 5)
|
||||
if err != nil || len(builds) != 1 || builds[0].ID != "b-1" {
|
||||
t.Fatalf("the build is not recorded once: %v %v", builds, err)
|
||||
}
|
||||
|
||||
named, _ := json.Marshal(map[string]any{"module": "idp", "version": "1", "resources": []any{
|
||||
map[string]any{"id": "server", "type": "container", "image": "x@sha256:aa",
|
||||
"env": map[string]any{"KC_HOSTNAME": "https://login.mesh-one.be"}}}})
|
||||
_, _, err = takeIn(ctx, open.inventory, link.BuildResult{
|
||||
ID: "b-2", Repository: "/r", On: "anchor", Commit: "0123456789", Manifest: named})
|
||||
if err == nil || !strings.Contains(err.Error(), "does not register it") {
|
||||
t.Fatalf("a definition naming an installation was taken in: %v", err)
|
||||
}
|
||||
if shelf, _ := open.inventory.Catalogue(ctx); shelf["idp"].Module != "" {
|
||||
t.Fatal("the refused module was registered anyway")
|
||||
}
|
||||
if builds, _ := open.inventory.Builds(ctx, "idp", 5); len(builds) != 1 {
|
||||
t.Fatalf("the refused build was not recorded: %v", builds)
|
||||
}
|
||||
|
||||
_, _, err = takeIn(ctx, open.inventory, link.BuildResult{ID: "b-3", Repository: "/r", On: "anchor", Failed: "no compiler"})
|
||||
if err == nil || !strings.Contains(err.Error(), "no compiler") {
|
||||
t.Fatalf("a failure is said in the builder's words: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq 04-ISSUES/215: a build asked at a commit is recorded as built from that commit, and the
|
||||
// module keeps following the branch it followed — a new one, the default branch.
|
||||
func TestABuildAtACommitKeepsTheBranchTheModuleFollows(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
manifest, _ := json.Marshal(map[string]any{"module": "unifi", "version": "1"})
|
||||
result := func(id, ref, commit string) link.BuildResult {
|
||||
return link.BuildResult{ID: id, Repository: "http://forge.internal:20000/novox/mesh-catalog.git",
|
||||
Path: "modules/unifi", Ref: ref, On: "anchor", Commit: commit, Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, result("b-1", "main", "1111111aaaa")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, result("b-2", "9c97a8a", "9c97a8a1d2c3")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
src, err := open.inventory.SourceOf(ctx, "unifi")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if src.Ref != "main" || src.BuiltFrom != "9c97a8a1d2c3" {
|
||||
t.Errorf("after a build at a commit the module follows %q, built from %q; want main, 9c97a8a1d2c3", src.Ref, src.BuiltFrom)
|
||||
}
|
||||
|
||||
// One new to the catalogue, first built at a commit, follows the default branch.
|
||||
other, _ := json.Marshal(map[string]any{"module": "letta", "version": "1"})
|
||||
r := result("b-3", "deadbeef", "deadbeefcafe")
|
||||
r.Manifest, r.Path = other, "modules/letta"
|
||||
if _, _, err := takeIn(ctx, open.inventory, r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if src, _ := open.inventory.SourceOf(ctx, "letta"); src.Ref != "" {
|
||||
t.Errorf("a module first built at a commit follows %q, want the default branch", src.Ref)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq 04-ISSUES/219: an older request heard after a newer one is recorded and not registered,
|
||||
// so a push sends what the newer request built.
|
||||
func TestAnOlderBuildHeardLaterDoesNotReplaceTheNewer(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
older := time.Date(2026, 10, 3, 21, 33, 45, 0, time.UTC)
|
||||
newer := time.Date(2026, 10, 3, 21, 51, 57, 0, time.UTC)
|
||||
result := func(asked time.Time, image string) link.BuildResult {
|
||||
manifest, _ := json.Marshal(map[string]any{"module": "postgres", "version": image})
|
||||
return link.BuildResult{ID: link.NewBuildID(asked), Repository: "http://forge.internal:20000/novox/mesh-catalog.git",
|
||||
Path: "modules/postgres", Ref: "main", On: "anchor", Commit: "efff5415", Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, result(newer, "4bcd5f73")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, err := takeIn(ctx, open.inventory, result(older, "0ab07fa9"))
|
||||
if !errors.Is(err, inventory.ErrSuperseded) {
|
||||
t.Fatalf("the older request's outcome was taken in as current: %v", err)
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := shelf["postgres"].Version; got != "4bcd5f73" {
|
||||
t.Errorf("postgres is %q; want the newer request's 4bcd5f73", got)
|
||||
}
|
||||
if builds, _ := open.inventory.Builds(ctx, "postgres", 5); len(builds) != 2 {
|
||||
t.Errorf("the late build was not recorded: %v", builds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABuildIDSaysWhenItWasAsked(t *testing.T) {
|
||||
at := time.Date(2026, 10, 3, 21, 51, 57, 392539762, time.UTC)
|
||||
|
||||
@@ -1,92 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **Every user's slow consumer is S9's, not the controller's alone** (novox/hq ADR 0357, issue 270). On
|
||||
// 2026-10-07 the bus closed two clients on the laptop as slow consumers and nothing raised a condition:
|
||||
// the controller heard only its own connection. Said by the bus module and read back, a user the bus
|
||||
// closes is one condition naming the module and its machine, raised at the first look and cleared an hour
|
||||
// after the last closure.
|
||||
func TestAUserTheBusClosedAsASlowConsumerIsOneConditionNamingItsModuleAndMachine(t *testing.T) {
|
||||
closed := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC)
|
||||
now := closed.Add(20 * time.Second)
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
||||
t.Cleanup(func() { k.Close(t.Context()) })
|
||||
|
||||
advisories := link.BusClosures([]link.BusClosure{
|
||||
{User: "laptop.claude-code", CID: 10, Reason: "Slow Consumer (Write Deadline)", Opened: closed.Add(-time.Hour),
|
||||
Closed: closed.Add(-10 * time.Minute)},
|
||||
{User: "laptop.claude-code", CID: 44, Reason: "Slow Consumer (Write Deadline)", Opened: closed.Add(-time.Minute),
|
||||
Closed: closed},
|
||||
{User: "anchor.desk", CID: 7, Reason: "Maximum Message Payload Exceeded", Opened: closed.Add(-time.Hour),
|
||||
Closed: closed},
|
||||
}, []string{link.AdvisorySlowConsumer, link.AdvisorySlowConsumer, link.AdvisoryConnectionClosed}, closed.Add(-time.Hour))
|
||||
look := func() {
|
||||
t.Helper()
|
||||
f := &signalFacts{now: now, host: "anchor", advisories: advisories}
|
||||
if err := k.Reconcile(t.Context(), "S9", watchAdvisories(f)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
look()
|
||||
|
||||
slow, found, err := conditions.ReadOne(t.Context(), store, "bus.laptop.claude-code.slow-consumer")
|
||||
if err != nil || !found {
|
||||
t.Fatalf("no slow-consumer condition for the user: %v %v", found, err)
|
||||
}
|
||||
if slow.Subject.Machine != "laptop" || slow.Observations != 2 || !slow.LastObserved.Equal(closed) {
|
||||
t.Fatalf("the slow consumer is said on %q, %d closure(s), last %s: %+v", slow.Subject.Machine, slow.Observations,
|
||||
slow.LastObserved, slow)
|
||||
}
|
||||
if !strings.Contains(slow.Summary, "claude-code on laptop") || strings.Contains(slow.Summary, "laptop.claude-code") {
|
||||
t.Fatalf("the summary does not name the module in words, or carries the dotted user: %q", slow.Summary)
|
||||
}
|
||||
if !strings.Contains(slow.Headline, "Claude-code on laptop") {
|
||||
t.Fatalf("the headline is %q", slow.Headline)
|
||||
}
|
||||
cut, found, err := conditions.ReadOne(t.Context(), store, "bus.anchor.desk.connection-closed")
|
||||
if err != nil || !found || cut.Subject.Machine != "anchor" || !strings.Contains(cut.Summary, "maximum message payload exceeded") {
|
||||
t.Fatalf("the user cut off for breaking the bus's rules: %v %v %+v", found, err, cut)
|
||||
}
|
||||
|
||||
// An hour after the newest closure without another, both clear.
|
||||
now = closed.Add(time.Hour + time.Minute)
|
||||
look()
|
||||
for _, key := range []string{"bus.laptop.claude-code.slow-consumer", "bus.anchor.desk.connection-closed"} {
|
||||
if _, found, _ := k.Get(t.Context(), key); found {
|
||||
t.Fatalf("%s is still open an hour after the last closure", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **A read of the bus module's events that fails does not blind S9** (review of #326): the controller's own
|
||||
// signals are still said, and the failed read is a condition of its own.
|
||||
func TestABusModuleReadThatFailsKeepsTheControllersOwnSignals(t *testing.T) {
|
||||
at := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC)
|
||||
f := &signalFacts{now: at.Add(time.Minute), host: "anchor",
|
||||
advisories: []link.Advisory{{Kind: link.AdvisorySlowConsumer, ID: "controller", Said: "the controller fell behind",
|
||||
First: at, Last: at, Count: 1}},
|
||||
busFaultsErr: errors.New("nats: timeout")}
|
||||
keys := map[string]bool{}
|
||||
for _, o := range watchAdvisories(f) {
|
||||
keys[o.Key()] = true
|
||||
}
|
||||
if !keys["bus.controller.slow-consumer"] || !keys["bus.bus-module.bus-faults-unread"] {
|
||||
t.Fatalf("observed %v: the controller's own slow consumer and the unread events, both", keys)
|
||||
}
|
||||
for _, row := range signalsTable {
|
||||
if row.Row == "S9" && row.needs(f) != nil {
|
||||
t.Fatal("the S9 row went blind for a read of the bus module's events")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,195 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A send held for the bus's planned step (novox/hq issue 336).
|
||||
//
|
||||
// **A wait only a person can end is said to that person at once.** No send replaces the bus but its planned
|
||||
// step (sendToEach, ADR 0236), and the step is a person's `bus upgrade`. So while a new bus build waits, every
|
||||
// walk whose tier sends to the bus's machine is refused, and tries again each tick. The refusal was kept only
|
||||
// as the walk's note; nothing was raised, and the operator found the hold by asking why a walk did not move.
|
||||
// Row S17 raises it on the first tick after the first refusal, in the bus's scope, for the operator: what
|
||||
// waits, behind which bus build, since when, and the verb. A walk held only by it is not late, so S3 leaves it
|
||||
// out, as it leaves out a walk under a paused build seat. It clears once the bus's machine has been sent the
|
||||
// build the mesh holds — the step was taken, and no send is refused for the bus any more — whatever the walks'
|
||||
// notes still say. Whether the new bus came up healthy is the step's own condition (probe DB).
|
||||
|
||||
// kindBusStepWaiting is S17's kind: bus.<module>.step-waiting.
|
||||
const kindBusStepWaiting = "bus-step-waiting"
|
||||
|
||||
// busFacts is a new bus build waiting for its planned step, and the sends held for it.
|
||||
type busFacts struct {
|
||||
module, to string
|
||||
// from is the build each machine of the bus runs; machines those whose bus the step would replace.
|
||||
from map[string]string
|
||||
machines []string
|
||||
waits []busWaitFacts
|
||||
}
|
||||
|
||||
// busWaitFacts is one walk whose send was refused because it would replace the bus.
|
||||
type busWaitFacts struct {
|
||||
plan, repository, commit string
|
||||
// modules are what its tier sends: what waits.
|
||||
modules []string
|
||||
// since is the first refusal: as this controller saw it, or, read back, the save that kept the refusal.
|
||||
since time.Time
|
||||
}
|
||||
|
||||
// busRefusedFirst is when this controller first saw each walk refused for the bus's step. The walk's note
|
||||
// keeps the refusal across a restart; this keeps its moment more exactly than the walk's last save.
|
||||
var busRefusedFirst = &firstSeen{at: map[string]time.Time{}}
|
||||
|
||||
type firstSeen struct {
|
||||
mu sync.Mutex
|
||||
at map[string]time.Time
|
||||
}
|
||||
|
||||
// mark keeps the first moment an id was seen.
|
||||
func (s *firstSeen) mark(id string, at time.Time) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if _, seen := s.at[id]; !seen {
|
||||
s.at[id] = at
|
||||
}
|
||||
}
|
||||
|
||||
// of is when an id was first seen; zero when it was not.
|
||||
func (s *firstSeen) of(id string) time.Time {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.at[id]
|
||||
}
|
||||
|
||||
// keepOnly forgets every id not given: a walk no longer held is not held since then.
|
||||
func (s *firstSeen) keepOnly(ids map[string]bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for id := range s.at {
|
||||
if !ids[id] {
|
||||
delete(s.at, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// refusedForTheBus is whether an error is the refusal of a send for the bus's planned step.
|
||||
func refusedForTheBus(err error) bool {
|
||||
return err != nil && strings.Contains(err.Error(), errBusWaits.Error())
|
||||
}
|
||||
|
||||
// busWaitsOf is the sends held for the bus's step, from the open walks: each walk not waiting for its
|
||||
// delivery's word whose note keeps a refusal for this very bus build, while the build would still replace
|
||||
// the bus on a machine. first is when this controller first saw a walk refused, zero when it did not.
|
||||
func busWaitsOf(plans []inventory.Plan, b busPending, first func(string) time.Time) busFacts {
|
||||
f := busFacts{module: b.module, to: b.to, from: b.from}
|
||||
for _, n := range b.machines {
|
||||
if b.moves(n) {
|
||||
f.machines = append(f.machines, n)
|
||||
}
|
||||
}
|
||||
if len(f.machines) == 0 {
|
||||
return f
|
||||
}
|
||||
for _, p := range plans {
|
||||
if !p.Open() || p.Waiting() || !strings.Contains(p.Note, errBusWaits.Error()) || !strings.Contains(p.Note, short(b.to)) {
|
||||
continue
|
||||
}
|
||||
since := p.Updated
|
||||
if seen := first(p.ID); !seen.IsZero() && (since.IsZero() || seen.Before(since)) {
|
||||
since = seen
|
||||
}
|
||||
var modules []string
|
||||
if p.Tier >= 0 && p.Tier < len(p.Tiers) {
|
||||
modules = append(modules, p.Tiers[p.Tier]...)
|
||||
} else {
|
||||
modules = planModules(p)
|
||||
}
|
||||
sort.Strings(modules)
|
||||
f.waits = append(f.waits, busWaitFacts{plan: p.ID, repository: p.Repository, commit: p.Commit, modules: modules,
|
||||
since: since})
|
||||
}
|
||||
sort.Slice(f.waits, func(i, j int) bool { return f.waits[i].since.Before(f.waits[j].since) })
|
||||
return f
|
||||
}
|
||||
|
||||
// heldByTheBus is the walks of a bus's facts, by id: what S3 leaves out.
|
||||
func (b busFacts) heldByTheBus() map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, w := range b.waits {
|
||||
out[w.plan] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// busUpgradeVerb is the call that ends the wait, as the summary names it: through the mesh MCP server, with
|
||||
// why, saying whether the new version can be undone ("reversible": "true") or not ("irreversible": "true").
|
||||
const busUpgradeVerb = "`mesh_call mesh-controller.bus {\"upgrade\": \"true\", \"why\": \"…\", \"reversible\" or \"irreversible\": \"true\"}`"
|
||||
|
||||
// watchBusWaits is S17: a send held for the bus's planned step, said at once to the operator.
|
||||
func watchBusWaits(f *signalFacts) []conditions.Observation {
|
||||
b := f.bus
|
||||
if len(b.waits) == 0 || len(b.machines) == 0 {
|
||||
return nil
|
||||
}
|
||||
since := b.waits[0].since
|
||||
var walks, what []string
|
||||
for _, w := range b.waits {
|
||||
walks = append(walks, fmt.Sprintf("%s (%s %s, tier: %s)", w.plan, w.repository, short(w.commit),
|
||||
strings.Join(w.modules, ", ")))
|
||||
what = append(what, deliveryWhat(w.modules, w.repository))
|
||||
}
|
||||
var from []string
|
||||
for _, n := range b.machines {
|
||||
from = append(from, short(orNotKnown(b.from[n])))
|
||||
}
|
||||
machines := namesWords(b.machines, 3)
|
||||
in := f.now.Sub(since)
|
||||
return []conditions.Observation{{Scope: conditions.ScopeBus, ID: b.module, Token: "step-waiting",
|
||||
Kind: kindBusStepWaiting, Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
||||
Machine: b.machines[0], Also: b.machines[1:],
|
||||
Summary: fmt.Sprintf("sends to %s wait for the bus's planned step: a new bus build (%s %s → %s) would replace "+
|
||||
"the bus there, which only a person's %s does; waiting since %s: %s", strings.Join(b.machines, ", "),
|
||||
b.module, strings.Join(sortedUnique(from), ", "), short(b.to), busUpgradeVerb,
|
||||
since.UTC().Format(time.RFC3339), strings.Join(walks, "; ")),
|
||||
Said: fmt.Sprintf("%d walk(s) refused since %s", len(b.waits), since.UTC().Format(time.RFC3339)),
|
||||
Headline: clipWords("Sends to "+machines+" wait for a bus upgrade", conditions.HeadlineMax),
|
||||
Explanation: clipWords(fmt.Sprintf("A new version of the mesh's message system is built, and only a person "+
|
||||
"installs it. Until then nothing else is sent to %s: %s waits, for %s so far.", machines,
|
||||
namesWords(sortedUnique(what), 3), humanDuration(in)), conditions.ExplanationMax),
|
||||
Needs: "start the bus upgrade " + FromMeshMCPServer,
|
||||
Resolved: "Resolved: the bus upgrade started, and sends go on"}}
|
||||
}
|
||||
|
||||
// sortedUnique is a list sorted, each once.
|
||||
func sortedUnique(xs []string) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, x := range xs {
|
||||
if !seen[x] {
|
||||
seen[x] = true
|
||||
out = append(out, x)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// clipWords keeps a plain sentence within a bound, at a word.
|
||||
func clipWords(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
cut := strings.LastIndex(s[:n-1], " ")
|
||||
if cut <= 0 {
|
||||
cut = n - 1
|
||||
}
|
||||
return s[:cut] + "…"
|
||||
}
|
||||
@@ -1,70 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
func TestASendHeldForTheBusStepIsFoundFromTheWalksItHolds(t *testing.T) {
|
||||
now := time.Date(2026, 10, 8, 18, 30, 0, 0, time.UTC)
|
||||
b := aBusOnAnchor()
|
||||
walk := func(id, repo, note string, updated time.Time) inventory.Plan {
|
||||
return inventory.Plan{ID: id, Repository: repo, Commit: "c0ffee001122", State: inventory.PlanRolling, Tier: 0,
|
||||
Tiers: [][]string{{"mesh-host"}}, Modules: map[string]*inventory.PlanModule{"mesh-host": {}}, Note: note,
|
||||
Updated: updated}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
bus busPending
|
||||
plans []inventory.Plan
|
||||
first map[string]time.Time
|
||||
want []string
|
||||
since time.Time
|
||||
}{
|
||||
{"a walk refused for the bus", b,
|
||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now.Add(-27*time.Minute))}, nil,
|
||||
[]string{"plan-1"}, now.Add(-27 * time.Minute)},
|
||||
{"refused earlier than its last save, as this controller saw it", b,
|
||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now.Add(-5*time.Minute))},
|
||||
map[string]time.Time{"plan-1": now.Add(-28 * time.Minute)}, []string{"plan-1"}, now.Add(-28 * time.Minute)},
|
||||
{"a walk refused for another reason", b,
|
||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", "tier 0: the build seat is paused — tried again", now)}, nil,
|
||||
nil, time.Time{}},
|
||||
{"refused for an older bus build than the one held now", func() busPending { o := b; o.to = "99999999cccc"; return o }(),
|
||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)}, nil, nil, time.Time{}},
|
||||
{"the bus already runs the build held: its step started", func() busPending {
|
||||
o := aBusOnAnchor()
|
||||
o.from = map[string]string{"anchor": o.to}
|
||||
return o
|
||||
}(), []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)}, nil, nil, time.Time{}},
|
||||
{"a walk waiting for its delivery's word asks no send", b, func() []inventory.Plan {
|
||||
p := walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)
|
||||
p.Delivery = &inventory.PlanDelivery{Awaits: "mesh-delivery"}
|
||||
return []inventory.Plan{p}
|
||||
}(), nil, nil, time.Time{}},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
got := busWaitsOf(c.plans, c.bus, func(id string) time.Time { return c.first[id] })
|
||||
var ids []string
|
||||
for _, w := range got.waits {
|
||||
ids = append(ids, w.plan)
|
||||
}
|
||||
if strings.Join(ids, ",") != strings.Join(c.want, ",") {
|
||||
t.Fatalf("held for the bus: %v, want %v", ids, c.want)
|
||||
}
|
||||
if len(c.want) > 0 {
|
||||
if !got.waits[0].since.Equal(c.since) {
|
||||
t.Errorf("waiting since %s, want %s", got.waits[0].since, c.since)
|
||||
}
|
||||
if strings.Join(got.machines, ",") != "anchor" || got.to != b.to || got.module != "nats" {
|
||||
t.Errorf("behind %+v", got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,99 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// novox/hq issue 336: a send held because it would replace the bus outside its planned step waits for a
|
||||
// person, so a person is told at once — what waits, behind which bus build, since when, and the verb that
|
||||
// ends it — and the wait is not read as a walk running late.
|
||||
|
||||
// aBusOnAnchor is a new bus build waiting for its step on anchor: nats 88135ad0 there, 32307bd1 held.
|
||||
func aBusOnAnchor() busPending {
|
||||
return busPending{module: "nats", machines: []string{"anchor"}, from: map[string]string{"anchor": "88135ad0aaaa"},
|
||||
to: "32307bd1bbbb", same: map[string]bool{}}
|
||||
}
|
||||
|
||||
// refusedNote is the note a walk keeps when its send was refused for the bus, as advanceHeld writes it.
|
||||
func refusedNote(b busPending, tier int) string {
|
||||
held := "sending anchor would replace the bus (nats " + short(b.from["anchor"]) + " → " + short(b.to) +
|
||||
"), which is a planned step: `bus upgrade --why …` snapshots its streams first and checks them after (novox/hq ADR 0236)"
|
||||
return "tier " + string(rune('0'+tier)) + ": " + errBusWaits.Error() + ": " + held + " — tried again"
|
||||
}
|
||||
|
||||
// **Said at once, not as lateness, and cleared when the step starts**: the walk held only for the bus raises
|
||||
// the bus's condition on the first tick, before any tier bound, naming what waits, the bus build from and to,
|
||||
// since when and `bus upgrade`, for the operator; S3 says nothing of it even past its bound; and the
|
||||
// condition clears once the bus's machine runs the new build.
|
||||
func TestASendHeldForTheBusStepIsSaidAtOnceAndNotAsLateness(t *testing.T) {
|
||||
now := time.Date(2026, 10, 8, 18, 30, 0, 0, time.UTC)
|
||||
b := aBusOnAnchor()
|
||||
held := func(entered time.Duration) *signalFacts {
|
||||
f := calm(now)
|
||||
f.plans = []planFacts{{id: "plan-1", repository: "novox/mesh-host", commit: "c0ffee00", tier: 0, tiers: 1,
|
||||
entered: now.Add(-entered), bound: 30 * time.Minute, waiting: refusedNote(b, 0), bus: true}}
|
||||
f.bus = busFacts{module: "nats", to: b.to, from: b.from, machines: []string{"anchor"},
|
||||
waits: []busWaitFacts{{plan: "plan-1", repository: "novox/mesh-host", commit: "c0ffee001122",
|
||||
modules: []string{"mesh-host"}, since: now.Add(-time.Minute)}}}
|
||||
return f
|
||||
}
|
||||
|
||||
f := held(time.Minute)
|
||||
got := watchBusWaits(f)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("a send held for the bus a minute ago raised %+v", got)
|
||||
}
|
||||
o := got[0]
|
||||
if o.Key() != "bus.nats.step-waiting" || o.Kind != kindBusStepWaiting || o.Resolver != conditions.ResolverOperator {
|
||||
t.Fatalf("raised %s (%s), resolver %q", o.Key(), o.Kind, o.Resolver)
|
||||
}
|
||||
for _, want := range []string{"anchor", "mesh-host", "88135ad0", "32307bd1", "mesh_call mesh-controller.bus",
|
||||
`"upgrade": "true"`, `"reversible"`, `"irreversible"`, "2026-10-08T18:29:00Z", "plan-1"} {
|
||||
if !strings.Contains(o.Summary, want) {
|
||||
t.Errorf("its summary does not say %q: %s", want, o.Summary)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(o.Explanation, "mesh-host") || !strings.Contains(o.Headline, "bus upgrade") || o.Needs == "" {
|
||||
t.Errorf("its words do not say what waits and what the operator does: %+v", o)
|
||||
}
|
||||
if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Explanation: o.Explanation,
|
||||
Resolved: o.Resolved, Needs: o.Needs}, "anchor"); !ok {
|
||||
t.Errorf("its words are not plain: %s", why)
|
||||
}
|
||||
|
||||
// Past S3's bound: still the bus's wait, never a stalled walk.
|
||||
late := held(45 * time.Minute)
|
||||
if s3 := watchPlans(late); len(s3) != 0 {
|
||||
t.Fatalf("a walk held only by the bus step was said stalled: %+v", s3)
|
||||
}
|
||||
// A walk held for something else past its bound is still stalled.
|
||||
other := held(45 * time.Minute)
|
||||
other.plans[0].bus = false
|
||||
if s3 := watchPlans(other); len(s3) != 1 {
|
||||
t.Fatalf("a walk held for something else past its bound raised %+v", s3)
|
||||
}
|
||||
|
||||
// Through the keeper: open at the first tick, cleared when the step started.
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: &conditions.Told{},
|
||||
Now: func() time.Time { return now }})
|
||||
defer k.Close(context.Background())
|
||||
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
||||
w.see(t.Context(), held(time.Minute))
|
||||
open, err := k.Open(t.Context())
|
||||
if err != nil || len(open) != 1 || open[0].Key != "bus.nats.step-waiting" {
|
||||
t.Fatalf("after the first tick, open: %+v (%v)", open, err)
|
||||
}
|
||||
started := held(time.Minute)
|
||||
started.bus = busFacts{module: "nats", to: b.to}
|
||||
started.plans[0].bus = false
|
||||
w.see(t.Context(), started)
|
||||
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||
t.Fatalf("the step started, and open: %+v", open)
|
||||
}
|
||||
}
|
||||
@@ -49,66 +49,18 @@ func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Ra
|
||||
// composed from, so a module the mesh grants a consumer's subjects has that consumer waiting.
|
||||
// Done on every raise, not only when a credential is issued: every module moved onto this bus
|
||||
// by the rollout was issued on the old one, and came up with nothing to bind to (2026-09-28).
|
||||
consumers, records, err := moduleConsumersAndRecords(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// And a grant of a restricted stream that composes nothing, said on every raise and send (novox/hq ADR 0376 §2).
|
||||
sayWithheld(records, " ")
|
||||
// And the work queues of seats that name their caller or their kind, with each holder's worker
|
||||
// (novox/hq ADR 0259 §3): an ask queues until the router takes it, a channel's work until that kind
|
||||
// takes it.
|
||||
trafficStreams, trafficWorkers, err := seatTrafficObjects(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// And every module's own stream (novox/hq ADR 0363), from the catalogue, before the consumers on it.
|
||||
modStreams, err := inv.DeclaredStreams(ctx)
|
||||
consumers, err := moduleConsumers(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Every one tried, and every failure named: one module's consumer the bus refuses is no reason
|
||||
// the modules after it in the list hear nothing (novox/hq issue 208, where this runs on each send).
|
||||
var failed []error
|
||||
for _, s := range broker.ModuleStreams(modStreams) {
|
||||
if err := r.EnsureStream(s); err != nil {
|
||||
failed = append(failed, fmt.Errorf("the module stream %s: %w", s.Name, err))
|
||||
}
|
||||
}
|
||||
for _, s := range trafficStreams {
|
||||
if err := r.EnsureStream(s); err != nil {
|
||||
failed = append(failed, fmt.Errorf("the work queue %s: %w", s.Name, err))
|
||||
}
|
||||
}
|
||||
for _, c := range trafficWorkers {
|
||||
if err := r.EnsureConsumer(c); err != nil {
|
||||
failed = append(failed, fmt.Errorf("the worker %s on %s: %w", c.Name, c.Stream, err))
|
||||
}
|
||||
}
|
||||
restricted := map[string]bool{}
|
||||
for _, s := range modStreams {
|
||||
restricted[s.Name] = s.Restricted
|
||||
}
|
||||
for _, c := range consumers {
|
||||
if err := broker.EnsureHeard(r, c.Consumer, restricted[c.Consumer.Stream]); err != nil {
|
||||
if err := r.EnsureConsumer(c.Consumer); err != nil {
|
||||
failed = append(failed, fmt.Errorf("how %s on %s hears what it consumes: %w", c.Module, c.Node, err))
|
||||
}
|
||||
}
|
||||
// **And a consumer no longer granted loses what it reads through** (novox/hq ADR 0363 §6): every consumer
|
||||
// on a module's stream the composition no longer asks for is removed, on this very send.
|
||||
if p, prunes := r.(broker.Pruner); prunes && len(modStreams) > 0 {
|
||||
var wanted []broker.Consumer
|
||||
for _, c := range consumers {
|
||||
wanted = append(wanted, c.Consumer)
|
||||
}
|
||||
removed, err := broker.PruneHeard(p, modStreams, wanted)
|
||||
if err != nil {
|
||||
failed = append(failed, err)
|
||||
}
|
||||
for _, gone := range removed {
|
||||
fmt.Printf(" removed the consumer %s: its module is no longer granted that stream (novox/hq ADR 0363)\n", gone)
|
||||
}
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
return nil, errors.Join(failed...)
|
||||
}
|
||||
@@ -167,52 +119,15 @@ func unassertedObservation(err error) conditions.Observation {
|
||||
|
||||
// moduleConsumers is every module's durable consumer, from the records the user list is composed from.
|
||||
func moduleConsumers(ctx context.Context, inv *inventory.Inventory) ([]broker.ModuleConsumer, error) {
|
||||
consumers, _, err := moduleConsumersAndRecords(ctx, inv)
|
||||
return consumers, err
|
||||
}
|
||||
|
||||
// moduleConsumersAndRecords is moduleConsumers with the records they were composed from.
|
||||
func moduleConsumersAndRecords(ctx context.Context, inv *inventory.Inventory) ([]broker.ModuleConsumer, broker.Records, error) {
|
||||
records, err := inv.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return nil, broker.Records{}, err
|
||||
return nil, err
|
||||
}
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
return nil, broker.Records{}, err
|
||||
return nil, err
|
||||
}
|
||||
return broker.ConsumersOf(users), records, nil
|
||||
}
|
||||
|
||||
// seatTrafficObjects is the work queues and workers of seats that name their caller or their kind, from
|
||||
// the records the user list is composed from.
|
||||
func seatTrafficObjects(ctx context.Context, inv *inventory.Inventory) ([]broker.Stream, []broker.Consumer, error) {
|
||||
records, err := inv.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
streams, workers := broker.SeatTrafficObjects(users)
|
||||
// And the queue of every such seat the catalogue declares, held or not: work queues from registration,
|
||||
// so what is submitted before a holder is assigned waits for it (the correctness review of 2026-10-08).
|
||||
declared, err := inv.DeclaredTrafficSeats(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
have := map[string]bool{}
|
||||
for _, s := range streams {
|
||||
have[s.Name] = true
|
||||
}
|
||||
for _, s := range broker.TrafficQueues(declared) {
|
||||
if !have[s.Name] {
|
||||
streams = append(streams, s)
|
||||
have[s.Name] = true
|
||||
}
|
||||
}
|
||||
return streams, workers, nil
|
||||
return broker.ConsumersOf(users), nil
|
||||
}
|
||||
|
||||
// moduleConsumerCount is how many modules hear what they consume, for the raise's one line.
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
|
||||
@@ -1,103 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A first-node gate waits for a busy container (novox/hq ADR 0335 §4).
|
||||
//
|
||||
// When a machine reports, on what the gate sent it, that a container of a judged module was left running
|
||||
// because it said busy, the judging has not started: the build is not yet what runs there. The walk says it
|
||||
// waits for a busy container on that machine, counts no pass and no bound, and starts the judging — its
|
||||
// Since — at the report that no longer lists the container waiting. The wait is the node-engine's, bounded
|
||||
// at 15 minutes; the controller's own bound (busyGateBound past the engine's) is for an engine that stopped
|
||||
// reporting while it waited, so no walk waits for ever. An engine older than the field never lists one, and
|
||||
// its gate starts as before.
|
||||
|
||||
// busyGateSlack is how long past the engine's bound the controller still waits for the report that says the
|
||||
// container replaced: a pass of the engine (its asks every 20 seconds) and the report's way here.
|
||||
var busyGateSlack = 5 * time.Minute
|
||||
|
||||
// busyEngineBound is the node-engine's bound on a wait (novox/hq ADR 0335 §3), counted again here on this
|
||||
// controller's clock from when it first read the wait.
|
||||
const busyEngineBound = 15 * time.Minute
|
||||
|
||||
// busyGateEnds is when the gate stops waiting: the engine's own word on its bound, or this controller's count of
|
||||
// it from when it first read the wait, whichever is sooner, and the slack — so neither a clock ahead on the
|
||||
// machine nor a wrong word from it holds a walk longer.
|
||||
func busyGateEnds(b *inventory.GateBusy) time.Time {
|
||||
until := b.Seen.Add(busyEngineBound)
|
||||
if !b.Until.IsZero() && b.Until.Before(until) {
|
||||
until = b.Until
|
||||
}
|
||||
return until.Add(busyGateSlack)
|
||||
}
|
||||
|
||||
// busyWait decides, before a judging, whether the gate waits for a busy container on a machine it judges:
|
||||
// true while it waits, the gate saying so. False when nothing waits, or no longer: a wait that ended moves the
|
||||
// gate's Since to its end. Pure but for the gate it is handed.
|
||||
func busyWait(g *inventory.PlanGate, pairs []judged, f gateFacts, now time.Time) bool {
|
||||
if g.Busy != nil && g.Busy.Ended != nil {
|
||||
return false
|
||||
}
|
||||
for _, j := range pairs {
|
||||
b, waits := busyOn(j, f)
|
||||
if !waits {
|
||||
continue
|
||||
}
|
||||
if g.Busy == nil {
|
||||
g.Busy = &inventory.GateBusy{Machine: j.node, Module: j.module, Container: b.Container, Since: b.Since,
|
||||
Seen: now}
|
||||
}
|
||||
if g.Busy.Seen.IsZero() {
|
||||
g.Busy.Seen = now
|
||||
}
|
||||
g.Busy.Until, g.Busy.Said = b.Until, b.Said
|
||||
if now.After(busyGateEnds(g.Busy)) {
|
||||
// The engine said it would have replaced it by now: the judging starts, and finds the module not
|
||||
// yet healthy while the machine still lists it waiting (busyStill), until the gate's own bound.
|
||||
end := now
|
||||
g.Busy.Ended, g.Busy.PastBound = &end, true
|
||||
g.Since, g.Passes, g.LastPass, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, nil, nil, nil
|
||||
return false
|
||||
}
|
||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, g.Busy.Words(), nil
|
||||
g.Read(now, false, g.Busy.Words())
|
||||
return true
|
||||
}
|
||||
if g.Busy == nil {
|
||||
return false
|
||||
}
|
||||
// Replaced: the judging starts at the report that said so.
|
||||
end := now
|
||||
if r, said := f.reports[g.Busy.Machine]; said && r.At != nil && r.At.After(g.Busy.Since) && r.At.Before(now) {
|
||||
end = r.At.UTC()
|
||||
}
|
||||
g.Busy.Ended = &end
|
||||
g.Since, g.Passes, g.LastPass, g.Last, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, "", nil, nil, nil
|
||||
return false
|
||||
}
|
||||
|
||||
// busyOn is the container of a judged module its machine last reported, on what the gate sent it, as left
|
||||
// running because it said busy.
|
||||
func busyOn(j judged, f gateFacts) (inventory.BusyWait, bool) {
|
||||
r, said := f.reports[j.node]
|
||||
if !said || r.At == nil || !f.reportedOn(j.node, r) {
|
||||
return inventory.BusyWait{}, false
|
||||
}
|
||||
return inventory.WaitingFor(r.Busy, j.module)
|
||||
}
|
||||
|
||||
// busyStill is a judged module whose container its machine still lists waiting once the gate stopped waiting
|
||||
// for it: not yet what was sent, so not healthy, whatever its health says of the container still running.
|
||||
func busyStill(j judged, f gateFacts) (string, bool) {
|
||||
b, waits := busyOn(j, f)
|
||||
if !waits {
|
||||
return "", false
|
||||
}
|
||||
return fmt.Sprintf("%s still runs its busy container %s, past its node-engine's bound of %s",
|
||||
j.node, b.Container, b.Until.UTC().Format(time.RFC3339)), true
|
||||
}
|
||||
@@ -1,148 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A walk whose module's container waits as busy on its first machine says so, and starts that machine's
|
||||
// first-machine gate only once the machine reports the container replaced (novox/hq ADR 0335 §4).
|
||||
func TestAGateWaitsForABusyContainerAndStartsWhenItIsReplaced(t *testing.T) {
|
||||
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent,
|
||||
Sent: map[string]inventory.SentDeclaration{"novox": {Digest: "d1", Sequence: 7}}}
|
||||
pairs := []judged{{module: "build-agent", node: "novox"}, {module: "mesh-controller", node: "novox"}}
|
||||
reported := sent.Add(20 * time.Second)
|
||||
waiting := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
||||
State: inventory.BusyWaiting, Since: reported, Until: reported.Add(15 * time.Minute),
|
||||
Said: "busy: novox/mesh-controller (check of #280), in hand since 00:58:00Z"}}
|
||||
facts := func(at time.Time, busy []inventory.BusyWait) gateFacts {
|
||||
// As judgeMoves hands it: the report held against the gate's own send.
|
||||
return gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"novox": {Node: "novox",
|
||||
Outcome: inventory.OutcomeApplied, At: &at, Declared: "d1", ReportedSequence: 7, Busy: busy}}}
|
||||
}
|
||||
|
||||
now := reported.Add(time.Minute)
|
||||
if !busyWait(g, pairs, facts(reported, waiting), now) {
|
||||
t.Fatal("a gate whose module's container waits as busy judged it")
|
||||
}
|
||||
if !g.Since.Equal(sent) || g.Busy == nil || g.Busy.Machine != "novox" || g.Busy.Ended != nil {
|
||||
t.Fatalf("the gate does not keep the wait: %+v %+v", g.Since, g.Busy)
|
||||
}
|
||||
for _, said := range []string{gateLine(g), g.Last} {
|
||||
if !strings.Contains(said, "waits for a busy container on novox") {
|
||||
t.Errorf("the walk does not say it waits for a busy container there: %s", said)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(g.Last, "check of #280") {
|
||||
t.Errorf("the wait does not say what the container holds: %s", g.Last)
|
||||
}
|
||||
|
||||
// Ten minutes on, still busy, within the engine's bound: still waiting, nothing judged, no bound run.
|
||||
now = reported.Add(10 * time.Minute)
|
||||
if !busyWait(g, pairs, facts(reported.Add(9*time.Minute), waiting), now) || g.Passes != 0 {
|
||||
t.Fatalf("the gate stopped waiting within the bound: %+v", g)
|
||||
}
|
||||
|
||||
// The machine reports it replaced: the judging starts at that report.
|
||||
replacedAt := reported.Add(12 * time.Minute)
|
||||
now = replacedAt.Add(5 * time.Second)
|
||||
replaced := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
||||
State: "replaced-when-idle", Since: reported, Until: reported.Add(15 * time.Minute)}}
|
||||
if busyWait(g, pairs, facts(replacedAt, replaced), now) {
|
||||
t.Fatal("the gate still waits after the container was replaced")
|
||||
}
|
||||
if g.Since == nil || !g.Since.Equal(replacedAt) || g.Busy.Ended == nil || !g.Busy.Ended.Equal(replacedAt) {
|
||||
t.Fatalf("the gate does not start at the report that said it replaced: since %v, %+v", g.Since, g.Busy)
|
||||
}
|
||||
if strings.Contains(gateLine(g), "waits for a busy container") {
|
||||
t.Errorf("a gate judging still says it waits: %s", gateLine(g))
|
||||
}
|
||||
// And a later report listing it waiting again does not hold a gate that started.
|
||||
if busyWait(g, pairs, facts(now, waiting), now.Add(time.Minute)) {
|
||||
t.Error("a gate that started waits again")
|
||||
}
|
||||
}
|
||||
|
||||
// An engine that stops reporting while it waits holds no walk for ever: past the engine's bound and the slack,
|
||||
// the judging starts, and finds the module not yet healthy while its container is still listed waiting.
|
||||
func TestAGateWaitsNoLongerThanTheEnginesBound(t *testing.T) {
|
||||
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent}
|
||||
pairs := []judged{{module: "build-agent", node: "novox"}}
|
||||
waiting := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
||||
State: inventory.BusyWaiting, Since: sent, Until: sent.Add(15 * time.Minute)}}
|
||||
f := gateFacts{reports: map[string]inventory.Reported{"novox": {Node: "novox", Outcome: inventory.OutcomeApplied,
|
||||
At: &sent, Current: true, Busy: waiting}}}
|
||||
now := sent.Add(15*time.Minute + busyGateSlack + time.Second)
|
||||
if busyWait(g, pairs, f, now) {
|
||||
t.Fatal("a gate waited past the engine's bound and the slack")
|
||||
}
|
||||
if !g.Busy.PastBound || !g.Since.Equal(now) {
|
||||
t.Fatalf("the gate does not start, said past the bound: %+v %v", g.Busy, g.Since)
|
||||
}
|
||||
if why, still := busyStill(pairs[0], f); !still || !strings.Contains(why, "past its node-engine's bound") {
|
||||
t.Fatalf("a module still running its busy container reads as what was sent: %q", why)
|
||||
}
|
||||
}
|
||||
|
||||
// A report from an engine older than the field lists nothing: the gate starts at the send, as before; and a
|
||||
// report not about the gate's send says nothing of it.
|
||||
func TestAGateOfAnEngineThatNeverWaitsStartsAsBefore(t *testing.T) {
|
||||
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent,
|
||||
Sent: map[string]inventory.SentDeclaration{"ace": {Digest: "d2", Sequence: 9}}}
|
||||
pairs := []judged{{module: "build-agent", node: "ace"}}
|
||||
at := sent.Add(time.Minute)
|
||||
older := gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"ace": {Node: "ace", Outcome: inventory.OutcomeApplied,
|
||||
At: &at, Declared: "d2", ReportedSequence: 9}}}
|
||||
if busyWait(g, pairs, older, at) || g.Busy != nil || !g.Since.Equal(sent) {
|
||||
t.Fatalf("a gate on an engine that never waits did not start at the send: %+v %v", g.Busy, g.Since)
|
||||
}
|
||||
elsewhere := gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"ace": {Node: "ace", Outcome: inventory.OutcomeApplied,
|
||||
At: &at, Declared: "d1", ReportedSequence: 8, Busy: []inventory.BusyWait{{ID: "build-agent.server",
|
||||
State: inventory.BusyWaiting, Until: at.Add(time.Hour)}}}}}
|
||||
if busyWait(g, pairs, elsewhere, at) {
|
||||
t.Fatal("a wait reported about an earlier declaration held the gate")
|
||||
}
|
||||
}
|
||||
|
||||
// The wait is bounded on this controller's clock: an engine whose clock runs an hour ahead, saying it waits until
|
||||
// then, holds the gate no longer than 15 minutes and the slack from when the controller first read the wait.
|
||||
func TestAGateWaitIsBoundedOnTheControllersClock(t *testing.T) {
|
||||
seen := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &seen}
|
||||
pairs := []judged{{module: "build-agent", node: "novox"}}
|
||||
ahead := seen.Add(time.Hour)
|
||||
f := gateFacts{reports: map[string]inventory.Reported{"novox": {Node: "novox", Outcome: inventory.OutcomeApplied,
|
||||
At: &seen, Current: true, Busy: []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
||||
State: inventory.BusyWaiting, Since: ahead, Until: ahead.Add(15 * time.Minute)}}}}}
|
||||
if !busyWait(g, pairs, f, seen) || !g.Busy.Seen.Equal(seen) {
|
||||
t.Fatalf("the gate did not start waiting, seen now: %+v", g.Busy)
|
||||
}
|
||||
if !busyWait(g, pairs, f, seen.Add(15*time.Minute+busyGateSlack-time.Second)) {
|
||||
t.Fatal("the gate stopped waiting within the bound on its own clock")
|
||||
}
|
||||
if busyWait(g, pairs, f, seen.Add(15*time.Minute+busyGateSlack+time.Second)) || !g.Busy.PastBound {
|
||||
t.Fatalf("an engine's later word held the gate past the controller's bound: %+v", g.Busy)
|
||||
}
|
||||
}
|
||||
|
||||
// `plan` says, for a machine whose node-engine is older than the field, each container whose busy is not sent.
|
||||
func TestAPlanSaysABusyAnOlderEngineIsNotSent(t *testing.T) {
|
||||
modules := []catalogue.Manifest{{Module: "build-agent", Resources: []map[string]any{
|
||||
{"id": "server", "type": "container", "busy": map[string]any{"command": "mesh-builder busy"}},
|
||||
{"id": "agent-env", "type": "file"}}}}
|
||||
lines := busyNotSent(modules, false)
|
||||
if len(lines) != 1 || !strings.Contains(lines[0], "build-agent.server declares busy") ||
|
||||
!strings.Contains(lines[0], "replaced at once") {
|
||||
t.Fatalf("the plan does not say the busy not sent: %v", lines)
|
||||
}
|
||||
if lines := busyNotSent(modules, true); len(lines) != 0 {
|
||||
t.Fatalf("a busy sent is said as not sent: %v", lines)
|
||||
}
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCallsIsServedAndSaysWhatItKeeps(t *testing.T) {
|
||||
handlers, behind, err := seatToolHandlers()
|
||||
if err != nil || len(behind) != 0 {
|
||||
t.Fatalf("%v %v", behind, err)
|
||||
}
|
||||
calls, ok := handlers["calls"]
|
||||
if !ok {
|
||||
t.Fatal("calls is not served")
|
||||
}
|
||||
if _, err := calls(context.Background(), json.RawMessage(`{"node":"anchor"}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), `"node"`) {
|
||||
t.Errorf("calls took an argument it does not declare: %v", err)
|
||||
}
|
||||
if _, err := calls(context.Background(), json.RawMessage(`{"call":"call-0-0"}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), "not across a restart") {
|
||||
t.Errorf("an unknown call was not said plainly: %v", err)
|
||||
}
|
||||
got, err := calls(context.Background(), json.RawMessage(`{}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, listed := got.(map[string]any)["calls"]; !listed {
|
||||
t.Errorf("calls answered %v", got)
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,9 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -49,3 +52,28 @@ func TestAPushAnswersBeforeItSends(t *testing.T) {
|
||||
}
|
||||
|
||||
// `calls` is served, takes a call's id and nothing else, and says plainly when it holds no such call.
|
||||
func TestCallsIsServedAndSaysWhatItKeeps(t *testing.T) {
|
||||
handlers, behind, err := seatToolHandlers()
|
||||
if err != nil || len(behind) != 0 {
|
||||
t.Fatalf("%v %v", behind, err)
|
||||
}
|
||||
calls, ok := handlers["calls"]
|
||||
if !ok {
|
||||
t.Fatal("calls is not served")
|
||||
}
|
||||
if _, err := calls(context.Background(), json.RawMessage(`{"node":"anchor"}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), `"node"`) {
|
||||
t.Errorf("calls took an argument it does not declare: %v", err)
|
||||
}
|
||||
if _, err := calls(context.Background(), json.RawMessage(`{"call":"call-0-0"}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), "not across a restart") {
|
||||
t.Errorf("an unknown call was not said plainly: %v", err)
|
||||
}
|
||||
got, err := calls(context.Background(), json.RawMessage(`{}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, listed := got.(map[string]any)["calls"]; !listed {
|
||||
t.Errorf("calls answered %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,12 +2,9 @@ package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
@@ -30,14 +27,6 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
|
||||
for _, e := range entries {
|
||||
byName[e.Manifest.Module] = e
|
||||
}
|
||||
// The machine holding the bus, whose declaration carries every module's grants (novox/hq issue 490).
|
||||
holder := ""
|
||||
for _, e := range entries {
|
||||
if e.Manifest.BusUsers != "" && e.Manifest.ClaimsSeat(catalogue.BrokerSeat) && len(e.On) > 0 {
|
||||
holder = e.On[0]
|
||||
}
|
||||
}
|
||||
var granting []string
|
||||
machines := map[string]*link.MachinePlan{}
|
||||
machine := func(name string) *link.MachinePlan {
|
||||
if machines[name] == nil {
|
||||
@@ -60,24 +49,9 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
|
||||
machine(on).Receives = append(machine(on).Receives, name)
|
||||
}
|
||||
}
|
||||
if !waits && holder != "" && !(len(e.On) == 1 && e.On[0] == holder) && len(e.On) > 0 &&
|
||||
!slices.Contains(granting, name) {
|
||||
granting = append(granting, name)
|
||||
}
|
||||
switch {
|
||||
case (name == "nats" || catalogue.ProvidesBus(e.Manifest)) && len(e.On) > 0:
|
||||
// Said before the merge (novox/hq issue 336): a new bus build holds every send to the bus's machine
|
||||
// until a person takes the step, so merging it is a promise to take it.
|
||||
p.Steps = append(p.Steps, fmt.Sprintf("%s: %s is never sent by an ordinary send, and until %s runs, "+
|
||||
"nothing else is sent to %s either — unless the new build turns out the same as the one running there "+
|
||||
"(issue 280)", busUpgradeNeeded, name, busUpgradeVerb, strings.Join(e.On, ", ")))
|
||||
case name == "nats" || catalogue.ProvidesBus(e.Manifest):
|
||||
case name == "nats":
|
||||
p.Steps = append(p.Steps, "a planned bus step: the bus is upgraded by `bus upgrade`, never by an ordinary send")
|
||||
case waits && len(e.On) > 0 && (u.From == catalogue.FromModule || u.From == catalogue.FromData):
|
||||
// Judged per machine by the walk, against the build each was last sent (novox/hq ADR 0377).
|
||||
p.Steps = append(p.Steps, fmt.Sprintf("%s waits for a person: its policy records (%s) — unless on a machine "+
|
||||
"its move recreates none of its containers and moves none of its data, which the walk judges and then sends",
|
||||
name, orNone(u.From)))
|
||||
case waits && len(e.On) > 0:
|
||||
p.Steps = append(p.Steps, fmt.Sprintf("%s waits for a person: its policy records (%s)", name,
|
||||
orNone(u.From)))
|
||||
@@ -95,13 +69,6 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(granting) > 0 {
|
||||
// Said before the merge (novox/hq issue 490): what the walk does when the change alters the bus's
|
||||
// user list. Whether it does is known only once the builds are registered, so it is said as a rule.
|
||||
p.Steps = append(p.Steps, fmt.Sprintf("%s: if this changes what the bus's user list says (a new tool, "+
|
||||
"subject or user), %s is sent that list alone, every build there kept, before %s is judged on its first "+
|
||||
"machine", grantsStepWord, holder, strings.Join(granting, ", ")))
|
||||
}
|
||||
var names []string
|
||||
for name := range machines {
|
||||
names = append(names, name)
|
||||
@@ -111,39 +78,9 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
|
||||
p.Machines = append(p.Machines, *machines[name])
|
||||
}
|
||||
p.Summary = summaryOf(p)
|
||||
p.Class = changeClassOf(p, entries)
|
||||
return p
|
||||
}
|
||||
|
||||
// classNone is the class of a change that moves no module: nothing for a walk to move (novox/hq ADR 0313 §2).
|
||||
const classNone = "none"
|
||||
|
||||
// changeClassOf is a change plan's class (novox/hq ADR 0313 §2): that of the slowest-budgeted module it moves,
|
||||
// itself or as a dependent — core when one is on the controller's own path or holds the mesh's resolver, leaf
|
||||
// otherwise — and none when it moves nothing. A module it adds a directory for counts by its own class: the
|
||||
// catalogue names a module's directory by the module, so the directory's last element is its name. The graph holds
|
||||
// no manifest of it yet, so a new module is core by its name alone (on the controller's own path), never by a
|
||||
// resolver claim its manifest makes.
|
||||
func changeClassOf(p link.ChangePlan, entries []inventory.Entry) string {
|
||||
if len(p.Moved) == 0 && len(p.Dependents) == 0 && len(p.New) == 0 {
|
||||
return classNone
|
||||
}
|
||||
modules := append(append([]string{}, p.Moved...), p.Dependents...)
|
||||
for _, dir := range p.New {
|
||||
if name := path.Base(strings.Trim(dir, "/")); name != "." && name != "/" && name != "" {
|
||||
modules = append(modules, name)
|
||||
}
|
||||
}
|
||||
return classOfModules(modules, entries)
|
||||
}
|
||||
|
||||
// grantsStepWord names the grants step (novox/hq issue 490): the bus's user list sent alone to the machine
|
||||
// holding the bus ahead of a walk's gate. Not the bus step, which replaces the bus itself.
|
||||
const grantsStepWord = "grants step"
|
||||
|
||||
// busUpgradeNeeded is how a change plan says that merging it holds the bus's machine for a person's step.
|
||||
const busUpgradeNeeded = "merging this needs a person's bus upgrade"
|
||||
|
||||
// summaryOf is a change plan in one line: what it builds, where it goes, and whether the bus moves.
|
||||
func summaryOf(p link.ChangePlan) string {
|
||||
if len(p.Moved) == 0 && len(p.New) == 0 {
|
||||
@@ -173,10 +110,7 @@ func summaryOf(p link.ChangePlan) string {
|
||||
}
|
||||
bus := "no bus step"
|
||||
for _, s := range p.Steps {
|
||||
switch {
|
||||
case strings.HasPrefix(s, busUpgradeNeeded):
|
||||
bus = busUpgradeNeeded
|
||||
case strings.HasPrefix(s, "a planned bus step") && bus == "no bus step":
|
||||
if strings.HasPrefix(s, "a planned bus step") {
|
||||
bus = "a bus step"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -45,7 +45,7 @@ func TestAChangePlanSaysWhatEachMachineReceives(t *testing.T) {
|
||||
}
|
||||
|
||||
p = plan("modules/nats/Dockerfile", "modules/photos/x.js")
|
||||
if !strings.Contains(p.Summary, "needs a person's bus upgrade") || !strings.Contains(p.Summary, "2 wait(s) for a person") ||
|
||||
if !strings.Contains(p.Summary, "a bus step") || !strings.Contains(p.Summary, "2 wait(s) for a person") ||
|
||||
!strings.Contains(p.Summary, "(+1 dependent(s))") {
|
||||
t.Errorf("the bus and a held module read %q", p.Summary)
|
||||
}
|
||||
@@ -58,11 +58,7 @@ func TestAChangePlanSaysWhatEachMachineReceives(t *testing.T) {
|
||||
t.Errorf("the deploy plan reads %v", got)
|
||||
}
|
||||
text := strings.Join(p.Steps, "\n")
|
||||
// Said before the merge (novox/hq issue 336): merging it holds every send to the bus's machine until a
|
||||
// person runs the bus's step, and the verb that does.
|
||||
for _, want := range []string{"merging this needs a person's bus upgrade", "nothing else is sent to anchor",
|
||||
"mesh_call mesh-controller.bus", "the same as the one running there", "photos waits for a person",
|
||||
"nats provides mesh-bus"} {
|
||||
for _, want := range []string{"a planned bus step", "photos waits for a person", "nats provides mesh-bus"} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Errorf("the steps do not say %q:\n%s", want, text)
|
||||
}
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// moduleCheck judges manifests where they are written, with no mesh (novox/hq ADR 0037, issue 148).
|
||||
@@ -33,9 +32,6 @@ import (
|
||||
// provision it wants that a manifest given here offers. An overflow is refused in the pull request
|
||||
// that introduces it — a new requirement, a lowered bound, a longer slug — instead of on the
|
||||
// provider's machine when a real machine's name first meets the module's.
|
||||
// SomeManifestsVar, set by the merge gate, says the manifests given are only some of their repository's.
|
||||
const SomeManifestsVar = "MESH_MODULE_CHECK_SOME"
|
||||
|
||||
func moduleCheck(paths []string, out io.Writer) error {
|
||||
return moduleCheckFor(paths, catalogue.DefaultLongestMachine, out)
|
||||
}
|
||||
@@ -48,24 +44,6 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
shelf := catalogue.Shelf{}
|
||||
faulted := map[string]bool{}
|
||||
failed := 0
|
||||
// The list of predefined keys at the root of the repository above each manifest (novox/hq ADR 0348 §7), read
|
||||
// once per repository: a manifest naming a key is judged against the list its own tree holds.
|
||||
lists := map[string]catalogue.PredefinedList{}
|
||||
listOf := func(path string) (catalogue.PredefinedList, error) {
|
||||
root := catalogue.RepositoryRoot(filepath.Dir(path))
|
||||
if root == "" {
|
||||
return nil, nil
|
||||
}
|
||||
if list, read := lists[root]; read {
|
||||
return list, nil
|
||||
}
|
||||
list, err := catalogue.ReadPredefined(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lists[root] = list
|
||||
return list, nil
|
||||
}
|
||||
for _, path := range paths {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
@@ -73,18 +51,7 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
list, err := listOf(path)
|
||||
if err != nil {
|
||||
fmt.Fprintf(out, "%s: %v\n", path, err)
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
m, err := catalogue.ParseManifestWith(raw, list)
|
||||
if err == nil {
|
||||
// Each template's text read from the tree beside the manifest, and judged as inline content is
|
||||
// (novox/hq ADR 0349 §2): a missing template is refused by name.
|
||||
m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(path))
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
fmt.Fprintf(out, "%s: %v\n", path, err)
|
||||
failed++
|
||||
@@ -98,37 +65,6 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
}
|
||||
// A definition names no installation (novox/hq ADR 0112, ADR 0155): judged here, in the
|
||||
// catalogue-wide test, and at registration, which refuses in the same words.
|
||||
if wrong := catalogue.TrustProblems(m); len(wrong) > 0 {
|
||||
for _, p := range wrong {
|
||||
fmt.Fprintf(out, "%s: %s\n", path, p)
|
||||
}
|
||||
failed += len(wrong)
|
||||
faulted[m.Module] = true
|
||||
}
|
||||
// A module that declares a setting names the keys of its mergeable files (novox/hq ADR 0326 §3,
|
||||
// issue 574): judged where a definition is written, as at registration, never where a stored
|
||||
// build is read again.
|
||||
if wrong := catalogue.WrittenSettingsProblems(m); len(wrong) > 0 {
|
||||
for _, p := range wrong {
|
||||
fmt.Fprintf(out, "%s: %s\n", path, p)
|
||||
}
|
||||
failed += len(wrong)
|
||||
faulted[m.Module] = true
|
||||
}
|
||||
// A module lives whole in one repository (novox/hq ADR 0420 §1, issue 1019): when the check is told the
|
||||
// repository it runs for (MESH_CHECK_REPOSITORY, set by the build's check), an artifact building from
|
||||
// another is refused, as the merge gate refuses it. A run by hand names no repository and judges none.
|
||||
if repo := os.Getenv("MESH_CHECK_REPOSITORY"); repo != "" && m.Build != nil {
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Context != nil && a.Context.Repository != "" && !sameRepository(a.Context.Repository,
|
||||
link.SourceMoved{Owner: ownerOf(repo), Repo: repoOf(repo)}) {
|
||||
fmt.Fprintf(out, "%s: %s builds artifact %s from %s: a module lives whole in one repository "+
|
||||
"(novox/hq ADR 0420 §1)\n", path, m.Module, a.Name, a.Context.Repository)
|
||||
failed++
|
||||
faulted[m.Module] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if named := catalogue.InstallationProblems(m); len(named) > 0 {
|
||||
for _, p := range named {
|
||||
fmt.Fprintf(out, "%s: %s\n", path, p)
|
||||
@@ -142,24 +78,12 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
// Between the manifests: a seat declared twice, a use of a seat nothing declares, a claim on
|
||||
// a seat that does not exist. Run only over what parsed, because a problem inside one manifest
|
||||
// has already been said and would be said again here in a worse form.
|
||||
//
|
||||
// **Over some of a repository's manifests, a seat none of them declares is a note** (novox/hq issue 364), as
|
||||
// this command's own word says above: the merge gate passes only the manifests a change touches, and says so
|
||||
// with SomeManifestsVar, so a module that uses or claims a seat another module declares (the operator
|
||||
// channel's, a channel bench) was refused there for a manifest it was not given. Given every manifest — the
|
||||
// catalogue's own check, and registration — it stays a refusal.
|
||||
some := os.Getenv(SomeManifestsVar) != ""
|
||||
problems := catalogue.CatalogueProblems(shelf)
|
||||
sort.Strings(problems)
|
||||
for _, p := range problems {
|
||||
if some && catalogue.IsUndeclaredSeat(p) {
|
||||
fmt.Fprintf(out, "note: %s among the manifests given; registration judges it against the whole catalogue, "+
|
||||
"and passing the declaring module's manifest too judges it here\n", p)
|
||||
continue
|
||||
}
|
||||
fmt.Fprintln(out, p)
|
||||
failed++
|
||||
}
|
||||
failed += len(problems)
|
||||
|
||||
// Between the manifests too: an identity against the bounds of the provisions it wants, which
|
||||
// only the provider's manifest states.
|
||||
@@ -206,13 +130,6 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
}
|
||||
}
|
||||
|
||||
// **A file that asks for a setting and does not say whether it is trusted counts as trusted** (novox/hq issue
|
||||
// 339): listed and counted, never refused, so an author can opt out a file nothing trusts.
|
||||
unsaid := 0
|
||||
for _, name := range names {
|
||||
unsaid += len(catalogue.UnsaidTrust(shelf[name]))
|
||||
}
|
||||
|
||||
for _, name := range names {
|
||||
m := shelf[name]
|
||||
if faulted[name] {
|
||||
@@ -233,14 +150,6 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
}
|
||||
fmt.Fprintf(out, ", keeps state %s", strings.Join(kept, ", "))
|
||||
}
|
||||
// And the logs it keeps, with their caps (novox/hq ADR 0297).
|
||||
if len(m.Logs) > 0 {
|
||||
kept := make([]string, 0, len(m.Logs))
|
||||
for _, l := range m.Logs {
|
||||
kept = append(kept, fmt.Sprintf("%s (%d MiB)", l.Name, l.Cap()))
|
||||
}
|
||||
fmt.Fprintf(out, ", keeps log %s", strings.Join(kept, ", "))
|
||||
}
|
||||
if len(m.Reads) > 0 {
|
||||
fmt.Fprintf(out, ", reads %s", strings.Join(m.Reads, ", "))
|
||||
}
|
||||
@@ -259,20 +168,9 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
checks = append(checks, fmt.Sprintf("%v by %s", r["id"], catalogue.HealthWords(h)))
|
||||
}
|
||||
}
|
||||
// And its own runtime's, where it runs as its own account (novox/hq ADR 0366).
|
||||
if own, has := catalogue.OwnRuntimeHealth(m); has {
|
||||
if h, _, _ := catalogue.ReadHealth(own); h.Kind != "" {
|
||||
checks = append(checks, fmt.Sprintf("%v by %s", own["id"], catalogue.HealthWords(h)))
|
||||
}
|
||||
}
|
||||
if len(checks) > 0 {
|
||||
fmt.Fprintf(out, ", ready: %s", strings.Join(checks, "; "))
|
||||
}
|
||||
if missing := catalogue.UnsaidTrust(m); len(missing) > 0 {
|
||||
fmt.Fprintf(out, "; WARNING: %s ask(s) for a setting and do(es) not say whether it is trusted, so it counts as "+
|
||||
"trusted: set at the terminal alone; say %q false where nothing trusts it (novox/hq issue 339)",
|
||||
strings.Join(missing, ", "), catalogue.TrustedField)
|
||||
}
|
||||
if missing := catalogue.Undeclared(m); len(missing) > 0 {
|
||||
fmt.Fprintf(out, "; WARNING: %s stay(s) up and say(s) not how it is ready — judged by liveness alone, "+
|
||||
"refused from %s (ADR 0240 rule 8)", strings.Join(missing, ", "), catalogue.HealthRequiredFrom.Format("2006-01-02"))
|
||||
@@ -281,7 +179,6 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
}
|
||||
// The count the catalogue keeps (ADR 0240 rule 8), in a line its merge check reads.
|
||||
fmt.Fprintf(out, "%s %d\n", UndeclaredHealthLine, undeclared)
|
||||
fmt.Fprintf(out, "%s %d\n", UnsaidTrustLine, unsaid)
|
||||
if failed > 0 {
|
||||
return fmt.Errorf("%d problem(s) in %d manifest(s)", failed, len(paths))
|
||||
}
|
||||
@@ -296,10 +193,6 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
// `health` in, over the manifests given: the catalogue's merge check compares it with the number it keeps.
|
||||
const UndeclaredHealthLine = "long-running resources without health:"
|
||||
|
||||
// UnsaidTrustLine starts the line `module check` says the count of files that ask for a setting and do not say
|
||||
// whether it is trusted, and so count as trusted (novox/hq issue 339).
|
||||
const UnsaidTrustLine = "files asking for a setting without saying whether it is trusted:"
|
||||
|
||||
// checkNow is the clock `module check` judges the date by; a test sets it.
|
||||
var checkNow = time.Now
|
||||
|
||||
|
||||
@@ -1,87 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A pull request's check builds go when it closes (novox/hq ADR 0350 §5): the build seat keeps them in the artifact
|
||||
// store's scratch namespace for the walk of its merge to take. **Let go of where the moment is heard**: the
|
||||
// controller ends the walk answering a merge — after that walk took what it reuses — and the delivery's owner, which
|
||||
// hears the forge say a pull request closed unmerged, asks the `forget-checks` verb; both let go through the store,
|
||||
// as the controller's own collection does. The build seat's holders let go of anything a week old on their own,
|
||||
// so a word lost here leaves a week's builds, never more.
|
||||
|
||||
// checkBuildsIn is the scratch namespace as the controller reaches the artifact store.
|
||||
func checkBuildsIn(ctx context.Context, inv *inventory.Inventory) (builder.Scratch, error) {
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
address, err := artifactStoreAddress(ctx, inv, shelf, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if address == "" {
|
||||
return nil, errors.New("the artifact store is not on the mesh's network")
|
||||
}
|
||||
return builder.Registry{Address: address}, nil
|
||||
}
|
||||
|
||||
// forgetChecks lets go of a pull request's check builds and answers what it did.
|
||||
var forgetChecks = func(ctx context.Context, inv *inventory.Inventory, pull builder.PullRef) (string, error) {
|
||||
scratch, err := checkBuildsIn(ctx, inv)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
n, err := scratch.Forget(ctx, pull)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s: %d check(s)' builds let go of, and not all: %w", pull, n, err)
|
||||
}
|
||||
return fmt.Sprintf("%s: %d check(s)' builds let go of", pull, n), nil
|
||||
}
|
||||
|
||||
// forgetChecksOfWalk lets go, on the save that ends a walk, of the check builds of every pull request whose merge
|
||||
// it answers: the walk took what it reuses. Said, never fatal.
|
||||
func forgetChecksOfWalk(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) {
|
||||
if p.WalkSaid != inventory.WalkEnded || p.Delivery == nil {
|
||||
return
|
||||
}
|
||||
for _, pull := range reuseAskOf(&p).Pulls {
|
||||
said, err := forgetChecks(ctx, inv, builder.PullRef{Owner: pull.Owner, Repo: pull.Repo, Number: pull.Number})
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: its check builds stay until the week's sweep: %v\n", p.ID, err)
|
||||
continue
|
||||
}
|
||||
fmt.Printf("%s: %s\n", p.ID, said)
|
||||
}
|
||||
}
|
||||
|
||||
// forgetChecksCommand is `forget-checks <owner> <repo> <number>`: the verb the delivery's owner asks when the forge
|
||||
// says a pull request closed unmerged (novox/hq ADR 0350 §5).
|
||||
func forgetChecksCommand(ctx context.Context, args []string) error {
|
||||
if len(args) != 3 {
|
||||
return errors.New("forget-checks <owner> <repo> <number>")
|
||||
}
|
||||
number, err := strconv.Atoi(args[2])
|
||||
if err != nil || number <= 0 {
|
||||
return fmt.Errorf("%q is not a pull request's number", args[2])
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
said, err := forgetChecks(ctx, open.inventory, builder.PullRef{Owner: args[0], Repo: args[1], Number: number})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(said)
|
||||
return nil
|
||||
}
|
||||
@@ -1,102 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func TestOnlyAWalksAskTakesAChecksBuildAndADryRunNever(t *testing.T) {
|
||||
p := &inventory.Plan{Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{
|
||||
{Repository: "novox/shop", Number: 7}, {Repository: "http://forge.invalid/novox/web.git", Number: 3},
|
||||
{Repository: "novox/shop", Number: 7}, {Repository: "novox/shop"}}}}
|
||||
asked := reuseAskOf(p)
|
||||
if want := []link.PullRef{{Owner: "novox", Repo: "shop", Number: 7}, {Owner: "novox", Repo: "web", Number: 3}}; !reflect.DeepEqual(asked.Pulls, want) {
|
||||
t.Fatalf("the walk names %+v", asked.Pulls)
|
||||
}
|
||||
walk := withReuse(t.Context(), asked)
|
||||
if reuseAsked(walk, false) != asked {
|
||||
t.Fatal("a walk's ask does not carry its leave")
|
||||
}
|
||||
if reuseAsked(walk, true) != nil {
|
||||
t.Fatal("a dry run carries a leave to take a check's build")
|
||||
}
|
||||
if reuseAsked(t.Context(), false) != nil {
|
||||
t.Fatal("an ask that is not a walk's carries a leave")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := g.open.inventory
|
||||
gateEvery = 0
|
||||
|
||||
digest := "sha256:" + strings.Repeat("c", 64)
|
||||
reused := func(id, reference string) link.BuildResult {
|
||||
manifest := mustJSON(t, catalogue.Manifest{Module: "app", Version: "c3", Resources: []map[string]any{
|
||||
{"id": "svc", "type": "container", "name": "app", "image": reference}}})
|
||||
return link.BuildResult{ID: id, Repository: "novox/mesh-catalog", Path: "modules/app", Ref: "main", On: "anchor",
|
||||
Commit: "c3", Manifest: manifest, Trunk: "main", OnTrunk: true, Branches: []string{"main"},
|
||||
Made: []link.MadeArtifact{{Name: "server", Kind: "image", Reference: reference}},
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"},
|
||||
Reused: "reused from the check of novox/mesh-catalog#9 at head abcdefabcdef (merged into trunk 123456123456): server " + digest,
|
||||
ReusedFrom: &link.ReusedFrom{Pull: link.PullRef{Owner: "novox", Repo: "mesh-catalog", Number: 9},
|
||||
Head: "abcdefabcdef", Trunk: "123456123456", Key: "r1:" + strings.Repeat("1", 64)}}
|
||||
}
|
||||
if _, _, err := takeIn(ctx, inv, reused(link.NewBuildID(time.Now()), "store.internal:5000/scratch/app/server@"+digest)); err == nil ||
|
||||
!strings.Contains(err.Error(), "scratch") {
|
||||
t.Fatalf("a build in the scratch namespace was registered: %v", err)
|
||||
}
|
||||
id := link.NewBuildID(time.Now().Add(time.Second))
|
||||
result := reused(id, "store.internal:5000/app/server@"+digest)
|
||||
if _, _, err := takeIn(ctx, inv, result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil || current["app"].Commit != "c3" {
|
||||
t.Fatalf("the reused build is not what app is registered at: %+v %v", current["app"], err)
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(shelf["app"].Resources)
|
||||
if !strings.Contains(string(raw), "app/server@"+digest) {
|
||||
t.Fatalf("app is registered as %s, not by the check's digest in its own namespace", raw)
|
||||
}
|
||||
|
||||
// The walk asked it: the walk keeps what its build did, and its gate judges the registered build.
|
||||
asked := time.Now().UTC()
|
||||
p := g.plan(t)
|
||||
p.Commit = "c3"
|
||||
p.Modules["app"] = &inventory.PlanModule{State: "asked", AskedAt: &asked, Build: id}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
planBuiltSaying(ctx, g.open, "app", "c3", "", asked, id, result.MadeWords())
|
||||
p = g.plan(t)
|
||||
if m := p.Modules["app"]; m.State != "built" || !strings.HasPrefix(m.Made, "reused from the check of novox/mesh-catalog#9") {
|
||||
t.Fatalf("the walk keeps %+v", m)
|
||||
}
|
||||
for i := 0; i < 6; i++ {
|
||||
advancePlans(ctx, g.open)
|
||||
}
|
||||
p = g.plan(t)
|
||||
gate := p.Modules["app"].Gate
|
||||
if gate == nil || gate.To != "c3" || gate.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("the gate judged %+v, not the reused build", gate)
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c3" {
|
||||
t.Fatalf("the first machine was sent %v, not the reused build", sent)
|
||||
}
|
||||
}
|
||||
@@ -1,193 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// One build per change (novox/hq ADR 0350), as the controller asks it and takes it in.
|
||||
|
||||
// The check of a change is asked to build what a merge of it would move, in the walk's order: a module of the
|
||||
// repository checked from the merge, one standing on it from its own repository as the walk builds it, and one
|
||||
// that reads the repository through a context not at all. A change that moves no module builds nothing.
|
||||
func TestACheckIsAskedToBuildWhatTheMergeWouldMove(t *testing.T) {
|
||||
entries := []inventory.Entry{
|
||||
{Manifest: catalogue.Manifest{Module: "app"}, Source: inventory.Source{Repository: "novox/shop", Seat: "git", Path: "modules/app"}},
|
||||
{Manifest: catalogue.Manifest{Module: "web"}, Source: inventory.Source{Repository: "novox/web", Seat: "git", Ref: "main"}},
|
||||
{Manifest: catalogue.Manifest{Module: "docs"}, Source: inventory.Source{Repository: "novox/docs", Seat: "git"}},
|
||||
}
|
||||
p := link.PullUpdated{Owner: "novox", Repo: "shop", Number: 7, Base: "main", Commit: "abc"}
|
||||
scope := checkScope{Reach: mergeReach{Packaging: entries[2:], Plan: inventory.Plan{Tiers: [][]string{{"app", "docs"}, {"web"}}}}}
|
||||
clone := func(s inventory.Source) (string, error) { return "http://forge.invalid/" + s.Repository + ".git", nil }
|
||||
|
||||
got := checkBuildsOf(p, scope, entries, clone)
|
||||
want := []link.CheckBuild{
|
||||
{Module: "app", Path: "modules/app", Merged: true, Tier: 0},
|
||||
{Module: "docs", Tier: 0, Skip: "its build reads novox/shop through a context, which a check cannot hand the merge"},
|
||||
{Module: "web", Repository: "http://forge.invalid/novox/web.git", Ref: "main", Tier: 1},
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("the check is asked to build\n%+v\nwant\n%+v", got, want)
|
||||
}
|
||||
if got := checkBuildsOf(p, checkScope{}, entries, clone); got != nil {
|
||||
t.Fatalf("a change that moves no module asks for builds: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A walk's ask carries its leave to take a check's build, naming the pull requests of the merges it answers; a dry
|
||||
// run never does, and neither does an ask that is not a walk's.
|
||||
|
||||
// A walk ended lets go of its merges' check builds — only once it ended, after it took what it reuses.
|
||||
func TestAWalkEndedLetsGoOfItsMergesCheckBuilds(t *testing.T) {
|
||||
var forgot []builder.PullRef
|
||||
was := forgetChecks
|
||||
forgetChecks = func(_ context.Context, _ *inventory.Inventory, pull builder.PullRef) (string, error) {
|
||||
forgot = append(forgot, pull)
|
||||
return "", nil
|
||||
}
|
||||
t.Cleanup(func() { forgetChecks = was })
|
||||
p := inventory.Plan{ID: "plan-1", Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{
|
||||
{Repository: "novox/shop", Number: 7}}}}
|
||||
|
||||
p.WalkSaid = inventory.WalkStarted
|
||||
forgetChecksOfWalk(t.Context(), nil, p)
|
||||
p.WalkSaid = ""
|
||||
forgetChecksOfWalk(t.Context(), nil, p)
|
||||
if len(forgot) != 0 {
|
||||
t.Fatalf("a walk not ended let go of %v, which it may still take", forgot)
|
||||
}
|
||||
p.WalkSaid = inventory.WalkEnded
|
||||
forgetChecksOfWalk(t.Context(), nil, p)
|
||||
if want := []builder.PullRef{{Owner: "novox", Repo: "shop", Number: 7}}; !reflect.DeepEqual(forgot, want) {
|
||||
t.Fatalf("the walk ended let go of %v", forgot)
|
||||
}
|
||||
}
|
||||
|
||||
// A build in the scratch namespace is never registered: only the walk's copy in the module's own namespace is.
|
||||
func TestABuildInTheScratchNamespaceIsNeverRegistered(t *testing.T) {
|
||||
if got := inScratch([]link.MadeArtifact{{Name: "server", Reference: "store.internal:5000/app/server@sha256:aa"}}); got != "" {
|
||||
t.Fatalf("a module's own build reads as a scratch build: %s", got)
|
||||
}
|
||||
scratch := "store.internal:5000/scratch/app/server@sha256:aa"
|
||||
if got := inScratch([]link.MadeArtifact{{Name: "server", Reference: scratch}}); got != scratch {
|
||||
t.Fatalf("a scratch build was not found: %q", got)
|
||||
}
|
||||
if got := inScratch([]link.MadeArtifact{{Name: "look", Reference: "http://store.internal:5000/v2/scratch/app/look/blobs/sha256:bb"}}); got == "" {
|
||||
t.Fatal("a scratch archive was not found")
|
||||
}
|
||||
}
|
||||
|
||||
// The walk says per module what it did: "reused from …" or "built: <why>" — and a build seat from before says
|
||||
// neither, which is said.
|
||||
func TestAWalksBuildSaysWhetherItReusedOrBuilt(t *testing.T) {
|
||||
if got := (link.BuildResult{Reused: "reused from the check of novox/shop#7"}).MadeWords(); got != "reused from the check of novox/shop#7" {
|
||||
t.Fatalf("said %q", got)
|
||||
}
|
||||
if got := (link.BuildResult{BuiltWhy: "no check built it"}).MadeWords(); got != "built: no check built it" {
|
||||
t.Fatalf("said %q", got)
|
||||
}
|
||||
if got := (link.BuildResult{}).MadeWords(); !strings.Contains(got, "predates ADR 0350") {
|
||||
t.Fatalf("said %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A walk's module the check's build stood in for is registered by the digest the check built — the copy in the
|
||||
// module's namespace — and the walk keeps "reused from …" for it; its first-node gate judges that registered
|
||||
// build as it judges one the walk built. A scratch reference is refused. Needs the store (MESH_TEST_POSTGRES), and
|
||||
// is named in testdata/store-tests.
|
||||
|
||||
// A walk's leave names the builds its own pull requests' verdicts made at the head each merge merged — never an
|
||||
// earlier head's — read once per walk, and none when they cannot be read: the builder takes nothing a verdict does
|
||||
// not name (novox/hq ADR 0350 §2, issue 703).
|
||||
func TestAWalksLeaveNamesTheBuildsItsPullRequestsVerdictsMade(t *testing.T) {
|
||||
was := checkedBuildsOf
|
||||
forgetWalkVerdicts()
|
||||
t.Cleanup(func() { checkedBuildsOf = was; forgetWalkVerdicts() })
|
||||
var asked [][]link.PullRef
|
||||
pull := link.PullRef{Owner: "novox", Repo: "shop", Number: 7}
|
||||
atHead := link.CheckedBuild{Pull: pull, Check: "build-2", Head: "bbbbbbbbbbbb", Built: link.CheckBuilt{Module: "app", Key: "r1:2"}}
|
||||
earlier := link.CheckedBuild{Pull: pull, Check: "build-1", Head: "aaaaaaaaaaaa", Built: link.CheckBuilt{Module: "app", Key: "r1:1"}}
|
||||
checkedBuildsOf = func(_ context.Context, pulls []link.PullRef) ([]link.CheckedBuild, error) {
|
||||
asked = append(asked, pulls)
|
||||
return []link.CheckedBuild{atHead, earlier}, nil
|
||||
}
|
||||
pulls := []link.PullRef{pull}
|
||||
heads := map[link.PullRef]string{pull: "bbbbbbbbbbbb0123456789"}
|
||||
for range 3 {
|
||||
if got := reuseWithVerdicts(t.Context(), "plan-1", &link.ReuseAsk{Pulls: pulls}, heads); !reflect.DeepEqual(got.Builds,
|
||||
[]link.CheckedBuild{atHead}) {
|
||||
t.Fatalf("the leave names %+v, not only the merged head's build", got.Builds)
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(asked, [][]link.PullRef{pulls}) {
|
||||
t.Fatalf("the verdicts were read %d times for one walk: %v", len(asked), asked)
|
||||
}
|
||||
// A merge whose head was not said takes nothing.
|
||||
if got := reuseWithVerdicts(t.Context(), "plan-1", &link.ReuseAsk{Pulls: pulls}, nil); got.Builds != nil {
|
||||
t.Fatalf("a merge of no said head names %+v", got.Builds)
|
||||
}
|
||||
checkedBuildsOf = func(context.Context, []link.PullRef) ([]link.CheckedBuild, error) {
|
||||
return nil, errors.New("no bus")
|
||||
}
|
||||
if got := reuseWithVerdicts(t.Context(), "plan-2", &link.ReuseAsk{Pulls: pulls}, heads); got.Builds != nil {
|
||||
t.Fatalf("verdicts that could not be read named %+v", got.Builds)
|
||||
}
|
||||
}
|
||||
|
||||
// A check is handed only the builds earlier checks of its own pull request at its own head made: another pull
|
||||
// request's build, or an earlier head's, is never kept (novox/hq issue 703). None when they cannot be read.
|
||||
func TestACheckIsHandedOnlyItsOwnHeadsBuilds(t *testing.T) {
|
||||
was := checkedBuildsOf
|
||||
t.Cleanup(func() { checkedBuildsOf = was })
|
||||
pull := link.PullRef{Owner: "novox", Repo: "shop", Number: 7}
|
||||
mine := link.CheckedBuild{Pull: pull, Check: "build-1", Head: "bbbbbbbbbbbb", Built: link.CheckBuilt{Module: "app", Key: "r1:1"}}
|
||||
earlier := link.CheckedBuild{Pull: pull, Check: "build-0", Head: "aaaaaaaaaaaa", Built: link.CheckBuilt{Module: "app", Key: "r1:1"}}
|
||||
another := link.CheckedBuild{Pull: link.PullRef{Owner: "novox", Repo: "shop", Number: 8}, Check: "build-9",
|
||||
Head: "bbbbbbbbbbbb", Built: link.CheckBuilt{Module: "app", Key: "r1:1"}}
|
||||
var asked [][]link.PullRef
|
||||
checkedBuildsOf = func(_ context.Context, pulls []link.PullRef) ([]link.CheckedBuild, error) {
|
||||
asked = append(asked, pulls)
|
||||
return []link.CheckedBuild{mine, earlier, another}, nil
|
||||
}
|
||||
p := link.PullUpdated{Owner: "novox", Repo: "shop", Number: 7, Commit: "bbbbbbbbbbbb0123456789"}
|
||||
builds := []link.CheckBuild{{Module: "app"}}
|
||||
if got := namedForTheCheck(t.Context(), p, builds); !reflect.DeepEqual(got, []link.CheckedBuild{mine}) ||
|
||||
!reflect.DeepEqual(asked, [][]link.PullRef{{pull}}) {
|
||||
t.Fatalf("the check is handed %+v, read for %v", got, asked)
|
||||
}
|
||||
checkedBuildsOf = func(context.Context, []link.PullRef) ([]link.CheckedBuild, error) {
|
||||
return nil, errors.New("no bus")
|
||||
}
|
||||
if got := namedForTheCheck(t.Context(), p, builds); got != nil {
|
||||
t.Fatalf("verdicts that could not be read named %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func forgetWalkVerdicts() {
|
||||
walkVerdicts.Lock()
|
||||
defer walkVerdicts.Unlock()
|
||||
walkVerdicts.read = map[string]verdictsRead{}
|
||||
}
|
||||
|
||||
// A walk's merged heads are the forge's word for each merge; a merge announced without its head, or a pull request
|
||||
// merged at two heads in one walk, has none, and its walk takes no check's build (novox/hq issue 703).
|
||||
func TestAWalksMergedHeadsAreTheForgesWord(t *testing.T) {
|
||||
merge := func(number int, head string) inventory.PlanMerge {
|
||||
return inventory.PlanMerge{Repository: "novox/shop", Number: number, Head: head}
|
||||
}
|
||||
p := &inventory.Plan{Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{
|
||||
merge(7, "aaaaaaaaaaaa1"), merge(8, ""), merge(9, "bbbbbbbbbbbb"), merge(9, "cccccccccccc")}}}
|
||||
heads := mergedHeadsOf(p)
|
||||
shop := func(n int) link.PullRef { return link.PullRef{Owner: "novox", Repo: "shop", Number: n} }
|
||||
if heads[shop(7)] != "aaaaaaaaaaaa1" || heads[shop(8)] != "" || heads[shop(9)] != "" {
|
||||
t.Fatalf("merged heads %v", heads)
|
||||
}
|
||||
}
|
||||
@@ -82,9 +82,7 @@ func checkHereCommand(ctx context.Context, args []string) error {
|
||||
if _, err := git("fetch", "--quiet", "origin", *base); err != nil {
|
||||
return fmt.Errorf("cannot fetch %s to say what the change touches: %w", *base, err)
|
||||
}
|
||||
// Without rename detection, so a file moved out of a build source is said under its old name as well:
|
||||
// its going is a change to the build that held it (novox/hq ADR 0267).
|
||||
changedText, err := git("diff", "--name-only", "--no-renames", "origin/"+*base+"...HEAD")
|
||||
changedText, err := git("diff", "--name-only", "origin/"+*base+"...HEAD")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1,165 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A delivery group's member is checked beside the other members' heads** (novox/hq ADR 0382, issue 867).
|
||||
//
|
||||
// A mesh-host change and a mesh-controller change that need each other could not both pass their own merge checks:
|
||||
// each was read beside the other's repository at the commit the mesh runs, which holds neither change. So the
|
||||
// controller asks mesh-delivery's `groups` once for every pull request's check it asks (§1), and names, in the ask,
|
||||
// the group's other open pull requests into a base of the same name, in a repository the build seat reads beside the
|
||||
// check — mesh-lab never — each with its whole head. The build seat reads those repositories at the sibling's merge (§2).
|
||||
// A pull request for which mesh-delivery answered and named no sibling is asked exactly as before (§7).
|
||||
|
||||
// wholeCommit is a commit as the forge says a head: forty hexadecimal characters.
|
||||
var wholeCommit = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
||||
|
||||
// besideRepositories are the repositories (lower case, owner/repo) a check reads beside its own, by the directory each
|
||||
// is found under — the core's, as besideOf clones them, and mesh-sdk beside the controller; never mesh-lab, which is
|
||||
// always read at its trunk.
|
||||
func besideRepositories(entries []inventory.Entry) map[string]string {
|
||||
read := map[string]string{}
|
||||
for _, e := range entries {
|
||||
dir, core := coreModules[e.Manifest.Module]
|
||||
if !core || e.Provided || e.Source.Repository == "" {
|
||||
continue
|
||||
}
|
||||
read[nameOf(e.Source.Repository)] = dir
|
||||
if dir == "mesh-controller" && e.Source.Seat != "" {
|
||||
read[nameOf(siblingOf(e.Source.Repository, "mesh-sdk"))] = "mesh-sdk"
|
||||
}
|
||||
}
|
||||
delete(read, "")
|
||||
return read
|
||||
}
|
||||
|
||||
// nameOf is a recorded repository as owner/repo in lower case, empty when it names no owner.
|
||||
func nameOf(repository string) string {
|
||||
owner, repo := sourceOwner(repository), sourceName(repository)
|
||||
if owner == "" || repo == "" {
|
||||
return ""
|
||||
}
|
||||
return strings.ToLower(owner + "/" + repo)
|
||||
}
|
||||
|
||||
// siblingsFor is the pull request's siblings, as mesh-delivery's groups say them (ADR 0382 §1): the other members of
|
||||
// the group its head branch names that are open, unmerged and into a base of its own base's name, in a repository the
|
||||
// check reads beside it (beside: the directories the ask already names). The second answer is why they could not be
|
||||
// learned, empty when they could: the group's members carry no head or base — a mesh-delivery from before the rule.
|
||||
func siblingsFor(p link.PullUpdated, entries []inventory.Entry, beside map[string]link.CheckedOut,
|
||||
lines []groupsLine) ([]link.CheckSibling, string) {
|
||||
own := strings.ToLower(p.Owner + "/" + p.Repo)
|
||||
var group *groupsLine
|
||||
if p.Head == "" {
|
||||
// A recheck (mesh-delivery's `recheck` through `delivery check`) carries no head branch name: the group is the one
|
||||
// whose members hold this very pull request, by repository and number.
|
||||
learnable := false
|
||||
for i := range lines {
|
||||
for _, m := range lines[i].Details {
|
||||
learnable = true
|
||||
if p.Number != 0 && m.Number == p.Number && strings.ToLower(m.Repository) == own {
|
||||
group = &lines[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
if group == nil {
|
||||
for _, l := range lines {
|
||||
if len(l.Details) == 0 && len(l.Members) > 0 && !learnable {
|
||||
return nil, "mesh-delivery named the groups' members without their pull requests (a mesh-delivery from before novox/hq ADR 0382): " +
|
||||
"the group of a pull request announced without its head branch cannot be found"
|
||||
}
|
||||
}
|
||||
return nil, "" // a lone pull request
|
||||
}
|
||||
} else {
|
||||
for i := range lines {
|
||||
if lines[i].ID == p.Head {
|
||||
group = &lines[i]
|
||||
}
|
||||
}
|
||||
if group == nil {
|
||||
return nil, ""
|
||||
}
|
||||
}
|
||||
if len(group.Details) == 0 && len(group.Members) > 0 {
|
||||
return nil, "mesh-delivery named the group's members without their heads and bases (a mesh-delivery from before novox/hq ADR 0382)"
|
||||
}
|
||||
read := besideRepositories(entries)
|
||||
var siblings []link.CheckSibling
|
||||
for _, m := range group.Details {
|
||||
repository := strings.ToLower(m.Repository)
|
||||
if repository == own {
|
||||
continue
|
||||
}
|
||||
if !m.Open {
|
||||
continue
|
||||
}
|
||||
if m.Head == "" || m.Base == "" {
|
||||
return nil, "mesh-delivery named " + m.ID + " without its head or base (a mesh-delivery from before novox/hq ADR 0382)"
|
||||
}
|
||||
dir, readBeside := read[repository]
|
||||
if m.Base != p.Base || !readBeside || dir == "mesh-lab" {
|
||||
continue
|
||||
}
|
||||
if _, there := beside[dir]; !there {
|
||||
continue
|
||||
}
|
||||
head := strings.ToLower(m.Head)
|
||||
if !wholeCommit.MatchString(head) {
|
||||
return nil, fmt.Sprintf("mesh-delivery named %s with a head that is not a whole commit (%q)", m.ID, m.Head)
|
||||
}
|
||||
siblings = append(siblings, link.CheckSibling{Repository: m.Repository, Number: m.Number, Head: head, Base: m.Base, Dir: dir})
|
||||
}
|
||||
sort.Slice(siblings, func(i, j int) bool { return siblings[i].Dir < siblings[j].Dir })
|
||||
return siblings, ""
|
||||
}
|
||||
|
||||
// withSiblings asks mesh-delivery's `groups` once and says in the request the pull request's siblings, or why they
|
||||
// could not be learned (ADR 0382 §1): a check never waits on mesh-delivery past groupAskWithin and never fails for it,
|
||||
// so a fix to mesh-delivery is still checked without it.
|
||||
func withSiblings(ctx context.Context, request *link.BuildRequest, p link.PullUpdated, entries []inventory.Entry) {
|
||||
lines, err := groupsAsked(ctx, deliverySeatHeld(entries))
|
||||
siblingsFrom(request, p, entries, lines, err)
|
||||
}
|
||||
|
||||
// siblingsFrom is withSiblings from mesh-delivery's answer already had.
|
||||
func siblingsFrom(request *link.BuildRequest, p link.PullUpdated, entries []inventory.Entry, lines []groupsLine, err error) {
|
||||
if request.Check == nil {
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
request.Check.SiblingsUnknown = fmt.Sprintf("mesh-delivery did not say the group's pull requests (%v): "+
|
||||
"the check reads no siblings", err)
|
||||
return
|
||||
}
|
||||
request.Check.Siblings, request.Check.SiblingsUnknown = siblingsFor(p, entries, request.Check.Beside, lines)
|
||||
}
|
||||
|
||||
// composedMember says the pull request is a member of a delivery group whose gate is the composed check's (novox/hq
|
||||
// ADR 0403 clauses 1 and 7): the group has two members or more and mesh-delivery says so on its line. The group is
|
||||
// found as siblingsFor finds it, by its head branch name, or, for a recheck that carries none, by the pull request's
|
||||
// repository and number — not by siblings, which name only what the check reads beside it.
|
||||
func composedMember(p link.PullUpdated, lines []groupsLine) bool {
|
||||
own := strings.ToLower(p.Owner + "/" + p.Repo)
|
||||
for _, l := range lines {
|
||||
found := p.Head != "" && l.ID == p.Head
|
||||
for _, m := range l.Details {
|
||||
if p.Head == "" && p.Number != 0 && m.Number == p.Number && strings.ToLower(m.Repository) == own {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if found {
|
||||
return l.ComposedGate && max(len(l.Details), len(l.Members)) >= 2
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -1,248 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A delivery group's member is checked beside the other members' heads** (novox/hq ADR 0382 §1, §7, issue 867): the
|
||||
// controller asks mesh-delivery's `groups` once for every check it asks, and names in the ask the group's other open
|
||||
// pull requests into a base of the same name, in a repository the build seat reads beside it — mesh-lab never.
|
||||
|
||||
const (
|
||||
headHost = "1111111111111111111111111111111111111111"
|
||||
headController = "2222222222222222222222222222222222222222"
|
||||
headSDK = "3333333333333333333333333333333333333333"
|
||||
headLab = "4444444444444444444444444444444444444444"
|
||||
headHq = "5555555555555555555555555555555555555555"
|
||||
)
|
||||
|
||||
func theCoreOnTheGitSeat() []inventory.Entry {
|
||||
onSeat := func(module, repository string) inventory.Entry {
|
||||
e := fromRepo(module, repository, "")
|
||||
e.Source.Seat = "git"
|
||||
return e
|
||||
}
|
||||
nats := onSeat("nats", "novox/mesh-catalog")
|
||||
nats.Source.Path = "modules/nats"
|
||||
held := catalogue.Manifest{Module: "mesh-delivery", Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}
|
||||
return []inventory.Entry{onSeat("mesh-controller", "novox/mesh-controller"), onSeat("mesh-host", "novox/mesh-host"),
|
||||
onSeat("node-tools", "novox/mesh-tools"), nats, {Manifest: held, On: []string{"anchor"}}}
|
||||
}
|
||||
|
||||
func besideTheCore() map[string]link.CheckedOut {
|
||||
return map[string]link.CheckedOut{"mesh-controller": {Repository: "c"}, "mesh-host": {Repository: "h"},
|
||||
"mesh-tools": {Repository: "t"}, "mesh-catalog": {Repository: "k"}, "mesh-sdk": {Repository: "s"}, "mesh-lab": {Repository: "l"}}
|
||||
}
|
||||
|
||||
func member(repository string, number int, head, base string, open bool) memberLine {
|
||||
return memberLine{ID: repository + "@" + head[:12], Repository: repository, State: "checked", Number: number, Head: head,
|
||||
Base: base, Open: open}
|
||||
}
|
||||
|
||||
func group(id string, members ...memberLine) groupsLine {
|
||||
line := groupsLine{ID: id, Details: members}
|
||||
for _, m := range members {
|
||||
line.Members = append(line.Members, m.ID+" ("+m.State+")")
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
func TestASiblingIsTheGroupsOtherOpenPullRequestIntoABaseOfTheSameName(t *testing.T) {
|
||||
entries := theCoreOnTheGitSeat()
|
||||
pull := link.PullUpdated{Owner: "novox", Repo: "mesh-host", Number: 114, Base: "main", Head: "feat/x", Commit: headHost}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
lines []groupsLine
|
||||
want []string
|
||||
unknown string
|
||||
}{
|
||||
{"the other member, open, same base", []groupsLine{group("feat/x",
|
||||
member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-controller", 346, headController, "main", true))},
|
||||
[]string{"mesh-controller=novox/mesh-controller#346@" + headController + " onto main"}, ""},
|
||||
{"a member into a base of another name is not one", []groupsLine{group("feat/x",
|
||||
member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-controller", 346, headController, "release", true))},
|
||||
nil, ""},
|
||||
{"a member not open (merged or closed) is not one", []groupsLine{group("feat/x",
|
||||
member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-controller", 346, headController, "main", false))},
|
||||
nil, ""},
|
||||
{"mesh-lab is never one", []groupsLine{group("feat/x",
|
||||
member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-lab", 5, headLab, "main", true))},
|
||||
nil, ""},
|
||||
{"a repository the check reads nothing of is not one", []groupsLine{group("feat/x",
|
||||
member("novox/mesh-host", 114, headHost, "main", true), member("novox/hq", 9, headHq, "main", true))},
|
||||
nil, ""},
|
||||
{"mesh-sdk, read beside the controller, is one", []groupsLine{group("feat/x",
|
||||
member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-sdk", 7, headSDK, "main", true))},
|
||||
[]string{"mesh-sdk=novox/mesh-sdk#7@" + headSDK + " onto main"}, ""},
|
||||
{"another group is none of its business", []groupsLine{group("feat/y",
|
||||
member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-controller", 346, headController, "main", true))},
|
||||
nil, ""},
|
||||
{"a group of its own alone has none", []groupsLine{group("feat/x", member("novox/mesh-host", 114, headHost, "main", true))},
|
||||
nil, ""},
|
||||
{"an older mesh-delivery names no heads", []groupsLine{{ID: "feat/x",
|
||||
Members: []string{"novox/mesh-host@111111111111 (checked)", "novox/mesh-controller@222222222222 (checked)"}}},
|
||||
nil, "without their heads and bases"},
|
||||
{"a member without its base", []groupsLine{group("feat/x",
|
||||
member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-controller", 346, headController, "", true))},
|
||||
nil, "without its head or base"},
|
||||
{"a head that is no commit", []groupsLine{group("feat/x",
|
||||
member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-controller", 346, headController[:12]+"zzzzzzzzzzzzzzzzzzzzzzzzzzzz", "main", true))},
|
||||
nil, "not a whole commit"},
|
||||
} {
|
||||
got, unknown := siblingsFor(pull, entries, besideTheCore(), c.lines)
|
||||
var said []string
|
||||
for _, s := range got {
|
||||
said = append(said, s.Dir+"="+s.Repository+"#"+itoa(s.Number)+"@"+s.Head+" onto "+s.Base)
|
||||
}
|
||||
if strings.Join(said, ";") != strings.Join(c.want, ";") {
|
||||
t.Errorf("%s: siblings %v, wanted %v", c.what, said, c.want)
|
||||
}
|
||||
if (c.unknown == "") != (unknown == "") || !strings.Contains(unknown, c.unknown) {
|
||||
t.Errorf("%s: could not learn %q, wanted %q", c.what, unknown, c.unknown)
|
||||
}
|
||||
}
|
||||
// A pull request into a release branch has siblings into the release branch of the same name (ADR 0358 §1).
|
||||
release := pull
|
||||
release.Base = "feat/57"
|
||||
lines := []groupsLine{group("feat/x", member("novox/mesh-host", 114, headHost, "feat/57", true),
|
||||
member("novox/mesh-controller", 346, headController, "feat/57", true), member("novox/mesh-tools", 8, headLab, "main", true))}
|
||||
if got, _ := siblingsFor(release, entries, besideTheCore(), lines); len(got) != 1 || got[0].Dir != "mesh-controller" {
|
||||
t.Errorf("into a release branch: %+v", got)
|
||||
}
|
||||
// A recheck announces the pull request without its head branch: its group is the one holding it (repository, number).
|
||||
pull.Head = ""
|
||||
got, unknown := siblingsFor(pull, entries, besideTheCore(), lines)
|
||||
if len(got) != 1 || got[0].Dir != "mesh-tools" || unknown != "" {
|
||||
t.Errorf("a recheck of a group member: %+v %q", got, unknown)
|
||||
}
|
||||
// A pull request in no group, rechecked, is alone: no sibling, and nothing said.
|
||||
pull.Number = 999
|
||||
if got, unknown := siblingsFor(pull, entries, besideTheCore(), lines); got != nil || unknown != "" {
|
||||
t.Errorf("a lone pull request: %+v %q", got, unknown)
|
||||
}
|
||||
if got, unknown := siblingsFor(pull, entries, besideTheCore(), nil); got != nil || unknown != "" {
|
||||
t.Errorf("no groups at all: %+v %q", got, unknown)
|
||||
}
|
||||
// A mesh-delivery that names members without their pull requests cannot find the group of a recheck: it says so.
|
||||
older := []groupsLine{{ID: "feat/x", Members: []string{"novox/mesh-host@111111111111 (checked)"}}}
|
||||
pull.Number = 114
|
||||
if got, unknown := siblingsFor(pull, entries, besideTheCore(), older); got != nil || unknown == "" {
|
||||
t.Errorf("an older mesh-delivery: %+v %q", got, unknown)
|
||||
}
|
||||
}
|
||||
|
||||
func itoa(n int) string { b, _ := json.Marshal(n); return string(b) }
|
||||
|
||||
// Every check asks mesh-delivery's `groups` once; the ask names the sibling's repository, head and base.
|
||||
func TestEveryCheckAsksMeshDeliveryOnceAndNamesItsSiblings(t *testing.T) {
|
||||
entries := theCoreOnTheGitSeat()
|
||||
asked := 0
|
||||
was := groupsAsked
|
||||
t.Cleanup(func() { groupsAsked = was })
|
||||
groupsAsked = func(_ context.Context, held bool) ([]groupsLine, error) {
|
||||
asked++
|
||||
if !held {
|
||||
t.Error("the seat is held, yet it was asked as not held")
|
||||
}
|
||||
return []groupsLine{group("feat/x", member("novox/mesh-host", 114, headHost, "main", true),
|
||||
member("novox/mesh-controller", 346, headController, "main", true))}, nil
|
||||
}
|
||||
request := link.BuildRequest{Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-host", Base: "main", Beside: besideTheCore()}}
|
||||
withSiblings(t.Context(), &request, link.PullUpdated{Owner: "novox", Repo: "mesh-host", Base: "main", Head: "feat/x"}, entries)
|
||||
if asked != 1 {
|
||||
t.Fatalf("mesh-delivery was asked %d times for one check", asked)
|
||||
}
|
||||
want := link.CheckSibling{Repository: "novox/mesh-controller", Number: 346, Head: headController, Base: "main", Dir: "mesh-controller"}
|
||||
if got := request.Check.Siblings; len(got) != 1 || got[0] != want || request.Check.SiblingsUnknown != "" {
|
||||
t.Fatalf("the ask names siblings %+v, unknown %q", got, request.Check.SiblingsUnknown)
|
||||
}
|
||||
body, _ := json.Marshal(request.Check)
|
||||
if !strings.Contains(string(body), `"siblings":[{"repository":"novox/mesh-controller","number":346,"head":"`+headController+
|
||||
`","base":"main","dir":"mesh-controller"}]`) {
|
||||
t.Fatalf("on the wire: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
// When mesh-delivery does not answer in time (or is not held, or answered without the fields), the check reads no
|
||||
// sibling and its request says it could not learn them, so a fix to mesh-delivery is still checked without it.
|
||||
func TestWhenMeshDeliveryDoesNotAnswerTheCheckReadsNoSiblingAndSaysSo(t *testing.T) {
|
||||
was := groupsAsked
|
||||
t.Cleanup(func() { groupsAsked = was })
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
err error
|
||||
held bool
|
||||
}{{"it did not answer", context.DeadlineExceeded, true}, {"nothing serves it", errors.New("nothing serves mesh-delivery.groups"), true},
|
||||
{"no holder on record", errors.New("no holder of mesh-delivery is on record to say it"), false}} {
|
||||
entries := theCoreOnTheGitSeat()
|
||||
if !c.held {
|
||||
entries = entries[:4]
|
||||
}
|
||||
groupsAsked = func(_ context.Context, held bool) ([]groupsLine, error) {
|
||||
if held != c.held {
|
||||
t.Errorf("%s: held %v", c.what, held)
|
||||
}
|
||||
return nil, c.err
|
||||
}
|
||||
request := link.BuildRequest{Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-host", Base: "main", Beside: besideTheCore()}}
|
||||
withSiblings(t.Context(), &request, link.PullUpdated{Owner: "novox", Repo: "mesh-host", Base: "main", Head: "feat/x"}, entries)
|
||||
if len(request.Check.Siblings) != 0 || !strings.Contains(request.Check.SiblingsUnknown, c.err.Error()) ||
|
||||
!strings.Contains(request.Check.SiblingsUnknown, "reads no siblings") {
|
||||
t.Errorf("%s: siblings %+v, unknown %q", c.what, request.Check.Siblings, request.Check.SiblingsUnknown)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **A pull request for which mesh-delivery answered and named no sibling is asked exactly as before** (ADR 0382 §7):
|
||||
// byte for byte the request it was without the rule.
|
||||
func TestAPullRequestWithNoSiblingIsAskedByteForByteAsBefore(t *testing.T) {
|
||||
entries := theCoreOnTheGitSeat()
|
||||
was := groupsAsked
|
||||
t.Cleanup(func() { groupsAsked = was })
|
||||
for _, answer := range [][]groupsLine{nil, {group("feat/other", member("novox/mesh-controller", 346, headController, "main", true))},
|
||||
{group("feat/x", member("novox/mesh-host", 114, headHost, "main", true), member("novox/hq", 3, headHq, "main", true))}} {
|
||||
groupsAsked = func(context.Context, bool) ([]groupsLine, error) { return answer, nil }
|
||||
build := func() link.BuildRequest {
|
||||
return link.BuildRequest{ID: "build-1", Repository: "r", Ref: headHost, Check: &link.CheckRequest{Owner: "novox",
|
||||
Repo: "mesh-host", Number: 114, Base: "main", Beside: besideTheCore(), Modules: []string{"mesh-host"}, Judge: "validator"}}
|
||||
}
|
||||
before, _ := json.Marshal(build())
|
||||
after := build()
|
||||
withSiblings(t.Context(), &after, link.PullUpdated{Owner: "novox", Repo: "mesh-host", Base: "main", Head: "feat/x"}, entries)
|
||||
got, _ := json.Marshal(after)
|
||||
if string(before) != string(got) {
|
||||
t.Fatalf("the request changed:\nbefore %s\nafter %s", before, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// What the build seat's verdict says of its siblings reaches the forge's holder in the controller's `checked`.
|
||||
func TestTheVerdictCarriesTheSiblingsAndWhereTheJudgeCameFrom(t *testing.T) {
|
||||
read := []link.CheckSibling{{Repository: "novox/mesh-controller", Number: 346, Head: headController, Base: "main", BaseCommit: headHost}}
|
||||
c := checkedOf(link.BuildResult{ID: "b", Ref: headHost, Checked: &link.CheckRequest{Owner: "novox", Repo: "mesh-host", Number: 114},
|
||||
Check: &link.CheckOutcome{Verdict: "pass", Summary: "ok", Siblings: read, JudgeFrom: "sibling novox/mesh-controller#346 merge abc",
|
||||
Validator: "the change's own validator"}})
|
||||
if len(c.Siblings) != 1 || c.Siblings[0] != read[0] || c.JudgeFrom == "" || c.Validator == "" {
|
||||
t.Fatalf("the verdict says %+v", c)
|
||||
}
|
||||
body, _ := json.Marshal(c)
|
||||
for _, want := range []string{`"siblings":[{"repository":"novox/mesh-controller","number":346,"head":"` + headController +
|
||||
`","base":"main","base_commit":"` + headHost + `"}]`, `"judge_from":"sibling novox/mesh-controller#346 merge abc"`} {
|
||||
if !strings.Contains(string(body), want) {
|
||||
t.Fatalf("the verdict lacks %s:\n%s", want, body)
|
||||
}
|
||||
}
|
||||
unknown := checkedOf(link.BuildResult{ID: "b", Check: &link.CheckOutcome{Verdict: "pass", SiblingsUnknown: "mesh-delivery did not answer"}})
|
||||
if body, _ := json.Marshal(unknown); !strings.Contains(string(body), `"siblings_unknown":"mesh-delivery did not answer"`) ||
|
||||
strings.Contains(string(body), `"siblings":`) {
|
||||
t.Fatalf("the verdict that could not learn them says %s", body)
|
||||
}
|
||||
}
|
||||
@@ -6,10 +6,8 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/beside"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The check anybody can run is the check registration runs (novox/hq issue 148, ADR 0037): a manifest
|
||||
@@ -58,7 +56,10 @@ func TestModuleCheckJudgesBetweenTheManifestsGiven(t *testing.T) {
|
||||
|
||||
// The real catalogue passes the command, the way it passes the test that used to be the only check.
|
||||
func TestModuleCheckPassesTheCatalogue(t *testing.T) {
|
||||
root := beside.Catalogue(t)
|
||||
root := filepath.Join("..", "..", "..", "mesh-catalog", "modules")
|
||||
if _, err := os.Stat(root); err != nil {
|
||||
t.Skipf("catalogue sibling not present: %v", err)
|
||||
}
|
||||
paths, err := manifestsUnder(root)
|
||||
if err != nil || len(paths) == 0 {
|
||||
t.Fatalf("no manifests under %s: %v", root, err)
|
||||
@@ -102,60 +103,3 @@ func TestTheControllersManifestServesEveryVerbOfItsSeat(t *testing.T) {
|
||||
t.Fatalf("the controller's own module.json fails module check: %v\n%s", err, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A module that uses a seat another module declares (novox/hq issue 364): refused over the whole catalogue when the
|
||||
// declarer is missing, a note when the gate says it gives only the manifests a change touches.
|
||||
func TestASeatAnotherModuleDeclaresIsANoteOverSomeManifests(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
user := filepath.Join(dir, "user.json")
|
||||
os.WriteFile(user, []byte(`{"module":"asker","version":"1","uses":["operator-channel"]}`), 0o600)
|
||||
var out bytes.Buffer
|
||||
if err := moduleCheck([]string{user}, &out); err == nil {
|
||||
t.Fatalf("a use of a seat nothing given declares passed the whole-catalogue check:\n%s", out.String())
|
||||
}
|
||||
t.Setenv(SomeManifestsVar, "1")
|
||||
out.Reset()
|
||||
if err := moduleCheck([]string{user}, &out); err != nil {
|
||||
t.Fatalf("over some manifests the use was refused:\n%s", out.String())
|
||||
}
|
||||
if !strings.Contains(out.String(), "note: asker uses the seat \"operator-channel\"") {
|
||||
t.Fatalf("the note was not said:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// **A module lives whole in one repository** (novox/hq ADR 0420 §1, issue 1019): `module check`, told the
|
||||
// repository it runs for, refuses an artifact whose build context names another; run by hand, it does not know.
|
||||
func TestModuleCheckRefusesAnArtifactContextNamingAnotherRepository(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
write := func(name, context string) string {
|
||||
path := filepath.Join(dir, name+".json")
|
||||
raw := `{"module":"` + name + `","version":"1","build":{"artifacts":[{"name":"server","kind":"image","from":"Dockerfile"` + context + `}]}}`
|
||||
if err := os.WriteFile(path, []byte(raw), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
other := write("other", `,"context":{"seat":"git","repository":"novox/mesh-controller","ref":"main"}`)
|
||||
own := write("own", `,"context":{"seat":"git","repository":"novox/mesh-catalog","ref":"main"}`)
|
||||
none := write("none", ``)
|
||||
|
||||
t.Setenv("MESH_CHECK_REPOSITORY", "novox/mesh-catalog")
|
||||
var out bytes.Buffer
|
||||
if err := moduleCheck([]string{other}, &out); err == nil {
|
||||
t.Fatalf("a context naming another repository passed:\n%s", out.String())
|
||||
}
|
||||
if !strings.Contains(out.String(), "other builds artifact server from novox/mesh-controller: a module lives whole in one repository (novox/hq ADR 0420 §1)") {
|
||||
t.Errorf("the refusal is not in the words of the gate:\n%s", out.String())
|
||||
}
|
||||
for _, path := range []string{own, none} {
|
||||
out.Reset()
|
||||
if err := moduleCheck([]string{path}, &out); err != nil {
|
||||
t.Errorf("%s was refused: %v\n%s", path, err, out.String())
|
||||
}
|
||||
}
|
||||
t.Setenv("MESH_CHECK_REPOSITORY", "")
|
||||
out.Reset()
|
||||
if err := moduleCheck([]string{other}, &out); err != nil {
|
||||
t.Errorf("a local run, with no repository named, refused it: %v\n%s", err, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
+27
-290
@@ -5,8 +5,6 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path"
|
||||
"regexp"
|
||||
"runtime/debug"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -145,7 +143,7 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
read, err := readForPlanning(ctx, inv)
|
||||
read, err := inv.ReadRepositories(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -154,24 +152,6 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||
return err
|
||||
}
|
||||
scope := pullScope(p, entries, read, edges)
|
||||
// **A cycle of edges fails the check, naming every module and edge** (novox/hq ADR 0419 clause 6), a lone pull
|
||||
// request's as a group's composed check is: no order exists for what it changes.
|
||||
alone := orderMember{ID: p.Owner + "/" + p.Repo + "@" + twelve(p.Commit), Repository: p.Owner + "/" + p.Repo, Base: p.Base,
|
||||
Head: p.Commit, Number: p.Number, Paths: p.Paths, PathsTruncated: p.PathsTruncated, Removed: p.Removed, BaseSHA: p.BaseSHA}
|
||||
if failing, cyclic, err := cycleVerdict(ctx, "", []orderMember{alone},
|
||||
map[string]orderReach{alone.ID: {Moved: scope.Modules}}, edges, entries); err != nil {
|
||||
return err
|
||||
} else if cyclic {
|
||||
failing.Commit, failing.Trunk = p.Commit, twelve(p.BaseSHA)
|
||||
fmt.Printf("%s/%s#%d (%.8s): %s\n", p.Owner, p.Repo, p.Number, p.Commit, failing.Summary)
|
||||
sayChecked(ctx, failing)
|
||||
return nil
|
||||
}
|
||||
// **A member of a group whose gate is the composed check's runs its repository's check alone** (novox/hq ADR 0403
|
||||
// clause 1): mesh-delivery is asked once, and says so on the group's line. Told nothing, the pull request is
|
||||
// checked alone, gate included (clause 6).
|
||||
lines, groupsErr := groupsAsked(ctx, deliverySeatHeld(entries))
|
||||
composed := groupsErr == nil && composedMember(p, lines)
|
||||
// The change plan of the commit at hand (ADR 0238): what a merge of it would build and send, posted
|
||||
// with the verdict whatever the verdict is.
|
||||
plan := changePlanOf(p.Owner+"/"+p.Repo, p.Base, p.Commit, scope.Reach, entries, func(module string) (inventory.Upgrade, bool) {
|
||||
@@ -179,11 +159,9 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||
return u, err == nil
|
||||
})
|
||||
fmt.Print(planText(plan))
|
||||
direct := directVerdict(p, plan)
|
||||
if composed {
|
||||
scope.Modules, scope.Dependents, scope.New, scope.Manifests = nil, nil, nil, nil
|
||||
composedGate(&direct)
|
||||
}
|
||||
direct := link.Checked{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Commit: p.Commit,
|
||||
ID: link.NewBuildID(time.Now()), Verdict: "pass", Summary: noModuleTouched,
|
||||
Gate: &link.CheckLayer{Verdict: "pass", Summary: noModuleTouched}, Plan: &plan}
|
||||
switch {
|
||||
case !scope.gated() && !scope.Mesh:
|
||||
fmt.Printf("%s/%s#%d (%.8s): %s, and the repository is not the mesh's: said, nothing run\n",
|
||||
@@ -201,9 +179,6 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||
return err
|
||||
}
|
||||
request.Check.Plan = &plan
|
||||
request.Check.Composed = composed
|
||||
// **The group's other open pull requests are read beside it** (novox/hq ADR 0382 §1).
|
||||
siblingsFrom(&request, p, entries, lines, groupsErr)
|
||||
seat := buildSeatHeld(ctx)
|
||||
ask, err := askOverOn(seat)
|
||||
if err != nil {
|
||||
@@ -213,13 +188,7 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||
if err := ask.Ask(ctx, request); err != nil {
|
||||
return err
|
||||
}
|
||||
// **A newer request replaces the one still waiting, and a newer key stops the one running** (novox/hq ADR 0406 §3, §4,
|
||||
// issue 974): the check queue keeps one request a pull request, and a running check watches its key. Nothing is asked
|
||||
// of the forge here (issue 726).
|
||||
what := "its own merge-check.sh alone: " + noModuleTouched
|
||||
if composed {
|
||||
what = "its own merge-check.sh alone: the gate is its group's composed check's"
|
||||
}
|
||||
if scope.gated() {
|
||||
what = fmt.Sprintf("the gate over %s (a merge would build %d module(s) in %d tier(s))",
|
||||
strings.Join(append(append([]string{}, scope.Modules...), prefixedAll("new:", scope.New)...), ", "),
|
||||
@@ -230,38 +199,6 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// composedWords is what a group member's own verdict says of its gate layer (novox/hq ADR 0403 clause 1).
|
||||
const composedWords = "judged by its group's composed check"
|
||||
|
||||
// gateComposed is a gate layer's verdict when no gate was run for the member alone: never a pass. Mesh-delivery reads it
|
||||
// as the composed check's to judge, and the forge's holder sets no mesh/merge-gate from it.
|
||||
const gateComposed = "composed"
|
||||
|
||||
// composedGate says a group member's own verdict has no gate of its own: its gate layer, and the verdict that is
|
||||
// the gate's, read "composed" in place of a pass.
|
||||
func composedGate(c *link.Checked) {
|
||||
c.Verdict, c.Summary = gateComposed, composedWords
|
||||
c.Gate = &link.CheckLayer{Verdict: gateComposed, Summary: composedWords}
|
||||
}
|
||||
|
||||
// directVerdict is the verdict of a change no build judges — it touches no module, and either the repository is not
|
||||
// the mesh's or the head holds no merge-check.sh. **It names the trunk commit it holds for** (novox/hq ADR 0323 §3):
|
||||
// the base branch's head the forge announced with the head, so the merge tool can weigh it against the trunk's head
|
||||
// and does not refuse it for ever (a verdict naming no trunk is never the trunk's head's). Nothing was built to
|
||||
// judge: the trunk commit is the one the forge said the change was announced against.
|
||||
func directVerdict(p link.PullUpdated, plan link.ChangePlan) link.Checked {
|
||||
trunk := p.BaseSHA
|
||||
if len(trunk) > 12 {
|
||||
trunk = trunk[:12]
|
||||
}
|
||||
if trunk != "" {
|
||||
plan.Trunk = trunk
|
||||
}
|
||||
return link.Checked{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Commit: p.Commit,
|
||||
ID: link.NewBuildID(time.Now()), Verdict: "pass", Summary: noModuleTouched,
|
||||
Gate: &link.CheckLayer{Verdict: "pass", Summary: noModuleTouched}, Plan: &plan, Trunk: trunk}
|
||||
}
|
||||
|
||||
// checkRequestFor is the ask for one pull request's head: the repository as the mesh clones it, the head,
|
||||
// and what is read beside it.
|
||||
func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scope checkScope,
|
||||
@@ -293,29 +230,7 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
|
||||
if err != nil {
|
||||
return link.BuildRequest{}, err
|
||||
}
|
||||
beside, err := besideOf(entries, func(module string) string { return current[module].Commit }, clone)
|
||||
if err != nil {
|
||||
return link.BuildRequest{}, err
|
||||
}
|
||||
builds := checkBuildsOf(p, scope, entries, clone)
|
||||
return link.BuildRequest{
|
||||
ID: link.NewBuildID(time.Now()),
|
||||
Repository: repository,
|
||||
Ref: p.Commit,
|
||||
Held: heldBy(ctx),
|
||||
Seats: seatBases(ctx),
|
||||
Source: sourceOnSeat(source),
|
||||
Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base, Trunk: p.BaseSHA, Ready: p.Ready,
|
||||
Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents, New: scope.New,
|
||||
Manifests: scope.Manifests, Judge: scope.Judge, Builds: builds, Named: namedForTheCheck(ctx, p, builds),
|
||||
Running: runningBy(ctx)},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// besideOf is what a check reads beside its own repository, by the directory it is found under: each core repository
|
||||
// by the module the mesh builds from it, at besideRefs. running is the commit the mesh runs of a module.
|
||||
func besideOf(entries []inventory.Entry, running func(module string) string,
|
||||
clone func(inventory.Source) (string, error)) (map[string]link.CheckedOut, error) {
|
||||
// Beside it, at what the mesh runs: each core repository by the module the mesh builds from it.
|
||||
beside := map[string]link.CheckedOut{}
|
||||
for _, e := range entries {
|
||||
dir, core := coreModules[e.Manifest.Module]
|
||||
@@ -324,115 +239,31 @@ func besideOf(entries []inventory.Entry, running func(module string) string,
|
||||
}
|
||||
url, err := clone(e.Source)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return link.BuildRequest{}, err
|
||||
}
|
||||
refs := besideRefs(dir, running(e.Manifest.Module))
|
||||
refs := besideRefs(dir, current[e.Manifest.Module].Commit)
|
||||
beside[dir] = link.CheckedOut{Repository: url, Ref: refs[dir]}
|
||||
if dir == "mesh-controller" {
|
||||
// The controller's main stays its main: it judges only a check the running controller predates.
|
||||
beside["mesh-controller-main"] = link.CheckedOut{Repository: url, Ref: refs["mesh-controller-main"]}
|
||||
if e.Source.Seat != "" {
|
||||
for _, sibling := range []string{"mesh-lab", "mesh-sdk"} {
|
||||
if url, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository,
|
||||
sibling)}); err == nil {
|
||||
beside[sibling] = link.CheckedOut{Repository: url, Ref: refs[sibling]}
|
||||
}
|
||||
if lab, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository,
|
||||
"mesh-lab")}); err == nil {
|
||||
beside["mesh-lab"] = link.CheckedOut{Repository: lab, Ref: refs["mesh-lab"]}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return beside, nil
|
||||
}
|
||||
|
||||
// trunkOf is the branch a module's source follows: main when it records none, or records a commit.
|
||||
func trunkOf(s inventory.Source) string {
|
||||
if t := strings.TrimSpace(followedBranch(s.Ref)); t != "" {
|
||||
return t
|
||||
}
|
||||
return "main"
|
||||
}
|
||||
|
||||
// sourceName is the name of a recorded repository, a path on the git seat or a URL: novox/mesh-host → mesh-host.
|
||||
func sourceName(repository string) string {
|
||||
parts := strings.Split(strings.Trim(strings.TrimSuffix(repository, ".git"), "/"), "/")
|
||||
if len(parts) < 2 {
|
||||
return ""
|
||||
}
|
||||
return parts[len(parts)-1]
|
||||
}
|
||||
|
||||
// namedForTheCheck are the builds earlier checks of the same pull request at the same head named, read once for the
|
||||
// check (novox/hq issue 703): asked again because only the trunk moved, the check keeps what they built of a key
|
||||
// still the same. Another pull request's build, or one of an earlier head, is never kept. None when they cannot be
|
||||
// read, and the check builds, as before.
|
||||
func namedForTheCheck(ctx context.Context, p link.PullUpdated, builds []link.CheckBuild) []link.CheckedBuild {
|
||||
if len(builds) == 0 || p.Commit == "" {
|
||||
return nil
|
||||
}
|
||||
pull := link.PullRef{Owner: p.Owner, Repo: p.Repo, Number: p.Number}
|
||||
read, err := checkedBuildsOf(ctx, []link.PullRef{pull})
|
||||
if err != nil {
|
||||
fmt.Printf(" the earlier checks' builds of %s/%s#%d could not be read, so the check builds: %v\n",
|
||||
p.Owner, p.Repo, p.Number, err)
|
||||
// empty-on-error: said above; a check handed no named build builds every module, as before issue 703
|
||||
return nil
|
||||
}
|
||||
var named []link.CheckedBuild
|
||||
for _, b := range read {
|
||||
if b.Pull == pull && link.SameHead(b.Head, p.Commit) {
|
||||
named = append(named, b)
|
||||
}
|
||||
}
|
||||
return named
|
||||
}
|
||||
|
||||
// checkBuildsOf are the modules a merge of the change would build, in the walk's order, as the check builds them
|
||||
// for the walk to take (novox/hq ADR 0350 §1): a module built from the repository checked, at the branch it merges
|
||||
// into, from the merge the check makes; any other — a module standing on one — from its own repository at the
|
||||
// branch it follows, as the walk builds it. A module whose build reads the repository checked through a context,
|
||||
// or that the merge deletes, cannot be built as the walk would: said, and nothing after it is built. A change that
|
||||
// moves no module builds nothing.
|
||||
func checkBuildsOf(p link.PullUpdated, scope checkScope, entries []inventory.Entry,
|
||||
clone func(inventory.Source) (string, error)) []link.CheckBuild {
|
||||
if len(scope.Reach.Plan.Tiers) == 0 {
|
||||
return nil
|
||||
}
|
||||
m := link.SourceMoved{Owner: p.Owner, Repo: p.Repo, Base: p.Base, CloneURL: p.CloneURL}
|
||||
byName := map[string]inventory.Entry{}
|
||||
for _, e := range entries {
|
||||
byName[e.Manifest.Module] = e
|
||||
}
|
||||
skip := map[string]string{}
|
||||
for _, e := range scope.Reach.Packaging {
|
||||
skip[e.Manifest.Module] = "its build reads " + p.Owner + "/" + p.Repo + " through a context, which a check " +
|
||||
"cannot hand the merge"
|
||||
}
|
||||
for _, e := range scope.Reach.Deleted {
|
||||
skip[e.Manifest.Module] = "the merge deletes it"
|
||||
}
|
||||
var out []link.CheckBuild
|
||||
for tier, names := range scope.Reach.Plan.Tiers {
|
||||
for _, name := range names {
|
||||
e, known := byName[name]
|
||||
b := link.CheckBuild{Module: name, Tier: tier, Skip: skip[name]}
|
||||
switch {
|
||||
case !known:
|
||||
b.Skip = "the catalogue does not hold it"
|
||||
case b.Skip != "":
|
||||
case sourceIs(e.Source, m):
|
||||
b.Merged, b.Path = true, e.Source.Path
|
||||
default:
|
||||
url, err := clone(e.Source)
|
||||
if err != nil {
|
||||
b.Skip = "its repository cannot be cloned: " + err.Error()
|
||||
break
|
||||
}
|
||||
b.Repository, b.Path, b.Ref = url, e.Source.Path, followedBranch(e.Source.Ref)
|
||||
}
|
||||
out = append(out, b)
|
||||
}
|
||||
}
|
||||
return out
|
||||
return link.BuildRequest{
|
||||
ID: link.NewBuildID(time.Now()),
|
||||
Repository: repository,
|
||||
Ref: p.Commit,
|
||||
Held: heldBy(ctx),
|
||||
Seats: seatBases(ctx),
|
||||
Source: sourceOnSeat(source),
|
||||
Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base,
|
||||
Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents, New: scope.New,
|
||||
Manifests: scope.Manifests, Judge: scope.Judge},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// besideRefs is the ref each repository is cloned at beside a check, by the directory it is found under:
|
||||
@@ -440,54 +271,17 @@ func checkBuildsOf(p link.PullUpdated, scope checkScope, entries []inventory.Ent
|
||||
// catalogue, whose checkout beside is what tests read its files from, at its main, what the next merge
|
||||
// builds from; and beside the controller its main, for a judge the running controller predates, and the
|
||||
// lab's main, whose replays every check runs. **One rule, read by the check the controller asks for and by
|
||||
// the facts snapshot** (Facts.Beside), so a check run by hand clones what the build seat clones. Beside the
|
||||
// controller also the SDK, checked out at the commit the running controller's go.mod pins (sdkPinned), not
|
||||
// one a desktop holds (novox/hq issue 449). The checkout only places the clone: the conformance test reads the
|
||||
// fixtures at the pin of the tree under check, from the clone's history, so a pull request moving the SDK is
|
||||
// judged against the SDK it moves to (internal/link/conformance_test.go).
|
||||
// the facts snapshot** (Facts.Beside), so a check run by hand clones what the build seat clones.
|
||||
func besideRefs(dir, running string) map[string]string {
|
||||
switch dir {
|
||||
case "mesh-catalog":
|
||||
return map[string]string{dir: "main"}
|
||||
case "mesh-controller":
|
||||
return map[string]string{dir: running, "mesh-controller-main": "main", "mesh-lab": "main",
|
||||
"mesh-sdk": sdkPinned()}
|
||||
return map[string]string{dir: running, "mesh-controller-main": "main", "mesh-lab": "main"}
|
||||
}
|
||||
return map[string]string{dir: running}
|
||||
}
|
||||
|
||||
// sdkModule is the Go module of the SDK the controller is built against.
|
||||
const sdkModule = "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
// pseudoCommit is the commit a Go pseudo-version names: v0.1.11-0.20261009143344-f047d0a4a970 → f047d0a4a970.
|
||||
var pseudoCommit = regexp.MustCompile(`-([0-9a-f]{12})$`)
|
||||
|
||||
// sdkPinned is the ref of the SDK repository this controller was built from, as its go.mod pins it and its
|
||||
// build records it: a pseudo-version's commit, or a release's tag (the SDK tags its Go module under go/).
|
||||
// "main" only for a binary that records no SDK version — a local replace — which a running controller is
|
||||
// not (novox/hq issue 449).
|
||||
func sdkPinned() string {
|
||||
info, ok := debug.ReadBuildInfo()
|
||||
if !ok {
|
||||
return "main"
|
||||
}
|
||||
for _, dep := range info.Deps {
|
||||
if dep.Path != sdkModule {
|
||||
continue
|
||||
}
|
||||
if dep.Replace != nil {
|
||||
dep = dep.Replace
|
||||
}
|
||||
if m := pseudoCommit.FindStringSubmatch(dep.Version); m != nil {
|
||||
return m[1]
|
||||
}
|
||||
if strings.HasPrefix(dep.Version, "v") {
|
||||
return "go/" + dep.Version
|
||||
}
|
||||
}
|
||||
return "main"
|
||||
}
|
||||
|
||||
// siblingOf is another repository of the same owner: novox/mesh-controller → novox/mesh-lab.
|
||||
func siblingOf(repository, name string) string {
|
||||
if cut := strings.LastIndex(repository, "/"); cut >= 0 {
|
||||
@@ -514,13 +308,6 @@ const maxCheckReport = 60 << 10
|
||||
// checked says a merge check's verdict as the controller's `checked`. Nothing is recorded or
|
||||
// registered: a check builds nothing (issue 240's rule for a dry run, kept for a check).
|
||||
func checked(ctx context.Context, result link.BuildResult) {
|
||||
// **A check stopped by its key says nothing** (novox/hq ADR 0406 §4, issue 974, review 2): it stopped because the head or
|
||||
// the trunk commit moved on, and the newer request will judge. Said as an error it would name the live head — red
|
||||
// statuses on the forge, a delivery refused — and written to the key it would stand in the newer request's way.
|
||||
if result.Failed == link.SupersededByKey {
|
||||
fmt.Printf("%s: stopped, its pull request's key moved on; the newer check will judge\n", result.ID)
|
||||
return
|
||||
}
|
||||
sayChecked(ctx, checkedOf(result))
|
||||
}
|
||||
|
||||
@@ -535,12 +322,6 @@ func checkedOf(result link.BuildResult) link.Checked {
|
||||
case result.Check != nil:
|
||||
c.Verdict, c.Summary, c.Report = result.Check.Verdict, result.Check.Summary, result.Check.Report
|
||||
c.Gate, c.RepoCheck = result.Check.Gate, result.Check.RepoCheck
|
||||
// What was judged (novox/hq ADR 0323 §3): the head merged into the trunk commit, and the commits of the
|
||||
// repositories read beside it.
|
||||
c.Trunk, c.Beside, c.Took = result.Check.Trunk, result.Check.Beside, result.Check.Took
|
||||
c.Built = result.Check.Built
|
||||
c.Validator = result.Check.Validator
|
||||
c.Siblings, c.SiblingsUnknown, c.JudgeFrom = result.Check.Siblings, result.Check.SiblingsUnknown, result.Check.JudgeFrom
|
||||
if c.Gate == nil {
|
||||
// A build seat from before the layers: its verdict is the gate's.
|
||||
c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
|
||||
@@ -558,12 +339,6 @@ func checkedOf(result link.BuildResult) link.Checked {
|
||||
c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
|
||||
c.RepoCheck = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
|
||||
}
|
||||
// **A group member's own check has no gate of its own** (novox/hq ADR 0403 clause 1): what ran was the repository's
|
||||
// check alone, and its gate layer is "composed", never a pass. A failure or an error stays what it was.
|
||||
if result.Checked != nil && result.Checked.Composed && result.Check != nil && c.Gate != nil &&
|
||||
(c.Gate.Verdict == "pass" || c.Gate.Verdict == "warning") {
|
||||
composedGate(&c)
|
||||
}
|
||||
if result.Checked != nil && c.Gate != nil && len(c.Gate.Modules) == 0 {
|
||||
c.Gate.Modules = append(append([]string{}, result.Checked.Modules...), prefixedAll("new:", result.Checked.New)...)
|
||||
}
|
||||
@@ -572,20 +347,14 @@ func checkedOf(result link.BuildResult) link.Checked {
|
||||
}
|
||||
if result.Checked != nil {
|
||||
c.Plan = result.Checked.Plan
|
||||
if c.Plan != nil && c.Trunk != "" {
|
||||
// The kept delivery plan records the trunk commit with its other inputs (ADR 0323 §3).
|
||||
plan := *c.Plan
|
||||
plan.Trunk = c.Trunk
|
||||
c.Plan = &plan
|
||||
}
|
||||
// A delivery group's composed check (novox/hq ADR 0239): every head it judged, this one first.
|
||||
if g := result.Checked; g.Group != "" {
|
||||
c.Group = g.Group
|
||||
c.Members = append(c.Members, link.CheckedMember{Owner: g.Owner, Repo: g.Repo, Number: g.Number,
|
||||
Commit: result.Ref, Trunk: c.Trunk})
|
||||
Commit: result.Ref})
|
||||
for _, m := range g.Members {
|
||||
c.Members = append(c.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo, Number: m.Number,
|
||||
Commit: m.Ref, Trunk: memberTrunk(result.Check, m)})
|
||||
Commit: m.Ref})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -594,40 +363,12 @@ func checkedOf(result link.BuildResult) link.Checked {
|
||||
l.Verdict = "error"
|
||||
}
|
||||
}
|
||||
// **A check run again says so** (novox/hq issue 594). The summaries are the lines the pull request
|
||||
// shows, clipped by the forge to a few words, so they get only a short marker after the verdict's own
|
||||
// words; the report gets the whole sentence. Layers copied, so the build's own outcome is left as it came.
|
||||
if mark := link.RunMarker(result.Delivery); mark != "" {
|
||||
c.Summary += " " + mark
|
||||
for _, l := range []**link.CheckLayer{&c.Gate, &c.RepoCheck} {
|
||||
if *l != nil {
|
||||
layer := **l
|
||||
layer.Summary += " " + mark
|
||||
*l = &layer
|
||||
}
|
||||
}
|
||||
c.Report = link.Restarted(result.Delivery) + "\n\n" + c.Report
|
||||
}
|
||||
if len(c.Report) > maxCheckReport {
|
||||
c.Report = "…" + c.Report[len(c.Report)-maxCheckReport:]
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// memberTrunk is the trunk commit the build seat merged one of a group's heads into (novox/hq ADR 0323 §6), empty
|
||||
// when it said none.
|
||||
func memberTrunk(outcome *link.CheckOutcome, m link.GroupMember) string {
|
||||
if outcome == nil {
|
||||
return ""
|
||||
}
|
||||
for _, j := range outcome.Members {
|
||||
if strings.EqualFold(j.Owner, m.Owner) && strings.EqualFold(j.Repo, m.Repo) && strings.HasPrefix(m.Ref, j.Commit) {
|
||||
return j.Trunk
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func prefixedAll(prefix string, items []string) []string {
|
||||
out := make([]string, 0, len(items))
|
||||
for _, i := range items {
|
||||
@@ -642,12 +383,8 @@ func sayChecked(ctx context.Context, c link.Checked) {
|
||||
if c.RepoCheck != nil {
|
||||
repo = strings.ToUpper(c.RepoCheck.Verdict) + " — " + c.RepoCheck.Summary
|
||||
}
|
||||
judged := ""
|
||||
if c.Trunk != "" {
|
||||
judged = " merged into trunk " + c.Trunk
|
||||
}
|
||||
fmt.Printf("%s: %s/%s#%d at %.8s%s checked on %s: gate %s — %s; repository %s\n", c.ID, c.Owner, c.Repo, c.Number,
|
||||
c.Commit, judged, orSomewhere(c.On), strings.ToUpper(c.Verdict), c.Summary, repo)
|
||||
fmt.Printf("%s: %s/%s#%d at %.8s checked on %s: gate %s — %s; repository %s\n", c.ID, c.Owner, c.Repo, c.Number,
|
||||
c.Commit, orSomewhere(c.On), strings.ToUpper(c.Verdict), c.Summary, repo)
|
||||
if checkEvents == nil {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
func TestACheckStoppedByItsKeyIsAnnouncedAsNoVerdict(t *testing.T) {
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
legacy, err := js.Conn().JetStream()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
was := checkEvents
|
||||
checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy}
|
||||
t.Cleanup(func() { checkEvents = was })
|
||||
heard := make(chan *nats.Msg, 4)
|
||||
sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyChecked), heard)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog", Number: 3}
|
||||
checked(t.Context(), link.BuildResult{ID: "stopped", Ref: "abc", Checked: asked, Failed: link.SupersededByKey})
|
||||
checked(t.Context(), link.BuildResult{ID: "broken", Ref: "abc", Checked: asked, Failed: "the facts snapshot cannot be read"})
|
||||
select {
|
||||
case m := <-heard:
|
||||
var c link.Checked
|
||||
_ = json.Unmarshal(m.Data, &c)
|
||||
if c.ID != "broken" || c.Verdict != "error" {
|
||||
t.Fatalf("the first verdict said is %+v; want only the broken check's error", c)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("nothing was said")
|
||||
}
|
||||
select {
|
||||
case m := <-heard:
|
||||
t.Fatalf("a second verdict was said: %s", m.Data)
|
||||
case <-time.After(300 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
@@ -1,97 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// Acceptance 5 of milestone 2 (novox/hq ADR 0406 §3, issue 974), live, with the store and a real bus: two merges into one
|
||||
// repository within a minute, with two open pull requests, leave each pull request with exactly one queued check, by the
|
||||
// stream's rule. Each merge moves the trunk, and mesh-delivery asks the controller to check both open pull requests again
|
||||
// against it (`delivery check`, which is the controller's own path for a head announced); nothing moves a request, nothing
|
||||
// asks the forge, and the queue is read back whole.
|
||||
func TestTwoMergesWithinAMinuteLeaveEachOpenPullRequestOneQueuedCheck(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
js, _ := aBuildQueue(t)
|
||||
servingBus.Store(js)
|
||||
t.Cleanup(func() { servingBus.Store(nil) })
|
||||
holdTheDeliverySeat(t, open)
|
||||
was := askDeliveryOwner
|
||||
t.Cleanup(func() { askDeliveryOwner = was })
|
||||
askDeliveryOwner = func(_ context.Context, _ *nats.Conn, verb string, _ map[string]any) (json.RawMessage, error) {
|
||||
if verb != "groups" {
|
||||
t.Fatalf("asked the owner %s", verb)
|
||||
}
|
||||
return json.RawMessage(`[]`), nil
|
||||
}
|
||||
// The repository the two pull requests are into, as a module the mesh builds from it.
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-controller", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
heads := map[int]string{7: "7777777777777777777777777777777777777777", 8: "8888888888888888888888888888888888888888"}
|
||||
check := func(number int, trunk string) {
|
||||
t.Helper()
|
||||
p := orderMember{ID: fmt.Sprintf("novox/mesh-controller@%.12s", heads[number]), Repository: "novox/mesh-controller",
|
||||
Base: "main", Head: heads[number], Number: number, Paths: []string{"main.go"}, ModuleDirsSaid: true,
|
||||
BaseSHA: trunk, Ready: number == 8}.pull()
|
||||
p.MergeCheck, p.MergeCheckSaid = true, true
|
||||
if err := (following{open: open, oneShot: true}).PullUpdated(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// Announced against the trunk as it stands, then asked again after each of two merges.
|
||||
for _, trunk := range []string{"1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222",
|
||||
"3333333333333333333333333333333333333333"} {
|
||||
check(7, trunk)
|
||||
check(8, trunk)
|
||||
}
|
||||
|
||||
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seen := map[int]string{}
|
||||
for _, a := range q.Asks {
|
||||
c := a.Request.Check
|
||||
if c == nil {
|
||||
t.Fatalf("%s is no check", a.ID)
|
||||
}
|
||||
if _, twice := seen[c.Number]; twice {
|
||||
t.Errorf("pull request %d has two queued checks", c.Number)
|
||||
}
|
||||
seen[c.Number] = c.Trunk
|
||||
if a.State != link.AskWaiting {
|
||||
t.Errorf("%s is %s", a.ID, a.State)
|
||||
}
|
||||
if want := "3333333333333333333333333333333333333333"; c.Trunk != want {
|
||||
t.Errorf("pull request %d is queued against %.8s; want the newest trunk %.8s", c.Number, c.Trunk, want)
|
||||
}
|
||||
}
|
||||
if len(seen) != 2 {
|
||||
t.Fatalf("queued checks for %v; want one for each of the two open pull requests", seen)
|
||||
}
|
||||
// By the streams' own count: the ready one on its stream, the other on its.
|
||||
for stream, want := range map[string]uint64{broker.ChecksReadyStream: 1, broker.ChecksOtherStream: 1} {
|
||||
info, err := js.Context().StreamInfo(stream)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.State.Msgs != want {
|
||||
t.Errorf("%s holds %d request(s); want %d", stream, info.State.Msgs, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -210,84 +210,3 @@ func TestABuildOffTheTrunkIsNeverPublished(t *testing.T) {
|
||||
t.Errorf("a build seat that said nothing was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// **A verdict names the trunk commit it judged** (novox/hq ADR 0323 §3, issue 341): the controller's `checked`
|
||||
// carries the head and the trunk commit the build seat merged it into, the commits of the repositories read beside
|
||||
// it, how long it took, and the kept delivery plan records the trunk commit with its other inputs. A group's
|
||||
// composed check names, per member, the trunk commit that head was merged into (§6).
|
||||
func TestAVerdictNamesTheTrunkCommitItJudged(t *testing.T) {
|
||||
plan := &link.ChangePlan{Repository: "novox/mesh-host", Base: "main", Head: "dfb1415aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
Summary: "builds mesh-host"}
|
||||
asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-host", Number: 66, Base: "main", Plan: plan}
|
||||
c := checkedOf(link.BuildResult{ID: "b", Ref: "dfb1415aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Checked: asked,
|
||||
Check: &link.CheckOutcome{Verdict: "pass", Took: "4m12s", Head: "dfb1415aaaaa", Trunk: "d28ca95bbbbb",
|
||||
Gate: &link.CheckLayer{Verdict: "pass"}, RepoCheck: &link.CheckLayer{Verdict: "pass"},
|
||||
Beside: map[string]string{"mesh-catalog": "0123456789ab", "mesh-lab": "ba9876543210"}}})
|
||||
if c.Trunk != "d28ca95bbbbb" || c.Commit != "dfb1415aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" {
|
||||
t.Fatalf("the verdict names head %s and trunk %q", c.Commit, c.Trunk)
|
||||
}
|
||||
if c.Beside["mesh-catalog"] != "0123456789ab" || c.Beside["mesh-lab"] != "ba9876543210" {
|
||||
t.Fatalf("the verdict names the repositories read beside it as %v", c.Beside)
|
||||
}
|
||||
if c.Took != "4m12s" {
|
||||
t.Fatalf("the verdict says it took %q", c.Took)
|
||||
}
|
||||
if c.Plan == nil || c.Plan.Trunk != "d28ca95bbbbb" || c.Plan.Head != plan.Head {
|
||||
t.Fatalf("the kept plan records %+v", c.Plan)
|
||||
}
|
||||
if plan.Trunk != "" {
|
||||
t.Fatal("the plan the controller asked with was changed: the verdict's is a copy")
|
||||
}
|
||||
|
||||
group := &link.CheckRequest{Owner: "novox", Repo: "mesh-controller", Number: 7, Base: "main", Group: "feat/x",
|
||||
Members: []link.GroupMember{{Owner: "novox", Repo: "mesh-host", Number: 8, Ref: "cccccccccccc0000", Base: "main"}}}
|
||||
c = checkedOf(link.BuildResult{ID: "g", Ref: "aaaaaaaaaaaa0000", Checked: group, Check: &link.CheckOutcome{
|
||||
Verdict: "pass", Head: "aaaaaaaaaaaa", Trunk: "111111111111", Gate: &link.CheckLayer{Verdict: "pass"},
|
||||
Members: []link.CheckedMember{{Owner: "novox", Repo: "mesh-host", Commit: "cccccccccccc", Trunk: "222222222222"}}}})
|
||||
if len(c.Members) != 2 || c.Members[0].Trunk != "111111111111" || c.Members[1].Trunk != "222222222222" ||
|
||||
c.Members[1].Commit != "cccccccccccc0000" {
|
||||
t.Fatalf("the composed check names its members %+v", c.Members)
|
||||
}
|
||||
}
|
||||
|
||||
// **The merge made for a check is never registered** (novox/hq ADR 0323 §2): a build of it — a commit on no branch
|
||||
// of the forge — is refused, naming the trunk. The rule of ADR 0238 §1 stands unchanged; this holds it for the
|
||||
// merge commit a check makes.
|
||||
func TestABuildOfACheckMergeIsRefusedNamingTheTrunk(t *testing.T) {
|
||||
merge := link.BuildResult{ID: "b", Commit: "4b5c6d7e8f90aabbccddeeff0011223344556677", Trunk: "main"}
|
||||
err := publishable(merge, "")
|
||||
if !errors.Is(err, errOffTheTrunk) || !strings.Contains(err.Error(), "not on main") {
|
||||
t.Fatalf("a build of a check's merge was answered %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// **A verdict no build judges names its trunk commit too** (novox/hq ADR 0323 §3): a change touching no module of a
|
||||
// repository with no merge-check.sh (novox/mesh-tools-go) is answered by the controller itself. Naming no trunk, the
|
||||
// merge tool refused it, asked again, got the same verdict, and refused for ever.
|
||||
func TestADirectVerdictNamesTheTrunkTheForgeAnnounced(t *testing.T) {
|
||||
p := link.PullUpdated{Owner: "novox", Repo: "mesh-tools-go", Number: 3, Base: "main", Commit: "1a2b3c4d5e6f7a8b9c0d",
|
||||
BaseSHA: "7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b"}
|
||||
c := directVerdict(p, link.ChangePlan{Repository: "novox/mesh-tools-go", Base: "main", Head: p.Commit})
|
||||
if c.Trunk != "7a8b9c0d1e2f" || c.Plan == nil || c.Plan.Trunk != "7a8b9c0d1e2f" {
|
||||
t.Fatalf("the direct verdict names trunk %q, its plan %+v", c.Trunk, c.Plan)
|
||||
}
|
||||
if c.Verdict != "pass" || c.Commit != p.Commit {
|
||||
t.Fatalf("the direct verdict is %+v", c)
|
||||
}
|
||||
if c := directVerdict(link.PullUpdated{Owner: "novox", Repo: "x", Commit: "abc"}, link.ChangePlan{}); c.Trunk != "" {
|
||||
t.Fatalf("a forge that said no base head gave the trunk %q", c.Trunk)
|
||||
}
|
||||
}
|
||||
|
||||
// A head checked again through mesh-delivery (`delivery check --members`, a recheck) reaches the same direct verdict:
|
||||
// it names the base head the forge last announced, kept on the member, or the merge tool would refuse it for ever.
|
||||
func TestARecheckedHeadsDirectVerdictNamesItsTrunk(t *testing.T) {
|
||||
m := orderMember{ID: "novox/mesh-tools-go@1a2b3c4d5e6f", Repository: "novox/mesh-tools-go", Base: "main",
|
||||
Head: "1a2b3c4d5e6f7a8b9c0d", Number: 3, BaseSHA: "7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b"}
|
||||
if c := directVerdict(m.pull(), link.ChangePlan{}); c.Trunk != "7a8b9c0d1e2f" {
|
||||
t.Fatalf("a rechecked head's direct verdict names trunk %q", c.Trunk)
|
||||
}
|
||||
}
|
||||
|
||||
// A check stopped by its pull request's key moving on says no verdict (issue 974, review 2): an error said for it would name the
|
||||
// live head, and the newer request is the one that judges. A check that failed for any other reason still says its error.
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A member of a group whose gate is the composed check's is asked of the build seat with no module: its repository's
|
||||
// check alone (novox/hq ADR 0403 clause 1). With no word from mesh-delivery, or one that does not say the gate is
|
||||
// composed, it is checked alone with its gate (clause 6).
|
||||
|
||||
func askedForMember(t *testing.T, answer []groupsLine, answerErr error) link.CheckRequest {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
js, _ := aBuildQueue(t)
|
||||
servingBus.Store(js)
|
||||
t.Cleanup(func() { servingBus.Store(nil) })
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-controller", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
was := groupsAsked
|
||||
groupsAsked = func(context.Context, bool) ([]groupsLine, error) { return answer, answerErr }
|
||||
t.Cleanup(func() { groupsAsked = was })
|
||||
p := link.PullUpdated{Owner: "novox", Repo: "mesh-controller", Number: 346, Base: "main", Head: "feat/x", Commit: headController,
|
||||
Paths: []string{"main.go"}, ModuleDirsSaid: true, MergeCheck: true, MergeCheckSaid: true}
|
||||
if err := (following{open: open, oneShot: true}).PullUpdated(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||
if err != nil || len(q.Asks) != 1 {
|
||||
t.Fatalf("asks %d: %v", len(q.Asks), err)
|
||||
}
|
||||
return *q.Asks[0].Request.Check
|
||||
}
|
||||
|
||||
func TestAComposedMembersCheckIsItsRepositoryChecksAlone(t *testing.T) {
|
||||
line := group("feat/x", member("novox/mesh-controller", 346, headController, "main", true), member("novox/mesh-catalog", 350, headHost, "main", true))
|
||||
line.ComposedGate = true
|
||||
c := askedForMember(t, []groupsLine{line}, nil)
|
||||
if len(c.Modules) != 0 || len(c.New) != 0 || len(c.Dependents) != 0 || !c.Composed {
|
||||
t.Fatalf("the member's check is asked as %+v; want no module and composed", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMemberOfAGroupOfAnOlderMeshDeliveryKeepsItsOwnGate(t *testing.T) {
|
||||
line := group("feat/x", member("novox/mesh-controller", 346, headController, "main", true), member("novox/mesh-catalog", 350, headHost, "main", true))
|
||||
c := askedForMember(t, []groupsLine{line}, nil)
|
||||
if !strings.Contains(strings.Join(c.Modules, ","), "mesh-controller") || c.Composed {
|
||||
t.Fatalf("an older mesh-delivery's member is asked as %+v; want its own gate", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMemberWhoseGroupCannotBeLearnedIsCheckedAloneWithItsGate(t *testing.T) {
|
||||
c := askedForMember(t, nil, context.DeadlineExceeded)
|
||||
if !strings.Contains(strings.Join(c.Modules, ","), "mesh-controller") || c.Composed ||
|
||||
!strings.Contains(c.SiblingsUnknown, "reads no siblings") {
|
||||
t.Fatalf("the member is asked as %+v; want its gate and why its siblings are unknown", c)
|
||||
}
|
||||
}
|
||||
@@ -1,136 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A delivery group's gate is the composed check's, and its members' own checks run their repository's check alone
|
||||
// (novox/hq ADR 0403 clauses 1 and 7). Pure: no store, no bus.
|
||||
|
||||
// A member of a group whose line says composed_gate is a composed member; one whose group says nothing of it (an older
|
||||
// mesh-delivery), one of a group of a single pull request and a lone pull request are not.
|
||||
func TestAMemberOfAComposedGroupIsKnownByTheGroupsLine(t *testing.T) {
|
||||
two := func(composed bool) groupsLine {
|
||||
l := group("feat/x", member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-controller", 346, headController, "main", true))
|
||||
l.ComposedGate = composed
|
||||
return l
|
||||
}
|
||||
p := link.PullUpdated{Owner: "novox", Repo: "mesh-host", Number: 114, Base: "main", Head: "feat/x", Commit: headHost}
|
||||
recheck := link.PullUpdated{Owner: "novox", Repo: "mesh-host", Number: 114, Base: "main", Commit: headHost}
|
||||
for name, c := range map[string]struct {
|
||||
p link.PullUpdated
|
||||
lines []groupsLine
|
||||
want bool
|
||||
}{
|
||||
"composed": {p, []groupsLine{two(true)}, true},
|
||||
"a recheck, by its number": {recheck, []groupsLine{two(true)}, true},
|
||||
"an older mesh-delivery": {p, []groupsLine{two(false)}, false},
|
||||
"another group": {p, []groupsLine{group("feat/y", member("novox/mesh-host", 1, headHost, "main", true))}, false},
|
||||
"no group": {p, nil, false},
|
||||
} {
|
||||
if got := composedMember(c.p, c.lines); got != c.want {
|
||||
t.Errorf("%s: composed %v, want %v", name, got, c.want)
|
||||
}
|
||||
}
|
||||
alone := group("feat/x", member("novox/mesh-host", 114, headHost, "main", true))
|
||||
alone.ComposedGate = true
|
||||
if composedMember(p, []groupsLine{alone}) {
|
||||
t.Error("a group of a single pull request is composed")
|
||||
}
|
||||
}
|
||||
|
||||
// A composed member's verdict never says its gate passed: the gate layer, and the verdict that is the gate's, read
|
||||
// "composed"; a failure or an error stays what it was.
|
||||
func TestAComposedMembersVerdictHasNoPassingGate(t *testing.T) {
|
||||
asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-host", Number: 114, Composed: true}
|
||||
pass := link.BuildResult{ID: "b", Ref: headHost, Checked: asked, Check: &link.CheckOutcome{Verdict: "pass", Summary: "no module touched",
|
||||
Gate: &link.CheckLayer{Verdict: "pass", Summary: "no module touched"}, RepoCheck: &link.CheckLayer{Verdict: "pass", Summary: "ok"}}}
|
||||
c := checkedOf(pass)
|
||||
if c.Gate == nil || c.Gate.Verdict != "composed" || c.Verdict != "composed" || c.RepoCheck == nil || c.RepoCheck.Verdict != "pass" {
|
||||
t.Fatalf("the member's verdict says %+v gate %+v", c, c.Gate)
|
||||
}
|
||||
asked.Composed = false
|
||||
if c := checkedOf(pass); c.Gate.Verdict != "pass" || c.Verdict != "pass" {
|
||||
t.Fatalf("a member of no composed group has its own gate: %+v", c.Gate)
|
||||
}
|
||||
asked.Composed = true
|
||||
failing := pass
|
||||
failing.Check = &link.CheckOutcome{Verdict: "fail", Summary: "conflict", Gate: &link.CheckLayer{Verdict: "fail", Summary: "conflict"},
|
||||
RepoCheck: &link.CheckLayer{Verdict: "fail", Summary: "conflict"}}
|
||||
if c := checkedOf(failing); c.Gate.Verdict != "fail" || c.Verdict != "fail" {
|
||||
t.Fatalf("a failure was read as composed: %+v", c.Gate)
|
||||
}
|
||||
broken := link.BuildResult{ID: "b", Ref: headHost, Checked: asked, Failed: "the facts snapshot cannot be read"}
|
||||
if c := checkedOf(broken); c.Verdict != "error" || c.Gate.Verdict != "error" {
|
||||
t.Fatalf("an error was read as composed: %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// The composed verdict on the bus carries every member, each with its trunk commit, for the forge's holder to set each
|
||||
// member's mesh/merge-gate from it.
|
||||
func TestTheComposedVerdictCarriesEveryMember(t *testing.T) {
|
||||
asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-host", Number: 114, Group: "feat/x",
|
||||
Members: []link.GroupMember{{Owner: "novox", Repo: "mesh-controller", Number: 346, Ref: headController}}}
|
||||
c := checkedOf(link.BuildResult{ID: "g", Ref: headHost, Checked: asked, Check: &link.CheckOutcome{Verdict: "pass", Summary: "ok",
|
||||
Trunk: "aaaaaaaaaaaa", Members: []link.CheckedMember{{Owner: "novox", Repo: "mesh-controller", Commit: headController[:12], Trunk: "bbbbbbbbbbbb"}}}})
|
||||
body, _ := json.Marshal(c)
|
||||
for _, want := range []string{`"group":"feat/x"`, `"members":[{"owner":"novox","repo":"mesh-host","number":114,"commit":"` + headHost + `","trunk":"aaaaaaaaaaaa"}`,
|
||||
`{"owner":"novox","repo":"mesh-controller","number":346,"commit":"` + headController + `","trunk":"bbbbbbbbbbbb"}`} {
|
||||
if !strings.Contains(string(body), want) {
|
||||
t.Fatalf("the composed verdict lacks %s:\n%s", want, body)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Members whose manifests order their modules in a cycle fail the composed check, naming every module and edge, and no
|
||||
// ask is made of the build seat.
|
||||
func TestAGroupInACycleFailsItsComposedCheckWithoutABuild(t *testing.T) {
|
||||
members := []orderMember{
|
||||
{ID: "novox/x@" + headHost[:12], Repository: "novox/x", Base: "main", Head: headHost, Number: 7, Paths: []string{"module.json"}},
|
||||
{ID: "novox/y@" + headController[:12], Repository: "novox/y", Base: "main", Head: headController, Number: 9, Paths: []string{"module.json"}},
|
||||
}
|
||||
reach := map[string]orderReach{members[0].ID: {Moved: []string{"mx"}}, members[1].ID: {Moved: []string{"my"}}}
|
||||
was, wasParent := manifestAt, parentOf
|
||||
t.Cleanup(func() { manifestAt, parentOf = was, wasParent })
|
||||
parentOf = func(context.Context, string, string, string) (string, error) { return "", nil }
|
||||
manifestAt = func(_ context.Context, _, repo, _, ref string) (catalogue.Manifest, bool, error) {
|
||||
if ref == "main" {
|
||||
return catalogue.Manifest{}, false, nil // what the pull requests merge into holds no such manifest
|
||||
}
|
||||
if repo == "x" {
|
||||
return catalogue.Manifest{Module: "mx", DefinesSeats: []catalogue.SeatDeclaration{{Name: "s", Serves: []catalogue.Verb{{Name: "a"}}}},
|
||||
Invokes: []string{"seat:s.b"}}, true, nil
|
||||
}
|
||||
return catalogue.Manifest{Module: "my", DefinesSeats: []catalogue.SeatDeclaration{{Name: "s", Serves: []catalogue.Verb{{Name: "b"}}}},
|
||||
Invokes: []string{"seat:s.a"}}, true, nil
|
||||
}
|
||||
c, cyclic, err := cycleVerdict(context.Background(), "feat/cycle", members, reach, []inventory.Edge(nil), nil)
|
||||
if err != nil || !cyclic {
|
||||
t.Fatalf("no cycle found: %v", err)
|
||||
}
|
||||
if c.Verdict != "fail" || c.Group != "feat/cycle" || len(c.Members) != 2 || c.Gate == nil || c.Gate.Verdict != "fail" {
|
||||
t.Fatalf("the verdict is %+v", c)
|
||||
}
|
||||
for _, w := range []string{"mx", "my", "s.a", "s.b"} {
|
||||
if !strings.Contains(c.Summary, w) {
|
||||
t.Errorf("the cycle does not name %s: %s", w, c.Summary)
|
||||
}
|
||||
}
|
||||
// Members with no edge between them are no cycle.
|
||||
manifestsReadMu.Lock()
|
||||
clear(manifestsRead)
|
||||
manifestsReadMu.Unlock()
|
||||
manifestAt = func(context.Context, string, string, string, string) (catalogue.Manifest, bool, error) {
|
||||
return catalogue.Manifest{}, false, nil
|
||||
}
|
||||
if _, cyclic, err := cycleVerdict(context.Background(), "feat/cycle", members, reach, nil, nil); err != nil || cyclic {
|
||||
t.Fatalf("a cycle where there is none: %v %v", cyclic, err)
|
||||
}
|
||||
}
|
||||
@@ -1,76 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The looks of the open conditions — how often each was observed, when last, its newest evidence — are
|
||||
// kept in the controller's store, not the conditions bucket (novox/hq issue 420, ADR 0289 §1): the bucket
|
||||
// is written when what a condition says changes, never for a look that saw it as it was.
|
||||
|
||||
// storeLooks keeps the conditions' looks in the inventory.
|
||||
type storeLooks struct{ inv *inventory.Inventory }
|
||||
|
||||
func (s storeLooks) KeepLooks(ctx context.Context, key string, l conditions.Looks) error {
|
||||
evidence, err := json.Marshal(l.Evidence)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.inv.KeepConditionLooks(ctx, inventory.ConditionLooks{Key: key, Observations: l.Observations,
|
||||
LastObserved: l.LastObserved, Evidence: evidence})
|
||||
}
|
||||
|
||||
func (s storeLooks) Looked(ctx context.Context, key string) (conditions.Looks, bool, error) {
|
||||
row, kept, err := s.inv.ConditionLooksOf(ctx, key)
|
||||
if err != nil || !kept {
|
||||
return conditions.Looks{}, kept, err
|
||||
}
|
||||
l, err := looksOf(row)
|
||||
return l, err == nil, err
|
||||
}
|
||||
|
||||
func (s storeLooks) AllLooks(ctx context.Context) (map[string]conditions.Looks, error) {
|
||||
rows, err := s.inv.AllConditionLooks(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(map[string]conditions.Looks, len(rows))
|
||||
for key, row := range rows {
|
||||
l, err := looksOf(row)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[key] = l
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s storeLooks) ForgetLooks(ctx context.Context, key string) error {
|
||||
return s.inv.ForgetConditionLooks(ctx, key)
|
||||
}
|
||||
|
||||
func looksOf(row inventory.ConditionLooks) (conditions.Looks, error) {
|
||||
l := conditions.Looks{Observations: row.Observations, LastObserved: row.LastObserved.UTC()}
|
||||
if len(row.Evidence) > 0 {
|
||||
if err := json.Unmarshal(row.Evidence, &l.Evidence); err != nil {
|
||||
return conditions.Looks{}, fmt.Errorf("the evidence kept for the condition %s cannot be read: %w", row.Key, err)
|
||||
}
|
||||
}
|
||||
return l, nil
|
||||
}
|
||||
|
||||
// looksHere is where a process that is not the serving controller reads the conditions' looks: the store,
|
||||
// opened for it, closed by the function returned. A store that cannot be opened is said on every condition
|
||||
// read through it, never read as none.
|
||||
func looksHere(ctx context.Context) (conditions.LookStore, func()) {
|
||||
inv, err := openInventory(ctx)
|
||||
if err != nil {
|
||||
return conditions.NoLooks{Err: err}, func() {}
|
||||
}
|
||||
return storeLooks{inv}, inv.Close
|
||||
}
|
||||
@@ -1,89 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// **Looks at an unchanged condition write the bucket once, and `conditions` still says how often and when
|
||||
// it was seen** (novox/hq issue 420): the looks are in the controller's store, and the verb, run as a
|
||||
// process of its own, reads them from there. On 2026-10-10 one open condition had rewritten the bucket 7 003
|
||||
// times in two days.
|
||||
func TestTheConditionsVerbReadsTheLooksTheBucketNoLongerHolds(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
bus := testbus.Start(t)
|
||||
serving, err := broker.Dial(bus.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer serving.Close()
|
||||
if err := serving.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := t.Context()
|
||||
store, history, err := conditions.OnTheBus(ctx, serving.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
k := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history, Looks: storeLooks{inv}})
|
||||
defer k.Close(context.Background())
|
||||
look := conditions.Observation{Scope: conditions.ScopeMachine, ID: "anchor", Kind: "silent", Machine: "anchor",
|
||||
Severity: conditions.Warning, Summary: "anchor has not been heard from", Source: "S1"}
|
||||
for range 4 {
|
||||
if _, err := k.Observe(ctx, look); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
api, err := jetstream.New(serving.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stream, err := api.Stream(ctx, "KV_"+broker.ConditionsBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := stream.Info(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.State.LastSeq != 1 {
|
||||
t.Fatalf("four looks at one unchanged condition wrote the bucket %d times, want 1", info.State.LastSeq)
|
||||
}
|
||||
|
||||
// The verb as a process other than the serving controller: it reads the bucket and the store.
|
||||
keptBefore := conditionsFrom
|
||||
conditionsFrom = nil
|
||||
defer func() { conditionsFrom = keptBefore }()
|
||||
t.Setenv(broker.NATSVar, bus.ClientURL())
|
||||
t.Setenv(broker.CertificateVar, "")
|
||||
var shown bytes.Buffer
|
||||
if err := showCondition(ctx, []string{"machine.anchor.silent", "--json"}, &shown); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var c conditions.Condition
|
||||
if err := json.Unmarshal(shown.Bytes(), &c); err != nil {
|
||||
t.Fatalf("%v: %s", err, shown.String())
|
||||
}
|
||||
if c.Observations != 4 || len(c.Evidence) != 4 || c.LastObserved.IsZero() || c.LooksUnread != "" ||
|
||||
time.Since(c.LastObserved) > time.Minute {
|
||||
t.Fatalf("conditions show reads %d observation(s), %d evidence line(s), last %s (%q)", c.Observations,
|
||||
len(c.Evidence), c.LastObserved, c.LooksUnread)
|
||||
}
|
||||
listed, err := openConditions(ctx)
|
||||
if err != nil || len(listed) != 1 || listed[0].Observations != 4 {
|
||||
t.Fatalf("conditions lists %+v: %v", listed, err)
|
||||
}
|
||||
}
|
||||
@@ -24,18 +24,16 @@ import (
|
||||
// **`status` used to be the only place the mesh said it was wrong, and only to whoever asked.** Every
|
||||
// core failure of research 031 was found by a person looking. A condition is the mesh saying it: raised
|
||||
// by a watchdog when a signal is late (signals.go), by a probe when an invariant does not hold
|
||||
// (doctor.go), or by an event a provider sends (standing.go); kept on the bus with since-when, and its
|
||||
// looks — how often, when last, the newest evidence — in the controller's store (condition_looks.go,
|
||||
// novox/hq issue 420); said on the bus as it changes, for the operator's channel to carry; and cleared when an
|
||||
// (doctor.go), or by an event a provider sends (standing.go); kept on the bus with since-when and
|
||||
// evidence; said on the bus as it changes, for the operator's channel to carry; and cleared when an
|
||||
// observation says it is resolved. Nobody resolves one by hand. A person who knows silences it, for a
|
||||
// while, with a reason, and that is recorded as a hand act.
|
||||
|
||||
// conditionsFrom is the serving controller's keeper; nil in any other process, which opens its own.
|
||||
var conditionsFrom *conditions.Keeper
|
||||
|
||||
// keeperOn is a keeper over the store on a connection, saying its transitions on that connection, with
|
||||
// the conditions' looks kept in looks (condition_looks.go).
|
||||
func keeperOn(ctx context.Context, conn *nats.Conn, looks conditions.LookStore) (*conditions.Keeper, error) {
|
||||
// keeperOn is a keeper over the store on a connection, saying its transitions on that connection.
|
||||
func keeperOn(ctx context.Context, conn *nats.Conn) (*conditions.Keeper, error) {
|
||||
store, history, err := conditions.OnTheBus(ctx, conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -44,12 +42,12 @@ func keeperOn(ctx context.Context, conn *nats.Conn, looks conditions.LookStore)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history, Looks: looks,
|
||||
return conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history,
|
||||
Teller: link.OverNATS{Conn: conn, JS: js},
|
||||
Say: func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) },
|
||||
// What status leads with changed: composed again soon (a nudge outside the serving controller
|
||||
// does nothing).
|
||||
Changed: func() { statusFrom.nudge(); askerFrom.nudge() },
|
||||
Changed: statusFrom.nudge,
|
||||
// Written under the lease, carrying its epoch (novox/hq to-be 45 §6).
|
||||
Epoch: func() (uint64, error) { return theLease.epoch(context.WithoutCancel(ctx)) }}), nil
|
||||
}
|
||||
@@ -60,10 +58,8 @@ func withKeeper(ctx context.Context, f func(*conditions.Keeper) error) error {
|
||||
if conditionsFrom != nil {
|
||||
return f(conditionsFrom)
|
||||
}
|
||||
looks, closeLooks := looksHere(ctx)
|
||||
defer closeLooks()
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
k, err := keeperOn(ctx, conn, looks)
|
||||
k, err := keeperOn(ctx, conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -86,8 +82,6 @@ func openConditions(ctx context.Context) ([]conditions.Condition, error) {
|
||||
return nil, fmt.Errorf("this process has no bus to read the conditions from: %w", err)
|
||||
}
|
||||
var out []conditions.Condition
|
||||
looks, closeLooks := looksHere(ctx)
|
||||
defer closeLooks()
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
store, _, err := conditions.OnTheBus(ctx, conn)
|
||||
if err != nil {
|
||||
@@ -95,7 +89,7 @@ func openConditions(ctx context.Context) ([]conditions.Condition, error) {
|
||||
}
|
||||
reading, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
out, err = conditions.ReadWith(reading, store, looks)
|
||||
out, err = conditions.Read(reading, store)
|
||||
return err
|
||||
})
|
||||
return out, err
|
||||
@@ -258,14 +252,12 @@ func showCondition(ctx context.Context, args []string, w io.Writer) error {
|
||||
if conditionsFrom != nil {
|
||||
c, found, err = conditionsFrom.Get(ctx, key)
|
||||
} else {
|
||||
looks, closeLooks := looksHere(ctx)
|
||||
defer closeLooks()
|
||||
err = onTheBus(func(conn *nats.Conn) error {
|
||||
store, _, err := conditions.OnTheBus(ctx, conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c, found, err = conditions.ReadOneWith(ctx, store, looks, key)
|
||||
c, found, err = conditions.ReadOne(ctx, store, key)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestAVerdictCarriesAProbesFindingsInBrief(t *testing.T) {
|
||||
var found []string
|
||||
for i := 0; i < 689; i++ {
|
||||
found = append(found, fmt.Sprintf("delivery d-%012x has been checking for 13h, past its bound of 45m", i))
|
||||
}
|
||||
run := doctorRun{Run: "run-1", Probes: []probeVerdict{{ID: "D14", Verdict: "failed", Found: found},
|
||||
{ID: "D1", Verdict: "passed"}}}
|
||||
answer, err := verdictAnswer(run, time.Now(), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
brief := answer["run"].(doctorRun).Probes[0]
|
||||
if len(brief.Found) != overviewFindings || brief.FoundCount != 689 || !strings.Contains(brief.More, "probe=D14") {
|
||||
t.Fatalf("the verdict carries %d findings of D14, count %d, more %q", len(brief.Found), brief.FoundCount, brief.More)
|
||||
}
|
||||
if len(run.Probes[0].Found) != 689 {
|
||||
t.Fatal("the overview cut the run it was made from")
|
||||
}
|
||||
whole, err := verdictAnswer(run, time.Now(), "d14")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p := whole["run"].(doctorRun).Probes; len(p) != 1 || len(p[0].Found) != 689 || p[0].More != "" {
|
||||
t.Fatalf("doctor probe=D14 answered %+v", p)
|
||||
}
|
||||
if _, err := verdictAnswer(run, time.Now(), "D99"); err == nil {
|
||||
t.Fatal("a probe the run does not have was answered")
|
||||
}
|
||||
if !strings.Contains(doctorText(answer), "probe=D14` gives them all") {
|
||||
t.Fatalf("the text does not say how to read them all:\n%s", doctorText(answer))
|
||||
}
|
||||
}
|
||||
@@ -74,3 +74,35 @@ func TestAListOfConditionsCarriesEachInBrief(t *testing.T) {
|
||||
|
||||
// The self-check's verdict carries at most overviewFindings of one probe's findings, with their count and
|
||||
// how to read them all; `doctor probe=` gives that probe whole (novox/hq issue 314).
|
||||
func TestAVerdictCarriesAProbesFindingsInBrief(t *testing.T) {
|
||||
var found []string
|
||||
for i := 0; i < 689; i++ {
|
||||
found = append(found, fmt.Sprintf("delivery d-%012x has been checking for 13h, past its bound of 45m", i))
|
||||
}
|
||||
run := doctorRun{Run: "run-1", Probes: []probeVerdict{{ID: "D14", Verdict: "failed", Found: found},
|
||||
{ID: "D1", Verdict: "passed"}}}
|
||||
answer, err := verdictAnswer(run, time.Now(), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
brief := answer["run"].(doctorRun).Probes[0]
|
||||
if len(brief.Found) != overviewFindings || brief.FoundCount != 689 || !strings.Contains(brief.More, "probe=D14") {
|
||||
t.Fatalf("the verdict carries %d findings of D14, count %d, more %q", len(brief.Found), brief.FoundCount, brief.More)
|
||||
}
|
||||
if len(run.Probes[0].Found) != 689 {
|
||||
t.Fatal("the overview cut the run it was made from")
|
||||
}
|
||||
whole, err := verdictAnswer(run, time.Now(), "d14")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p := whole["run"].(doctorRun).Probes; len(p) != 1 || len(p[0].Found) != 689 || p[0].More != "" {
|
||||
t.Fatalf("doctor probe=D14 answered %+v", p)
|
||||
}
|
||||
if _, err := verdictAnswer(run, time.Now(), "D99"); err == nil {
|
||||
t.Fatal("a probe the run does not have was answered")
|
||||
}
|
||||
if !strings.Contains(doctorText(answer), "probe=D14` gives them all") {
|
||||
t.Fatalf("the text does not say how to read them all:\n%s", doctorText(answer))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,32 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestUnreadableConditionsAreNotAWellMesh(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
_, store := withConditionsInMemory(t)
|
||||
store.SetFail(errors.New("the bus is away"))
|
||||
asked, err := theThreeQuestions(t.Context(), open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if asked.well() || asked.conditionsUnread == "" {
|
||||
t.Fatalf("well with its conditions unread: %+v", asked.conditionsUnread)
|
||||
}
|
||||
said := printed(t, func() error { return printStatus(asked) })
|
||||
if !strings.HasPrefix(said, "the open conditions could NOT be read") || strings.Contains(said, "no open conditions") {
|
||||
t.Fatalf("%s", said)
|
||||
}
|
||||
store.SetFail(nil)
|
||||
asked, _ = theThreeQuestions(t.Context(), open)
|
||||
said = printed(t, func() error { return printStatus(asked) })
|
||||
if asked.well() && !strings.Contains(said, "no open conditions;") {
|
||||
t.Fatalf("the all-well sentence does not say no conditions are open:\n%s", said)
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -82,3 +83,25 @@ func TestASilenceThroughTheVerbHoldsAndTheConditionStaysOpen(t *testing.T) {
|
||||
}
|
||||
|
||||
// **Conditions that cannot be read are not none open**: status says so, and is not well.
|
||||
func TestUnreadableConditionsAreNotAWellMesh(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
_, store := withConditionsInMemory(t)
|
||||
store.Fail = errors.New("the bus is away")
|
||||
asked, err := theThreeQuestions(t.Context(), open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if asked.well() || asked.conditionsUnread == "" {
|
||||
t.Fatalf("well with its conditions unread: %+v", asked.conditionsUnread)
|
||||
}
|
||||
said := printed(t, func() error { return printStatus(asked) })
|
||||
if !strings.HasPrefix(said, "the open conditions could NOT be read") || strings.Contains(said, "no open conditions") {
|
||||
t.Fatalf("%s", said)
|
||||
}
|
||||
store.Fail = nil
|
||||
asked, _ = theThreeQuestions(t.Context(), open)
|
||||
said = printed(t, func() error { return printStatus(asked) })
|
||||
if asked.well() && !strings.Contains(said, "no open conditions;") {
|
||||
t.Fatalf("the all-well sentence does not say no conditions are open:\n%s", said)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,616 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **The move of config** (novox/hq ADR 0348 §9 and its consequences, design 57 K5, issue 664): every config value a
|
||||
// settings layer holds is set in mesh-config, one of the moves of data a release runs once after its delivery, after
|
||||
// the move of placement. A module's layer on a node becomes its own value on that node (`module.<node>.<module>.<key>`).
|
||||
// A whole-mesh layer has no module value for the whole mesh (ADR 0348 §3): its value becomes mesh config for the
|
||||
// predefined key it falls back to where that module alone falls back to the key (claude-code's managed settings
|
||||
// become `agent-managed-settings`), and otherwise the module's own value on each node it is assigned to, laid under
|
||||
// that node's layer as composition lays it. A key a layer still says by its file's name (nodered's `timeZone`) is
|
||||
// set under its config key (`time-zone`, issue 751).
|
||||
//
|
||||
// **Every node composes byte for byte as before.** The move sets the values, reads every module's resolved values
|
||||
// back through mesh-config and compares them with what the layers lay today; only when they agree does it empty the
|
||||
// layers it moved (each kept whole in its history first) and turn on the switch (inventory.FinishConfigMove), after
|
||||
// which the controller reads mesh-config where it read the layers. A module still in the settings form keeps its
|
||||
// layers and is said so. Anything that cannot move as it composes refuses the whole move, by name, before anything
|
||||
// is set. Asked again, it sets nothing that is set already.
|
||||
|
||||
// configLevel is where mesh-config keeps a value: mesh config, node config, or a module's own value on a node.
|
||||
type configLevel struct {
|
||||
Level string `json:"level"`
|
||||
Node string `json:"node,omitempty"`
|
||||
Module string `json:"module,omitempty"`
|
||||
}
|
||||
|
||||
func (l configLevel) valueKey(key string) string {
|
||||
switch l.Level {
|
||||
case "node":
|
||||
return "node." + l.Node + "." + key
|
||||
case "module":
|
||||
return "module." + l.Node + "." + l.Module + "." + key
|
||||
}
|
||||
return "mesh." + key
|
||||
}
|
||||
|
||||
func (l configLevel) String() string {
|
||||
switch l.Level {
|
||||
case "node":
|
||||
return "node config on " + l.Node
|
||||
case "module":
|
||||
return l.Module + "'s own value on " + l.Node
|
||||
}
|
||||
return "mesh config"
|
||||
}
|
||||
|
||||
// configSet is one value the move sets in mesh-config, and the layers it comes from.
|
||||
type configSet struct {
|
||||
configLevel
|
||||
Key string `json:"key"`
|
||||
Value any `json:"value"`
|
||||
Trusted bool `json:"trusted,omitempty"`
|
||||
// From is the layers it comes from, as `<module> on <where>: <key as the layer says it>`.
|
||||
From []string `json:"from"`
|
||||
// Already is true when mesh-config holds this value at this level already: nothing is set.
|
||||
Already bool `json:"already,omitempty"`
|
||||
}
|
||||
|
||||
// configMoveAnswer is what the move of config says it did, or would do.
|
||||
type configMoveAnswer struct {
|
||||
// DryRun is true when nothing was set: asked without --confirm.
|
||||
DryRun bool `json:"dry-run"`
|
||||
Sets []configSet `json:"sets"`
|
||||
// Moved is each layer whose values the move sets, emptied once they read back.
|
||||
Moved []string `json:"moved"`
|
||||
// Stays is each layer of a module still in the settings form: kept, and composed from as before.
|
||||
Stays []string `json:"stays,omitempty"`
|
||||
// Unread is each layer that composes nothing (its module is not assigned where it says): kept in the history,
|
||||
// emptied, and set nowhere.
|
||||
Unread []string `json:"unread,omitempty"`
|
||||
// Problems is why the move refuses, each by name; nothing is set while there is one.
|
||||
Problems []string `json:"problems,omitempty"`
|
||||
// Switched is true once the controller reads mesh-config where it read the layers.
|
||||
Switched bool `json:"switched"`
|
||||
}
|
||||
|
||||
// configMovePlan is the move worked out from the layers, before anything is asked of mesh-config.
|
||||
type configMovePlan struct {
|
||||
answer configMoveAnswer
|
||||
// moved is the layers the move empties once the values read back.
|
||||
moved []inventory.LayerToMove
|
||||
// expected is, for each module the move reaches, keyed `<node>.<module>`, every value composition lays over
|
||||
// its files today: what mesh-config must resolve for it from mesh config, node config or its own value.
|
||||
expected map[string]map[string]any
|
||||
// modules is each module in the config form, by the nodes it is assigned to: whose resolved values are read back.
|
||||
modules map[string][]string
|
||||
}
|
||||
|
||||
func whereOf(node string) string {
|
||||
if node == "" {
|
||||
return "the whole mesh"
|
||||
}
|
||||
return node
|
||||
}
|
||||
|
||||
// planConfigMove works out the move from every layer that holds a value, the catalogue, and where each module is
|
||||
// assigned.
|
||||
func planConfigMove(layers []inventory.LayerToMove, shelf map[string]catalogue.Manifest,
|
||||
on map[string][]string) configMovePlan {
|
||||
p := configMovePlan{answer: configMoveAnswer{Sets: []configSet{}, Moved: []string{}},
|
||||
expected: map[string]map[string]any{}, modules: map[string][]string{}}
|
||||
problem := func(format string, a ...any) {
|
||||
p.answer.Problems = append(p.answer.Problems, fmt.Sprintf(format, a...))
|
||||
}
|
||||
for name, m := range shelf {
|
||||
if m.DeclaresConfig() {
|
||||
p.modules[name] = on[name]
|
||||
}
|
||||
}
|
||||
|
||||
byModule := map[string]*layersOfModule{}
|
||||
var names []string
|
||||
for _, l := range layers {
|
||||
said := fmt.Sprintf("%s on %s", l.Module, whereOf(l.Node))
|
||||
m, known := shelf[l.Module]
|
||||
if !known || !m.DeclaresConfig() {
|
||||
p.answer.Stays = append(p.answer.Stays, said+" (its module is in the settings form)")
|
||||
continue
|
||||
}
|
||||
if _, placement := catalogue.SplitPlacement(l.Values); len(placement) > 0 {
|
||||
problem("%s still holds placement (%s): run move-placement first", said, strings.Join(movedKeys(placement), ", "))
|
||||
continue
|
||||
}
|
||||
assigned := on[l.Module]
|
||||
if (l.Node == "" && len(assigned) == 0) || (l.Node != "" && !containsString(assigned, l.Node)) {
|
||||
p.answer.Unread = append(p.answer.Unread, said+" (the module is not assigned there, so it composes nothing)")
|
||||
p.moved = append(p.moved, l)
|
||||
continue
|
||||
}
|
||||
// The layer's keys as config keys: a key the file names otherwise is set under its config key.
|
||||
as := catalogue.TakenAs(m)
|
||||
values := map[string]any{}
|
||||
from := map[string]string{}
|
||||
for _, k := range movedKeys(l.Values) {
|
||||
key := k
|
||||
if _, declared := m.Settings[k]; !declared {
|
||||
if renamed, ok := as[k]; ok {
|
||||
key = renamed
|
||||
} else {
|
||||
problem("%s says %q, which %s does not declare: declare it, or clear it from the layer", said, k, l.Module)
|
||||
continue
|
||||
}
|
||||
}
|
||||
if _, both := values[key]; both {
|
||||
// The layer says the key and the file's name for it: the config key wins, as composition has it (issue 751).
|
||||
if key != k {
|
||||
continue
|
||||
}
|
||||
}
|
||||
values[key] = l.Values[k]
|
||||
from[key] = said + ": " + k
|
||||
}
|
||||
om := byModule[l.Module]
|
||||
if om == nil {
|
||||
om = &layersOfModule{nodes: map[string]map[string]any{}, saidAt: map[string]map[string]string{}}
|
||||
byModule[l.Module] = om
|
||||
names = append(names, l.Module)
|
||||
}
|
||||
om.saidAt[l.Node] = from
|
||||
if l.Node == "" {
|
||||
om.mesh = values
|
||||
} else {
|
||||
om.nodes[l.Node] = values
|
||||
}
|
||||
p.moved = append(p.moved, l)
|
||||
p.answer.Moved = append(p.answer.Moved, said)
|
||||
}
|
||||
sort.Strings(names)
|
||||
|
||||
// Which predefined key each module falls back to, so a whole-mesh value goes to mesh config only where no other
|
||||
// module's value moves with it.
|
||||
fallingBack := map[string][]string{}
|
||||
for name, m := range shelf {
|
||||
for _, d := range m.Settings {
|
||||
if d.Predefined != "" && !containsString(fallingBack[d.Predefined], name) {
|
||||
fallingBack[d.Predefined] = append(fallingBack[d.Predefined], name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, name := range names {
|
||||
m, om := shelf[name], byModule[name]
|
||||
asked := catalogue.AskedKeys(m)
|
||||
keys := map[string]bool{}
|
||||
for k := range om.mesh {
|
||||
keys[k] = true
|
||||
}
|
||||
for _, values := range om.nodes {
|
||||
for k := range values {
|
||||
keys[k] = true
|
||||
}
|
||||
}
|
||||
for _, key := range movedKeys(keys) {
|
||||
d := m.Settings[key]
|
||||
meshValue, meshSays := om.mesh[key]
|
||||
alone := d.Predefined != "" && len(fallingBack[d.Predefined]) == 1
|
||||
if meshSays && alone {
|
||||
p.set(configLevel{Level: "mesh"}, d.Predefined, meshValue, d, om.said("", key))
|
||||
}
|
||||
for _, node := range on[name] {
|
||||
nodeValue, nodeSays := om.nodes[node][key]
|
||||
value := meshValue
|
||||
switch {
|
||||
case nodeSays && meshSays:
|
||||
_, under := meshValue.(map[string]any)
|
||||
_, over := nodeValue.(map[string]any)
|
||||
if under && over && asked[key] {
|
||||
problem("%s on %s lays an object for %q over the whole mesh's, and a placeholder takes it whole "+
|
||||
"where its file merges the two: say the value on %s whole, then move", name, node, key, node)
|
||||
continue
|
||||
}
|
||||
value = catalogue.LaidOver(meshValue, nodeValue)
|
||||
case nodeSays:
|
||||
value = nodeValue
|
||||
case !meshSays:
|
||||
continue
|
||||
}
|
||||
p.expect(node, name, key, value)
|
||||
if nodeSays || !alone {
|
||||
saidBy := om.said(node, key)
|
||||
if meshSays {
|
||||
saidBy = append(om.said("", key), saidBy...)
|
||||
}
|
||||
p.set(configLevel{Level: "module", Node: node, Module: name}, key, value, d, saidBy)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(p.answer.Problems)
|
||||
return p
|
||||
}
|
||||
|
||||
// layersOfModule is one module's layers as the move reads them: its whole-mesh layer's config values, each node's,
|
||||
// and where each key was said, by node ("" for the whole mesh).
|
||||
type layersOfModule struct {
|
||||
mesh map[string]any
|
||||
nodes map[string]map[string]any
|
||||
saidAt map[string]map[string]string
|
||||
}
|
||||
|
||||
func (om *layersOfModule) said(node, key string) []string {
|
||||
if s, ok := om.saidAt[node][key]; ok {
|
||||
return []string{s}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// set is one value the move sets, judged against its declaration as mesh-config will judge it.
|
||||
func (p *configMovePlan) set(level configLevel, key string, value any, d catalogue.SettingDeclaration, from []string) {
|
||||
p.answer.Sets = append(p.answer.Sets, configSet{configLevel: level, Key: key, Value: value, Trusted: d.Trusted,
|
||||
From: from})
|
||||
for _, wrong := range catalogue.ValueProblems(d, value) {
|
||||
p.answer.Problems = append(p.answer.Problems, fmt.Sprintf("%s at %s: %s", key, level, wrong))
|
||||
}
|
||||
}
|
||||
|
||||
func (p *configMovePlan) expect(node, module, key string, value any) {
|
||||
k := node + "." + module
|
||||
if p.expected[k] == nil {
|
||||
p.expected[k] = map[string]any{}
|
||||
}
|
||||
p.expected[k][key] = value
|
||||
}
|
||||
|
||||
func movedKeys[V any](m map[string]V) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// configService is mesh-config as the move asks it: what it holds at each level, what it resolved, and a value set
|
||||
// as the controller carries it from its terminal.
|
||||
type configService interface {
|
||||
// Values is every value set, by its key in mesh-config_values (`mesh.<key>`, `node.<node>.<key>`,
|
||||
// `module.<node>.<module>.<key>`).
|
||||
Values(ctx context.Context) (map[string]json.RawMessage, error)
|
||||
// Resolved is every resolved value.
|
||||
Resolved(ctx context.Context) ([]resolvedEntry, error)
|
||||
// Set sets one value at its level, by whom and why.
|
||||
Set(ctx context.Context, level configLevel, key string, value any, by, why string) error
|
||||
}
|
||||
|
||||
// runConfigMove is the move against a store and mesh-config: a dry run says what it would set.
|
||||
func runConfigMove(ctx context.Context, inv *inventory.Inventory, config configService, apply bool, by,
|
||||
why string) (configMoveAnswer, error) {
|
||||
layers, holding, err := inv.LayersToMove(ctx)
|
||||
if err != nil {
|
||||
return configMoveAnswer{}, err
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return configMoveAnswer{}, err
|
||||
}
|
||||
shelf := map[string]catalogue.Manifest{}
|
||||
on := map[string][]string{}
|
||||
for _, e := range entries {
|
||||
shelf[e.Manifest.Module] = e.Manifest
|
||||
on[e.Manifest.Module] = e.On
|
||||
}
|
||||
p := planConfigMove(layers, shelf, on)
|
||||
if len(holding) > 0 && len(p.answer.Problems) == 0 {
|
||||
p.answer.Problems = append(p.answer.Problems, "a layer still holds placement ("+strings.Join(holding, "; ")+
|
||||
"): run move-placement first")
|
||||
}
|
||||
moved, err := inv.ConfigMoved(ctx)
|
||||
if err != nil {
|
||||
return configMoveAnswer{}, err
|
||||
}
|
||||
p.answer.Switched = moved.Moved
|
||||
if config == nil {
|
||||
return p.answer, errors.New("mesh-config cannot be asked, so nothing was moved")
|
||||
}
|
||||
held, err := config.Values(ctx)
|
||||
if err != nil {
|
||||
return p.answer, fmt.Errorf("what mesh-config holds cannot be read, so nothing was moved: %w", err)
|
||||
}
|
||||
for i := range p.answer.Sets {
|
||||
s := &p.answer.Sets[i]
|
||||
if was, ok := held[s.valueKey(s.Key)]; ok && sameJSON(was, s.Value) {
|
||||
s.Already = true
|
||||
}
|
||||
}
|
||||
p.answer.DryRun = !apply
|
||||
if !apply || len(p.answer.Problems) > 0 {
|
||||
if apply {
|
||||
return p.answer, fmt.Errorf("the move of config refuses, and nothing was set: %s",
|
||||
strings.Join(p.answer.Problems, "; "))
|
||||
}
|
||||
return p.answer, nil
|
||||
}
|
||||
if len(p.moved) == 0 && moved.Moved {
|
||||
return p.answer, nil
|
||||
}
|
||||
for _, s := range p.answer.Sets {
|
||||
if s.Already {
|
||||
continue
|
||||
}
|
||||
said := "moved from the settings layer of " + strings.Join(s.From, " and ") + " (issue 664): " + why
|
||||
if err := config.Set(ctx, s.configLevel, s.Key, s.Value, by, said); err != nil {
|
||||
return p.answer, fmt.Errorf("%s at %s was not set, and no layer was emptied; ask the move again: %w",
|
||||
s.Key, s.configLevel, err)
|
||||
}
|
||||
}
|
||||
// Read back through mesh-config before anything points at it (design 57): every module the move reaches, and
|
||||
// every other in the config form, must resolve exactly what its layers lay today.
|
||||
if differs, err := readsBack(ctx, config, p); err != nil {
|
||||
return p.answer, err
|
||||
} else if len(differs) > 0 {
|
||||
return p.answer, fmt.Errorf("mesh-config does not resolve what the layers lay today, so no layer was emptied and "+
|
||||
"the controller still composes from the layers: %s", strings.Join(differs, "; "))
|
||||
}
|
||||
if err := inv.FinishConfigMove(ctx, p.moved, by, why); err != nil {
|
||||
return p.answer, err
|
||||
}
|
||||
p.answer.Switched = true
|
||||
return p.answer, nil
|
||||
}
|
||||
|
||||
// readsBack compares what mesh-config resolved with what the layers lay, for every module in the config form on
|
||||
// every node it is assigned to, and says each difference. A value the layers do not lay and mesh-config resolves from
|
||||
// a level would be laid once the switch is on: a difference too.
|
||||
func readsBack(ctx context.Context, config configService, p configMovePlan) ([]string, error) {
|
||||
var resolved []resolvedEntry
|
||||
var err error
|
||||
// mesh-config writes the resolved values with each set; a pass that missed one is put right by its next
|
||||
// resolution, so a difference is read again for a little while before it is said.
|
||||
for wait := 0; ; wait++ {
|
||||
if resolved, err = config.Resolved(ctx); err != nil {
|
||||
return nil, fmt.Errorf("mesh-config's resolved values cannot be read back, so no layer was emptied: %w", err)
|
||||
}
|
||||
differs, err := differences(resolved, p)
|
||||
if err != nil || len(differs) == 0 || wait >= readBackTries {
|
||||
return differs, err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(readBackEvery):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// How often, and how many times, the move reads mesh-config's resolved values back before it says they differ.
|
||||
var (
|
||||
readBackEvery = 2 * time.Second
|
||||
readBackTries = 15
|
||||
)
|
||||
|
||||
func differences(resolved []resolvedEntry, p configMovePlan) ([]string, error) {
|
||||
var differs []string
|
||||
for _, module := range movedKeys(p.modules) {
|
||||
for _, node := range p.modules[module] {
|
||||
laid, err := configLaid(resolved, node, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
want := p.expected[node+"."+module]
|
||||
if want == nil {
|
||||
want = map[string]any{}
|
||||
}
|
||||
for _, k := range movedKeys(mergedKeys(laid, want)) {
|
||||
got, has := laid[k]
|
||||
w, wants := want[k]
|
||||
switch {
|
||||
case !wants:
|
||||
differs = append(differs, fmt.Sprintf("%s on %s resolves %s from a level no layer says; clear it "+
|
||||
"there, or set it in a layer first", module, node, k))
|
||||
case !has:
|
||||
differs = append(differs, fmt.Sprintf("%s on %s does not resolve %s from the value moved", module, node, k))
|
||||
case !reflect.DeepEqual(normalJSON(got), normalJSON(w)):
|
||||
differs = append(differs, fmt.Sprintf("%s on %s resolves %s as %s, and its layers lay %s", module,
|
||||
node, k, compactJSON(got), compactJSON(w)))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return differs, nil
|
||||
}
|
||||
|
||||
func mergedKeys(a, b map[string]any) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for k := range a {
|
||||
out[k] = true
|
||||
}
|
||||
for k := range b {
|
||||
out[k] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func normalJSON(v any) any {
|
||||
raw, _ := json.Marshal(v)
|
||||
var out any
|
||||
_ = json.Unmarshal(raw, &out)
|
||||
return out
|
||||
}
|
||||
|
||||
func compactJSON(v any) string {
|
||||
raw, _ := json.Marshal(v)
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
func sameJSON(raw json.RawMessage, v any) bool {
|
||||
var held any
|
||||
if err := json.Unmarshal(raw, &held); err != nil {
|
||||
return false
|
||||
}
|
||||
return reflect.DeepEqual(held, normalJSON(v))
|
||||
}
|
||||
|
||||
// busConfig is mesh-config on this controller's bus: its buckets read directly, a value set through its verb in
|
||||
// the controller's own name, so mesh-config takes a trusted value as given at the controller's terminal (ADR 0348 §8).
|
||||
type busConfig struct{}
|
||||
|
||||
func (busConfig) Values(ctx context.Context) (map[string]json.RawMessage, error) {
|
||||
out := map[string]json.RawMessage{}
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
|
||||
defer cancel()
|
||||
kv, err := js.KeyValue(ctx, broker.BucketName("mesh-config", "values"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w, err := kv.WatchAll(ctx, jetstream.IgnoreDeletes())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = w.Stop() }()
|
||||
for e := range w.Updates() {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
var v struct {
|
||||
Value json.RawMessage `json:"value"`
|
||||
}
|
||||
if err := json.Unmarshal(e.Value(), &v); err != nil {
|
||||
return fmt.Errorf("mesh-config holds %q, which is not a value: %w", e.Key(), err)
|
||||
}
|
||||
out[e.Key()] = v.Value
|
||||
}
|
||||
return errors.New("mesh-config's values stopped answering")
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
func (busConfig) Resolved(ctx context.Context) ([]resolvedEntry, error) { return readResolved(ctx) }
|
||||
|
||||
func (busConfig) Set(ctx context.Context, level configLevel, key string, value any, by, why string) error {
|
||||
args := map[string]any{"action": "set", "key": key, "value": value, "why": why, "by": by}
|
||||
if level.Node != "" {
|
||||
args["node"] = level.Node
|
||||
}
|
||||
if level.Module != "" {
|
||||
args["module"] = level.Module
|
||||
}
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
answer, err := link.AskMeshSeatToolAs(ctx, conn, broker.ControllerName, "mesh-config", "config", args, 30*time.Second)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return errors.New(answer.Error)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// moveConfigCommand is `move-config`: the move of config (issue 664).
|
||||
func moveConfigCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("move-config", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "answer as JSON")
|
||||
confirm := set.Bool("confirm", false, "set the values and switch, rather than only say what would move")
|
||||
f := addHandActFlags(set)
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 0 {
|
||||
return errors.New("move-config [--json] [--confirm --why <text>]")
|
||||
}
|
||||
if !*confirm && f.given() {
|
||||
return errors.New("move-config takes --why only with --confirm: without it nothing is moved. Nothing was done")
|
||||
}
|
||||
if *confirm {
|
||||
if err := f.require("move-config"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
if *confirm {
|
||||
f.record(ctx, "move-config", nil)
|
||||
}
|
||||
why := ""
|
||||
if *confirm {
|
||||
why = strings.TrimSpace(*f.why)
|
||||
}
|
||||
answer, err := runConfigMove(ctx, open.inventory, busConfig{}, *confirm, "the move of config, run by "+link.Caller(), why)
|
||||
if *asJSON {
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
if encErr := encoder.Encode(answer); encErr != nil {
|
||||
return encErr
|
||||
}
|
||||
return err
|
||||
}
|
||||
fmt.Print(describeConfigMove(answer))
|
||||
return err
|
||||
}
|
||||
|
||||
// describeConfigMove says the answer as a person reads it.
|
||||
func describeConfigMove(a configMoveAnswer) string {
|
||||
var b strings.Builder
|
||||
verb := "set"
|
||||
if a.DryRun {
|
||||
verb = "would set"
|
||||
}
|
||||
for _, s := range a.Sets {
|
||||
state := verb
|
||||
if s.Already {
|
||||
state = "holds already"
|
||||
}
|
||||
fmt.Fprintf(&b, "%s %s = %s at %s (from %s)\n", state, s.Key, compactJSON(s.Value), s.configLevel,
|
||||
strings.Join(s.From, " and "))
|
||||
}
|
||||
for _, s := range a.Unread {
|
||||
fmt.Fprintf(&b, "emptied, set nowhere: %s\n", s)
|
||||
}
|
||||
for _, s := range a.Stays {
|
||||
fmt.Fprintf(&b, "stays: %s\n", s)
|
||||
}
|
||||
for _, s := range a.Problems {
|
||||
fmt.Fprintf(&b, "refused: %s\n", s)
|
||||
}
|
||||
switch {
|
||||
case len(a.Problems) > 0:
|
||||
b.WriteString("nothing was set: the move refuses until each is put right\n")
|
||||
case a.DryRun:
|
||||
fmt.Fprintf(&b, "%d layer(s), %d value(s); nothing was set — with --confirm --why <text> they are\n",
|
||||
len(a.Moved), len(a.Sets))
|
||||
case a.Switched:
|
||||
fmt.Fprintf(&b, "%d layer(s) moved into mesh-config, each kept whole in its history first; the controller "+
|
||||
"reads mesh-config where it read them\n", len(a.Moved))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -1,467 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
func aConfiguredMesh(t *testing.T) (*stores, configuredMesh, map[string]map[string]any) {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
for i, name := range []string{"workstation", "desktop"} {
|
||||
record, err := inv.AddNode(ctx, name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.SetPlace(ctx, name, name+".example:51820", "here", false, fmt.Sprintf("10.77.0.%d", i+3)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordProfile(ctx, record.ID, map[string]any{"capabilities": []any{
|
||||
map[string]any{"name": "container-runtime", "present": true},
|
||||
map[string]any{"name": "systemd", "present": true}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, name, overlay.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "config-moved", "mesh.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var w configuredMesh
|
||||
if err := json.Unmarshal(raw, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
list, err := catalogue.ParsePredefined(w.Predefined)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
catalogue.UsePredefined(list)
|
||||
served := map[string]map[string]any{}
|
||||
for _, text := range w.Modules {
|
||||
var shape map[string]json.RawMessage
|
||||
if err := json.Unmarshal(text, &shape); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var name string
|
||||
_ = json.Unmarshal(shape["module"], &name)
|
||||
if s, ok := shape["serves-checked"]; ok {
|
||||
var serves map[string]any
|
||||
if err := json.Unmarshal(s, &serves); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
served[name] = serves
|
||||
delete(shape, "serves-checked")
|
||||
}
|
||||
text, err := json.Marshal(shape)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(text)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
register(t, open, m)
|
||||
}
|
||||
for _, l := range w.Layers {
|
||||
values, err := json.Marshal(l.Values)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if l.Node == "" {
|
||||
inv.ExecForTest(t, `insert into settings (node, module, values) values (null, $1, $2)`, l.Module, string(values))
|
||||
continue
|
||||
}
|
||||
inv.ExecForTest(t, `insert into settings (node, module, values) select id, $2, $3 from node where name = $1`,
|
||||
l.Node, l.Module, string(values))
|
||||
}
|
||||
for _, module := range movedKeys(w.On) {
|
||||
for _, node := range w.On[module] {
|
||||
if _, err := assign(ctx, open, node, module); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", node, module, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return open, w, served
|
||||
}
|
||||
|
||||
func TestEveryMachineIsToldTheSameAfterTheMoveOfConfig(t *testing.T) {
|
||||
open, w, served := aConfiguredMesh(t)
|
||||
ctx := t.Context()
|
||||
readBackTries = 0
|
||||
t.Cleanup(func() { readBackTries = 15 })
|
||||
inv := open.inventory
|
||||
config := &standInMeshConfig{open: open, values: map[string]any{}}
|
||||
inv.ReadsConfigFrom(config.laid)
|
||||
before := toldEveryMachine(t, open, served, w.On)
|
||||
listedBefore := effectiveEverywhere(t, open, w)
|
||||
factsBefore, err := gatherFacts(ctx, open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
asked, err := runConfigMove(ctx, inv, config, false, "the move of config, run by a test", "")
|
||||
if err != nil || !asked.DryRun || asked.Switched || len(asked.Problems) > 0 || len(config.sets) > 0 {
|
||||
t.Fatalf("the move, not confirmed, says %+v (%v) and set %v", asked, err, config.sets)
|
||||
}
|
||||
// nodered's live layer says the file's name for its time zone: set under the config key (issue 751).
|
||||
if s := setAt(asked, "module.anchor.nodered.time-zone"); len(s) != 1 || s[0].Value != "Europe/Brussels" ||
|
||||
!strings.HasSuffix(s[0].From[0], ": timeZone") {
|
||||
t.Errorf("nodered's time zone moves as %+v", s)
|
||||
}
|
||||
for _, s := range asked.Sets {
|
||||
if s.Key == "timeZone" {
|
||||
t.Errorf("a key is set under the file's name: %+v", s)
|
||||
}
|
||||
}
|
||||
// claude-code's whole-mesh managed settings become mesh config for agent-managed-settings (ADR 0348's
|
||||
// consequences), and its whole-mesh layer sets no module value.
|
||||
if s := setAt(asked, "mesh.agent-managed-settings"); len(s) != 1 || s[0].From[0] != "claude-code on the whole mesh: managed_settings" {
|
||||
t.Errorf("claude-code's managed settings move as %+v", s)
|
||||
}
|
||||
for _, node := range fourMachines {
|
||||
if s := setAt(asked, "module."+node+".claude-code.managed_settings"); len(s) > 0 {
|
||||
t.Errorf("claude-code's managed settings are also set on %s: %+v", node, s)
|
||||
}
|
||||
}
|
||||
// A whole-mesh layer of a value no predefined key holds goes to each machine it is assigned to, under that
|
||||
// machine's layer: power's on the workstation is the workstation's, its own elsewhere.
|
||||
if s := setAt(asked, "module.workstation.power.handle-power-key"); len(s) != 1 || s[0].Value != "suspend" {
|
||||
t.Errorf("power's key on the workstation moves as %+v", s)
|
||||
}
|
||||
if s := setAt(asked, "module.anchor.power.handle-power-key"); len(s) != 1 || s[0].Value != "poweroff" {
|
||||
t.Errorf("power's key on the anchor moves as %+v", s)
|
||||
}
|
||||
for _, s := range asked.Sets {
|
||||
if s.Level == "node" {
|
||||
t.Errorf("a machine's layer became node config: %+v", s)
|
||||
}
|
||||
}
|
||||
var stays []string
|
||||
for _, s := range asked.Stays {
|
||||
stays = append(stays, strings.Fields(s)[0])
|
||||
}
|
||||
sort.Strings(stays)
|
||||
if want := []string{"bazarr", "bazarr", "bookshelf", "jackett", "kernel", "kometa", "lidarr", "nzbget", "ombi",
|
||||
"plex", "qbittorrent", "radarr", "sonarr", "tautulli"}; len(stays) < 10 || !containsString(stays, "kernel") {
|
||||
t.Errorf("the layers that stay are %v, want those of %v", stays, want)
|
||||
}
|
||||
if after := toldEveryMachine(t, open, served, w.On); !reflect.DeepEqual(after, before) {
|
||||
t.Fatal("a dry run changed what a machine is told")
|
||||
}
|
||||
|
||||
moved, err := runConfigMove(ctx, inv, config, true, "the move of config, run by a test", "release 57 is delivered")
|
||||
if err != nil || moved.DryRun || !moved.Switched {
|
||||
t.Fatalf("the move says %+v (%v)", moved, err)
|
||||
}
|
||||
if len(config.sets) != len(asked.Sets) {
|
||||
t.Fatalf("the move set %d values, and said it would set %d", len(config.sets), len(asked.Sets))
|
||||
}
|
||||
if held := inv.QueryForTest(t, `select module from settings s where values <> '{}'
|
||||
and module in (select name from module where manifest ? 'config')`); len(held) != 0 {
|
||||
t.Fatalf("a layer of a module in the config form still holds values: %v", held)
|
||||
}
|
||||
after := toldEveryMachine(t, open, served, w.On)
|
||||
for _, node := range fourMachines {
|
||||
if after[node] != before[node] {
|
||||
t.Errorf("%s is told something else after the move:\n%s\nwas:\n%s", node, after[node], before[node])
|
||||
}
|
||||
}
|
||||
if listedAfter := effectiveEverywhere(t, open, w); !reflect.DeepEqual(listedAfter, listedBefore) {
|
||||
t.Errorf("settings lists other values after the move:\n%v\nwas:\n%v", listedAfter, listedBefore)
|
||||
}
|
||||
// And config lists what settings listed: each module's resolved value on each machine is the value settings
|
||||
// said before the move.
|
||||
resolved, err := config.Resolved(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range resolved {
|
||||
said, listed := listedBefore[r.Node+"."+r.Module+"."+r.Key]
|
||||
if r.Source == "unset" {
|
||||
if listed && said != nil {
|
||||
t.Errorf("%s of %s on %s is unset in config, and settings said %v", r.Key, r.Module, r.Node, said)
|
||||
}
|
||||
continue
|
||||
}
|
||||
var v any
|
||||
_ = json.Unmarshal(r.Value, &v)
|
||||
if !listed || !reflect.DeepEqual(v, said) {
|
||||
t.Errorf("config resolves %s of %s on %s as %v, and settings said %v", r.Key, r.Module, r.Node, v, said)
|
||||
}
|
||||
}
|
||||
|
||||
// The facts snapshot carries mesh-config's values for the merge gate, as each machine's layer of each module, and
|
||||
// each machine's declaration composes to the digest it did before the move.
|
||||
factsAfter, err := gatherFacts(ctx, open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Machines are named in a snapshot by stand-ins of its own; each is found by what it composes.
|
||||
digests := func(f snapshotFacts) ([]string, string) {
|
||||
var out []string
|
||||
zone := ""
|
||||
for _, m := range f.Machines {
|
||||
out = append(out, m.Declaration.Digest)
|
||||
for _, l := range m.Settings {
|
||||
if l.Module == "nodered" {
|
||||
zone = compactJSON(l.Values["time-zone"])
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, zone
|
||||
}
|
||||
was, _ := digests(factsBefore)
|
||||
now, zone := digests(factsAfter)
|
||||
if zone != `"Europe/Brussels"` {
|
||||
t.Errorf("the facts snapshot does not carry nodered's time zone from mesh-config: %s", zone)
|
||||
}
|
||||
if len(now) != len(fourMachines) || now[0] == "" || !reflect.DeepEqual(now, was) {
|
||||
t.Errorf("the declarations in the facts snapshot are %v after the move, %v before", now, was)
|
||||
}
|
||||
for _, s := range factsAfter.Settings {
|
||||
if _, isConfig := map[string]bool{"betterbird": true, "claude-code": true, "power": true, "xdg": true}[s.Module]; isConfig &&
|
||||
len(s.Values) > 0 {
|
||||
t.Errorf("the facts snapshot still carries the whole mesh's layer of %s: %v", s.Module, s.Values)
|
||||
}
|
||||
}
|
||||
|
||||
config.sets = nil
|
||||
again, err := runConfigMove(ctx, inv, config, true, "the move of config, run by a test", "again")
|
||||
if err != nil || len(config.sets) != 0 || len(again.Moved) != 0 {
|
||||
t.Fatalf("asked again, the move set %v and says %+v (%v)", config.sets, again, err)
|
||||
}
|
||||
if after := toldEveryMachine(t, open, served, w.On); !reflect.DeepEqual(after, before) {
|
||||
t.Fatal("the move asked again changed what a machine is told")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheMoveOfConfigRefusesWhatWouldChangeAMachine(t *testing.T) {
|
||||
open, _, _ := aConfiguredMesh(t)
|
||||
ctx := t.Context()
|
||||
readBackTries = 0
|
||||
t.Cleanup(func() { readBackTries = 15 })
|
||||
inv := open.inventory
|
||||
config := &standInMeshConfig{open: open, values: map[string]any{}}
|
||||
inv.ReadsConfigFrom(config.laid)
|
||||
|
||||
inv.ExecForTest(t, `update settings set values = values || '{"endpoints": {"web": {"reach": "public"}}}'
|
||||
where module = 'nodered'`)
|
||||
inv.ExecForTest(t, `update settings set values = values || '{"colour": "blue"}' where module = 'unifi'`)
|
||||
inv.ExecForTest(t, `update settings set values = '{"font-size": 900}' where module = 'dunst'`)
|
||||
answer, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered")
|
||||
if err == nil || answer.Switched || len(config.sets) > 0 {
|
||||
t.Fatalf("the move went ahead: %+v (%v), set %v", answer, err, config.sets)
|
||||
}
|
||||
for _, want := range []string{"nodered on anchor still holds placement (endpoints): run move-placement first",
|
||||
`unifi on anchor says "colour", which unifi does not declare`, "font-size at dunst's own value on workstation"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not say %q: %v", want, err)
|
||||
}
|
||||
}
|
||||
inv.ExecForTest(t, `update settings set values = values - 'endpoints' where module = 'nodered'`)
|
||||
inv.ExecForTest(t, `update settings set values = values - 'colour' where module = 'unifi'`)
|
||||
inv.ExecForTest(t, `update settings set values = '{"font-size": 13, "width": 500}' where module = 'dunst'`)
|
||||
|
||||
// Set in mesh-config before the move, on a module whose layers say nothing of it: the switch would lay it.
|
||||
config.values["module.anchor.grafana."+movedKeys(func() map[string]catalogue.SettingDeclaration {
|
||||
shelf, _ := inv.Catalogue(ctx)
|
||||
return shelf["grafana"].Settings
|
||||
}())[0]] = "set before the move"
|
||||
answer, err = runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered")
|
||||
if err == nil || !strings.Contains(err.Error(), "grafana on anchor resolves") || answer.Switched {
|
||||
t.Fatalf("a value no layer says was laid by the switch: %+v (%v)", answer, err)
|
||||
}
|
||||
if moved, _ := inv.ConfigMoved(ctx); moved.Moved {
|
||||
t.Fatal("the switch is on after a refused move")
|
||||
}
|
||||
if held := inv.QueryForTest(t, `select count(*) from settings where values <> '{}'`); held[0][0] == "0" {
|
||||
t.Fatal("a refused move emptied the layers")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSettingsVerbForwardsToConfigOnceTheValuesMoved(t *testing.T) {
|
||||
open, w, served := aConfiguredMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
readBackTries = 0
|
||||
t.Cleanup(func() { readBackTries = 15 })
|
||||
config := &standInMeshConfig{open: open, values: map[string]any{}}
|
||||
inv.ReadsConfigFrom(config.laid)
|
||||
savedRead, savedAsk := readConfigValues, configAsk
|
||||
readConfigValues, configAsk = config.laid, config.ask
|
||||
t.Cleanup(func() { readConfigValues, configAsk = savedRead, savedAsk })
|
||||
if _, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
shown := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"show", "nodered", "--node", "anchor"}) })
|
||||
for _, want := range []string{`"time-zone": "Europe/Brussels"`, `time-zone = "Europe/Brussels" (module)`,
|
||||
"mesh-config.config", `"mqtt": {`} {
|
||||
if !strings.Contains(shown, want) {
|
||||
t.Errorf("settings show does not say %q:\n%s", want, shown)
|
||||
}
|
||||
}
|
||||
if strings.Contains(shown, "timeZone") {
|
||||
t.Errorf("settings show still says the file's name:\n%s", shown)
|
||||
}
|
||||
|
||||
config.asked = nil
|
||||
said := stdoutOf(t, func() error {
|
||||
return settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12, "width": 500}`, "--node", "workstation"})
|
||||
})
|
||||
if config.values["module.workstation.dunst.font-size"] != float64(12) || !strings.Contains(said, "~ font-size") ||
|
||||
containsString(config.asked, "set module.workstation.dunst.width") {
|
||||
t.Fatalf("settings set reached mesh-config as %v (%v):\n%s", config.values, config.asked, said)
|
||||
}
|
||||
told := toldEveryMachine(t, open, served, w.On)["workstation"]
|
||||
if !strings.Contains(told, "12") {
|
||||
t.Error("the workstation is not told the font size set through settings")
|
||||
}
|
||||
if err := settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12}`, "--node", "workstation"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "would no longer set width") {
|
||||
t.Fatalf("a set dropping a value without --replace: %v", err)
|
||||
}
|
||||
if err := settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12}`}); err == nil ||
|
||||
!strings.Contains(err.Error(), "no own value for the whole mesh") {
|
||||
t.Fatalf("a module value for the whole mesh: %v", err)
|
||||
}
|
||||
|
||||
t.Setenv(verbVar, "settings")
|
||||
for _, c := range []struct{ module, node, values, key, held, why string }{
|
||||
// A file root reads: refused as before the move (novox/hq issue 339).
|
||||
{"hostname", "anchor", `{"hostname": "elsewhere"}`, "module.anchor.hostname.hostname", "Ace", "issue 339"},
|
||||
{"messenger", "laptop", `{"desktop-machines": ["g14", "shanks"], "time-zone": "UTC"}`,
|
||||
"module.laptop.messenger.time-zone", "Europe/Brussels", "issue 339"},
|
||||
} {
|
||||
err := settingsCommand(ctx, []string{"set", c.module, c.values, "--node", c.node})
|
||||
if err == nil || !strings.Contains(err.Error(), c.why) || config.values[c.key] != c.held {
|
||||
t.Fatalf("a trusted value of %s through a verb: %v, holds %v", c.module, err, config.values[c.key])
|
||||
}
|
||||
}
|
||||
// And a trusted value no file makes the terminal's is refused as trusted, since mesh-config takes the controller's
|
||||
// word for it (ADR 0348 §8).
|
||||
if err := refuseTrustedThroughAVerb(ctx, inv, "messenger", "laptop", []string{"time-zone"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "ADR 0348 §8") {
|
||||
t.Fatalf("a trusted value through a verb: %v", err)
|
||||
}
|
||||
if err := refuseTrustedThroughAVerb(ctx, inv, "dunst", "workstation", []string{"width"}); err != nil {
|
||||
t.Fatalf("a value nobody trusts through a verb: %v", err)
|
||||
}
|
||||
stdoutOf(t, func() error {
|
||||
return settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12, "width": 600}`, "--node", "workstation"})
|
||||
})
|
||||
if config.values["module.workstation.dunst.width"] != float64(600) {
|
||||
t.Fatalf("a value nobody trusts was not set through a verb: %v", config.values["module.workstation.dunst.width"])
|
||||
}
|
||||
t.Setenv(verbVar, "")
|
||||
|
||||
stdoutOf(t, func() error { return settingsCommand(ctx, []string{"clear", "dunst", "--node", "workstation"}) })
|
||||
for _, k := range []string{"font-size", "width"} {
|
||||
if _, held := config.values["module.workstation.dunst."+k]; held {
|
||||
t.Errorf("settings clear left dunst's %s on the workstation", k)
|
||||
}
|
||||
}
|
||||
|
||||
config.asked = nil
|
||||
proposed := stdoutOf(t, func() error {
|
||||
return settingsCommand(ctx, []string{"propose", "hostname", `{"hostname": "elsewhere"}`, "--node", "anchor"})
|
||||
})
|
||||
if !containsString(config.asked, "propose module.anchor.hostname.hostname") || !strings.Contains(proposed, "p-1") {
|
||||
t.Fatalf("settings propose reached mesh-config as %v:\n%s", config.asked, proposed)
|
||||
}
|
||||
|
||||
config.asked = nil
|
||||
kept := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"show", "kernel", "--node", "laptop"}) })
|
||||
if !strings.Contains(kept, "transparent-huge-pages") || len(config.asked) > 0 {
|
||||
t.Fatalf("a module in the settings form is shown as %q, asking mesh-config %v", kept, config.asked)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnApprovalRaisedBeforeTheMoveOfConfigIsRefusedAfterIt(t *testing.T) {
|
||||
open, _, _ := aConfiguredMesh(t)
|
||||
ctx := t.Context()
|
||||
readBackTries = 0
|
||||
t.Cleanup(func() { readBackTries = 15 })
|
||||
inv := open.inventory
|
||||
config := &standInMeshConfig{open: open, values: map[string]any{}}
|
||||
inv.ReadsConfigFrom(config.laid)
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := movedKeys(func() map[string]catalogue.SettingDeclaration { return shelf["grafana"].Settings }())[0]
|
||||
values := map[string]any{key: "raised before the move"}
|
||||
p := settingsProposal{Module: "grafana", Node: "anchor", Values: values, From: "g14/claude-code", At: time.Now(),
|
||||
Digest: layerDigest(values), BeforeDigest: layerDigest(nil)}
|
||||
if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has {
|
||||
t.Fatalf("grafana has a layer on anchor before the test: %v %v", has, err)
|
||||
}
|
||||
if moved, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil || !moved.Switched {
|
||||
t.Fatalf("the move says %+v (%v)", moved, err)
|
||||
}
|
||||
if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads {
|
||||
t.Fatalf("grafana does not read config after the move: %v %v", reads, err)
|
||||
}
|
||||
_, err = setLayerIn(open)(ctx, p, "s30", "approved by the operator")
|
||||
if err == nil || !strings.Contains(err.Error(), "moved to mesh-config since the operator was shown the change") ||
|
||||
!strings.Contains(err.Error(), "propose it again") {
|
||||
t.Fatalf("the stale approval: %v", err)
|
||||
}
|
||||
if layer, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has && len(layer) > 0 {
|
||||
t.Errorf("the refused approval wrote the layer: %v %v %v", layer, has, err)
|
||||
}
|
||||
if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads {
|
||||
t.Errorf("the refused approval turned ReadsConfig off: %v %v", reads, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPlacementOnlyApprovalRaisedBeforeTheMoveIsSetAfterIt(t *testing.T) {
|
||||
open, _, _ := aConfiguredMesh(t)
|
||||
ctx := t.Context()
|
||||
readBackTries = 0
|
||||
t.Cleanup(func() { readBackTries = 15 })
|
||||
inv := open.inventory
|
||||
config := &standInMeshConfig{open: open, values: map[string]any{}}
|
||||
inv.ReadsConfigFrom(config.laid)
|
||||
values := map[string]any{catalogue.PlacesSetting: map[string]any{"data": "/srv/grafana-data"}}
|
||||
p := settingsProposal{Module: "grafana", Node: "anchor", Values: values, From: "g14/claude-code", At: time.Now(),
|
||||
Digest: layerDigest(values), BeforeDigest: layerDigest(nil)}
|
||||
if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has {
|
||||
t.Fatalf("grafana has a layer on anchor before the test: %v %v", has, err)
|
||||
}
|
||||
if moved, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil || !moved.Switched {
|
||||
t.Fatalf("the move says %+v (%v)", moved, err)
|
||||
}
|
||||
if _, err := setLayerIn(open)(ctx, p, "s31", "approved by the operator"); err != nil {
|
||||
t.Fatalf("the placement-only approval after the move: %v", err)
|
||||
}
|
||||
if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || !has {
|
||||
t.Errorf("the approval did not set the layer: %v %v", has, err)
|
||||
}
|
||||
if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads {
|
||||
t.Errorf("the approval turned ReadsConfig off: %v %v", reads, err)
|
||||
}
|
||||
}
|
||||
@@ -1,336 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
snapshot "github.com/novox/mesh-controller/internal/facts"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// configuredMesh is every live settings layer (issue 664) — read with the settings verb on 2026-10-15, on the four
|
||||
// machines as anchor, laptop, workstation and desktop, the whole mesh's with no node; personal values stood in for,
|
||||
// and the placement words left out, which move-placement moves first — with each module in the config form shaped
|
||||
// as its definition on the feature branch: its config, its files (templates inlined) and directories, placeholders
|
||||
// of kinds this test's mesh does not hold stood in for. A module still in the settings form (the media catalogue's,
|
||||
// and kernel) stands in as a mergeable file that takes every key. `on` is where each is assigned, as live.
|
||||
type configuredMesh struct {
|
||||
Modules []json.RawMessage `json:"modules"`
|
||||
Layers []configuredLayer `json:"layers"`
|
||||
On map[string][]string `json:"on"`
|
||||
// Predefined is the catalogue's list of predefined keys, which a module falls back to.
|
||||
Predefined json.RawMessage `json:"predefined"`
|
||||
}
|
||||
|
||||
type configuredLayer struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Values map[string]any `json:"values"`
|
||||
}
|
||||
|
||||
var fourMachines = []string{"anchor", "laptop", "workstation", "desktop"}
|
||||
|
||||
// aConfiguredMesh is aMesh with two machines more, every module of the configured mesh registered and assigned, and
|
||||
// its layers held as the store holds them once move-placement has run.
|
||||
|
||||
// standInMeshConfig is mesh-config as this test holds it: each value at its level, and resolved from the catalogue
|
||||
// the store holds as mesh-config resolves (novox/hq ADR 0348 §2): the module's own value on the node, else node
|
||||
// config then mesh config for the predefined key it falls back to, else its default; a required value nothing sets is
|
||||
// unset.
|
||||
type standInMeshConfig struct {
|
||||
open *stores
|
||||
values map[string]any
|
||||
sets []string
|
||||
// asked is each call of its verb: the action and the level's key.
|
||||
asked []string
|
||||
}
|
||||
|
||||
func (c *standInMeshConfig) Values(context.Context) (map[string]json.RawMessage, error) {
|
||||
out := map[string]json.RawMessage{}
|
||||
for k, v := range c.values {
|
||||
raw, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[k] = raw
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *standInMeshConfig) Set(_ context.Context, level configLevel, key string, value any, by, why string) error {
|
||||
if by == "" || why == "" {
|
||||
return fmt.Errorf("%s at %s is set without who or why", key, level)
|
||||
}
|
||||
c.values[level.valueKey(key)] = normalJSON(value)
|
||||
c.sets = append(c.sets, level.valueKey(key))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *standInMeshConfig) Resolved(ctx context.Context) ([]resolvedEntry, error) {
|
||||
entries, err := c.open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []resolvedEntry
|
||||
for _, e := range entries {
|
||||
m := e.Manifest
|
||||
if !m.DeclaresConfig() {
|
||||
continue
|
||||
}
|
||||
for _, node := range e.On {
|
||||
for _, key := range movedKeys(m.Settings) {
|
||||
d := m.Settings[key]
|
||||
r := resolvedEntry{Node: node, Module: m.Module, Key: key, Source: "default"}
|
||||
value, has := c.values["module."+node+"."+m.Module+"."+key]
|
||||
switch {
|
||||
case has:
|
||||
r.Source = "module"
|
||||
case d.Predefined != "" && c.values["node."+node+"."+d.Predefined] != nil:
|
||||
value, r.Source = c.values["node."+node+"."+d.Predefined], "node"
|
||||
case d.Predefined != "" && c.values["mesh."+d.Predefined] != nil:
|
||||
value, r.Source = c.values["mesh."+d.Predefined], "mesh"
|
||||
case d.Required:
|
||||
r.Source = "unset"
|
||||
default:
|
||||
value = d.Default
|
||||
}
|
||||
if r.Source != "unset" {
|
||||
if r.Value, err = json.Marshal(value); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// laid is the stand-in as composition reads it once the switch is on (inventory.ConfigValues).
|
||||
func (c *standInMeshConfig) laid(ctx context.Context, node, module string) (map[string]any, error) {
|
||||
resolved, err := c.Resolved(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return configLaid(resolved, node, module)
|
||||
}
|
||||
|
||||
// snapshotFacts is the facts snapshot as gatherFacts gives it.
|
||||
type snapshotFacts = snapshot.Facts
|
||||
|
||||
// toldEveryMachine is what each of the four machines is told — every resource and the filter's rules — and what
|
||||
// each module that serves something serves, settled from its layers, as JSON a person can diff.
|
||||
func toldEveryMachine(t *testing.T, open *stores, served map[string]map[string]any, on map[string][]string) map[string]string {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
told := map[string]string{}
|
||||
for _, node := range fourMachines {
|
||||
plan, settings, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", node, err)
|
||||
}
|
||||
with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Allocating)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", node, err)
|
||||
}
|
||||
composed, err := plan.Compose(with)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", node, err)
|
||||
}
|
||||
rules, err := plan.Rules(with)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", node, err)
|
||||
}
|
||||
settles := map[string]any{}
|
||||
for module, serves := range served {
|
||||
if !containsString(on[module], node) {
|
||||
continue
|
||||
}
|
||||
layers, err := open.inventory.SettingsFor(ctx, node, module)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, s := range serves {
|
||||
fact, _ := s.(map[string]any)
|
||||
settled, err := catalogue.Settle(fact, catalogue.WithDefaults(shelf[module], layers))
|
||||
if err != nil {
|
||||
settles[module+" serves "+name] = "error: " + err.Error()
|
||||
continue
|
||||
}
|
||||
settles[module+" serves "+name] = settled
|
||||
}
|
||||
}
|
||||
got, err := json.MarshalIndent(map[string]any{"resources": composed.Resources, "left-out": composed.LeftOut,
|
||||
"rules": rules, "served": settles}, "", " ")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
told[node] = string(got)
|
||||
}
|
||||
return told
|
||||
}
|
||||
|
||||
// effectiveEverywhere is each module's every value on each machine it is assigned to, as `settings show` and
|
||||
// `settings preferences` say them (catalogue.Effective): the value, never where it came from, which the move
|
||||
// changes by design; a key a layer says by its file's name is read as its config key.
|
||||
func effectiveEverywhere(t *testing.T, open *stores, w configuredMesh) map[string]any {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := map[string]any{}
|
||||
for _, module := range movedKeys(w.On) {
|
||||
if !shelf[module].DeclaresConfig() {
|
||||
continue
|
||||
}
|
||||
for _, node := range w.On[module] {
|
||||
layers, err := open.inventory.SettingsFor(ctx, node, module)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A layer's old spelling of a key (nodered's timeZone) is listed beside the key it is the file's name
|
||||
// for, and reaches the file as that key does (issue 751): read as that key, which the move sets.
|
||||
as := catalogue.TakenAs(shelf[module])
|
||||
for _, v := range catalogue.Effective(shelf[module], layers) {
|
||||
if key, old := as[v.Key]; old {
|
||||
out[node+"."+module+"."+key] = normalJSON(v.Value)
|
||||
continue
|
||||
}
|
||||
if _, said := out[node+"."+module+"."+v.Key]; said && v.FromDefault {
|
||||
continue
|
||||
}
|
||||
out[node+"."+module+"."+v.Key] = normalJSON(v.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func setAt(a configMoveAnswer, key string) []configSet {
|
||||
var out []configSet
|
||||
for _, s := range a.Sets {
|
||||
if s.valueKey(s.Key) == key {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// **The settings layers move into mesh-config and no machine is told anything new** (novox/hq ADR 0348 §9, design
|
||||
// 57 K5, issue 664). Every live layer is seeded as the store holds it once move-placement has run, on modules shaped
|
||||
// as their definitions, and every machine composed. The move, asked without confirming, says each value it would set
|
||||
// and where from, and sets nothing; confirmed, it sets them in mesh-config, reads them back, empties the layers it
|
||||
// moved and switches. Each machine is then told byte for byte what it was told before, every module serves the same,
|
||||
// and `settings` lists the same values `config` resolves. Asked again, it sets nothing.
|
||||
|
||||
// **The move refuses what it cannot move as it composes, and sets nothing** (issue 664): a layer that still holds
|
||||
// placement (move-placement has not run), a key its module does not declare, and a value its declaration refuses —
|
||||
// each said by name; and a value mesh-config resolves from a level no layer says (it would be laid once the switch
|
||||
// is on) leaves every layer and the switch as they were.
|
||||
|
||||
// The verb's command line: a dry run unless confirmed, and a confirmed move says why, recorded as a person's step of
|
||||
// the release, never a repair (issue 664).
|
||||
func TestTheMoveOfConfigIsADryRunUnlessConfirmed(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
args map[string]any
|
||||
want string
|
||||
}{
|
||||
{map[string]any{}, "move-config --json"},
|
||||
{map[string]any{"confirm": "true", "why": "release 57 is delivered"}, "move-config --json --confirm --why release 57 is delivered"},
|
||||
} {
|
||||
argv, err := argvFor("move-config", c.args)
|
||||
if err != nil || strings.Join(argv, " ") != c.want {
|
||||
t.Errorf("%v: %q %v, want %q", c.args, argv, err, c.want)
|
||||
}
|
||||
}
|
||||
for _, args := range []map[string]any{{"confirm": "true"}, {"why": "tidy"}, {"node": "anchor"}} {
|
||||
if argv, err := argvFor("move-config", args); err == nil {
|
||||
t.Errorf("%v was composed as %q", args, argv)
|
||||
}
|
||||
}
|
||||
if repairingCommand([]string{"move-config", "--json", "--confirm", "--why", "x"}) != "move-config" {
|
||||
t.Error("a confirmed move through the generic verb would go unrecorded")
|
||||
}
|
||||
if !personsDecision(link.HandAct{Verb: "move-config"}) {
|
||||
t.Error("a move of the release would count toward a healer the mesh lacks")
|
||||
}
|
||||
}
|
||||
|
||||
// ask is mesh-config's verb as the stand-in answers it: list, set, clear and propose.
|
||||
func (c *standInMeshConfig) ask(ctx context.Context, args map[string]any) (json.RawMessage, error) {
|
||||
str := func(k string) string { s, _ := args[k].(string); return s }
|
||||
level := configLevel{Level: "mesh"}
|
||||
switch {
|
||||
case str("module") != "" && str("node") != "":
|
||||
level = configLevel{Level: "module", Node: str("node"), Module: str("module")}
|
||||
case str("node") != "":
|
||||
level = configLevel{Level: "node", Node: str("node")}
|
||||
}
|
||||
c.asked = append(c.asked, str("action")+" "+level.valueKey(str("key")))
|
||||
switch str("action") {
|
||||
case "list":
|
||||
resolved, err := c.Resolved(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]any{"resolved": []resolvedEntry{}, "set": []any{}}
|
||||
var listed []resolvedEntry
|
||||
for _, r := range resolved {
|
||||
if r.Module == str("module") && (str("node") == "" || r.Node == str("node")) {
|
||||
listed = append(listed, r)
|
||||
}
|
||||
}
|
||||
out["resolved"] = listed
|
||||
var set []any
|
||||
for _, k := range movedKeys(c.values) {
|
||||
parts := strings.SplitN(k, ".", 4)
|
||||
if len(parts) == 4 && parts[0] == "module" && parts[2] == str("module") {
|
||||
set = append(set, map[string]any{"level": configLevel{Level: "module", Node: parts[1], Module: parts[2]},
|
||||
"key": parts[3], "value": c.values[k], "by": "a test"})
|
||||
}
|
||||
}
|
||||
out["set"] = set
|
||||
return json.Marshal(out)
|
||||
case "set":
|
||||
if str("why") == "" || str("by") == "" {
|
||||
return nil, fmt.Errorf("a set without why or by: %v", args)
|
||||
}
|
||||
return nil, c.Set(ctx, level, str("key"), args["value"], str("by"), str("why"))
|
||||
case "clear":
|
||||
if _, had := c.values[level.valueKey(str("key"))]; !had {
|
||||
return nil, fmt.Errorf("nothing is set for %s", str("key"))
|
||||
}
|
||||
delete(c.values, level.valueKey(str("key")))
|
||||
return json.RawMessage(`{}`), nil
|
||||
case "propose":
|
||||
return json.RawMessage(`{"proposal":"p-1"}`), nil
|
||||
}
|
||||
return nil, fmt.Errorf("the stand-in does not %s", str("action"))
|
||||
}
|
||||
|
||||
// **`settings` answers by forwarding to `config`** (novox/hq ADR 0348 §1, design 57 K5, issue 664): once the move
|
||||
// has run, a module in the config form is shown from mesh-config, its values set and cleared there as its own on the
|
||||
// machine (placement still beside the assignment), and proposed through it; a trusted value is refused through a
|
||||
// verb, a module value for the whole mesh is refused, and a module still in the settings form keeps today's verb.
|
||||
|
||||
// **An approval for a module whose config has moved is refused, and writes nothing** (issue 830, novox/hq ADR 0348):
|
||||
// a proposal raised while grafana read its layer (it had none, so the layer is as the operator was shown it) and
|
||||
// approved after the move would put its values back into the layer, ReadsConfig would turn false, and the module
|
||||
// would compose from emptied layers everywhere.
|
||||
|
||||
// **A proposal of placement only is set after the move, as before** (issue 830): forwardPropose sends it down the
|
||||
// old path on purpose, so refusing it would loop the operator (refused, proposed again, refused).
|
||||
@@ -1,128 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **Where the controller reads config values once the move has run** (novox/hq ADR 0348 §9, design 57 K5, issue 664):
|
||||
// mesh-config_resolved, the bucket mesh-config alone writes, keyed `<node>.<module>.<key>`, each value with the
|
||||
// source it came from. Read only for a module the switch reaches (inventory.ReadsConfig), so a controller before the
|
||||
// move never asks the bus for config. The exception to ADR 0348 §9 ("the controller holds no config value") that
|
||||
// ADR 0349 clause 17's old way needs, until design 57's R1 hands every module to the provisioner.
|
||||
|
||||
// resolvedEntry is one key of mesh-config_resolved as the controller reads it: the value and where it came from.
|
||||
type resolvedEntry struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value,omitempty"`
|
||||
Source string `json:"source"`
|
||||
}
|
||||
|
||||
// The sources a resolved value is laid over a module's files from: mesh config, node config and the module's own
|
||||
// value. Its manifest's default is composition's already, and a taken key's default is never laid over its file;
|
||||
// a required value nothing sets (`unset`) is refused by composition by name, as it is today.
|
||||
var laidSources = map[string]bool{"mesh": true, "node": true, "module": true}
|
||||
|
||||
// configLaid is the values of one module on one node that are laid over its files, from the resolved entries.
|
||||
func configLaid(entries []resolvedEntry, node, module string) (map[string]any, error) {
|
||||
out := map[string]any{}
|
||||
for _, e := range entries {
|
||||
if e.Node != node || e.Module != module || !laidSources[e.Source] {
|
||||
continue
|
||||
}
|
||||
var v any
|
||||
if err := json.Unmarshal(e.Value, &v); err != nil {
|
||||
return nil, fmt.Errorf("mesh-config's value %s of %s on %s cannot be read: %w", e.Key, module, node, err)
|
||||
}
|
||||
out[e.Key] = v
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// resolvedRead is what one read of the whole bucket held, and when it was read.
|
||||
type resolvedRead struct {
|
||||
mu sync.Mutex
|
||||
at time.Time
|
||||
entries []resolvedEntry
|
||||
}
|
||||
|
||||
// resolvedFresh is how long one read of the bucket answers: a plan reads every module of a machine, and one read
|
||||
// serves them all; a value set since is read by the next composition.
|
||||
const resolvedFresh = 5 * time.Second
|
||||
|
||||
var theResolved resolvedRead
|
||||
|
||||
// readConfigValues is where an opened inventory reads config values once the switch is on; a test replaces it.
|
||||
var readConfigValues inventory.ConfigValues = busConfigValues
|
||||
|
||||
// busConfigValues reads one module's config values on one node from mesh-config_resolved (inventory.ConfigValues).
|
||||
func busConfigValues(ctx context.Context, node, module string) (map[string]any, error) {
|
||||
theResolved.mu.Lock()
|
||||
defer theResolved.mu.Unlock()
|
||||
if theResolved.entries == nil || time.Since(theResolved.at) > resolvedFresh {
|
||||
entries, err := readResolved(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
theResolved.entries, theResolved.at = entries, time.Now()
|
||||
}
|
||||
return configLaid(theResolved.entries, node, module)
|
||||
}
|
||||
|
||||
// readResolved is every entry of mesh-config_resolved, read on the bus.
|
||||
func readResolved(ctx context.Context) ([]resolvedEntry, error) {
|
||||
entries := []resolvedEntry{}
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
|
||||
defer cancel()
|
||||
kv, err := js.KeyValue(ctx, broker.ConfigResolvedBucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s cannot be opened: %w", broker.ConfigResolvedBucket, err)
|
||||
}
|
||||
w, err := kv.WatchAll(ctx, jetstream.IgnoreDeletes())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = w.Stop() }()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("%s was not read whole: %w", broker.ConfigResolvedBucket, ctx.Err())
|
||||
case e, open := <-w.Updates():
|
||||
if !open {
|
||||
return errors.New(broker.ConfigResolvedBucket + " stopped answering")
|
||||
}
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
var r resolvedEntry
|
||||
if err := json.Unmarshal(e.Value(), &r); err != nil {
|
||||
return fmt.Errorf("%s holds %q, which is not a resolved value: %w", broker.ConfigResolvedBucket,
|
||||
e.Key(), err)
|
||||
}
|
||||
if r.Node == "" || r.Module == "" || r.Key == "" || !strings.HasPrefix(e.Key(), r.Node+"."+r.Module+".") {
|
||||
return fmt.Errorf("%s holds %q, whose value names another key", broker.ConfigResolvedBucket, e.Key())
|
||||
}
|
||||
entries = append(entries, r)
|
||||
}
|
||||
}
|
||||
})
|
||||
return entries, err
|
||||
}
|
||||
@@ -1,305 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ecdh"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The core's own credentials, sealed by the vault (novox/hq ADR 0349 §16, issue 755; the lead's decision on issue 755,
|
||||
// notes 4 and 5).
|
||||
//
|
||||
// For a machine whose engine opens the vault's box (link.VaultBoxContract), each own secret of a module on the core's
|
||||
// list (catalogue.CoreCredentialModules) travels as the vault sealed it to that machine's key, which the controller
|
||||
// asks for on the send path (`get`, answered to the controller alone and sealed only to the key that machine's engine
|
||||
// stated: mesh-vault's core.go) and carries without being able to open it. A machine below that contract is sent the
|
||||
// controller's own sealed value, exactly as before.
|
||||
//
|
||||
// **The vault seals every answer afresh**, so the box last got is kept (inventory.CoreBox) and carried again while the
|
||||
// vault's generation and the machine's key are the same: a declaration that changed with every answer would never be
|
||||
// current. Every question but a send reads the kept box and asks nothing.
|
||||
//
|
||||
// **Until the vault holds a machine's core credentials, the controller seals them itself** (novox/hq ADR 0396, issue
|
||||
// 908; it replaces the ADR 0349 §16 rule that the controller never does). When the vault does not answer, a credential
|
||||
// with a kept box keeps it — the machine keeps the file it has — and one with none has its module composed as for an
|
||||
// engine below the contract: with the controller's own sealed value, never left out (a core module left out took the
|
||||
// bus's port from the packet filter and cut the remote machines off). Either way the send raises a condition naming
|
||||
// the vault and the machine, cleared by the first send the vault answers whole. Only a machine with no sealing key, or
|
||||
// one whose key cannot be read, leaves its module out.
|
||||
|
||||
// engineOpensVaultBox says whether a machine's node-engine opens the vault's box, by its own newest statement.
|
||||
func engineOpensVaultBox(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) {
|
||||
stated, had, err := inv.HealthOf(ctx, node)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return had && stated.Contract >= link.VaultBoxContract, nil
|
||||
}
|
||||
|
||||
// coreAnswer is the vault's `get` answer as the controller reads it: the name, the generation and the box, never
|
||||
// opened.
|
||||
type coreAnswer struct {
|
||||
Name string `json:"name"`
|
||||
Generation int `json:"generation"`
|
||||
Sealed json.RawMessage `json:"sealed"`
|
||||
}
|
||||
|
||||
// coreAsk asks the vault for one secret sealed to the given key.
|
||||
type coreAsk func(name, publicKey string) (coreAnswer, error)
|
||||
|
||||
// coreBoxes is what the vault's boxes make of one machine's composition.
|
||||
type coreBoxes struct {
|
||||
// sealed is each core credential's box, by module and name (catalogue.Rendering.VaultSealed).
|
||||
sealed map[string]map[string]json.RawMessage
|
||||
// notPlaced is each core module left out, with why (catalogue.Rendering.NotPlaced).
|
||||
notPlaced map[string]string
|
||||
// keep is each box newly answered, to keep.
|
||||
keep []inventory.CoreBox
|
||||
// byController is each core module the vault did not answer for and no box was kept for, which is composed with
|
||||
// the controller's own seal until the vault holds it (ADR 0396).
|
||||
byController []string
|
||||
// held is each credential the vault did not answer for whose kept box is carried again; silent is why the vault
|
||||
// did not answer, per credential.
|
||||
held, silent []string
|
||||
}
|
||||
|
||||
// coreModulesOf is every module of the plan on the core's list whose own secrets this machine's declaration carries,
|
||||
// with their plain own secret names, sorted: one handed to the provisioner has its secrets from the vault through the
|
||||
// client, and one issued outside the mesh is never the vault's to seal here.
|
||||
func coreModulesOf(modules []catalogue.Manifest, handed func(catalogue.Manifest) bool) map[string][]string {
|
||||
out := map[string][]string{}
|
||||
for _, m := range modules {
|
||||
if !catalogue.CarriesVaultSealed(m.Module) || (handed != nil && handed(m)) {
|
||||
continue
|
||||
}
|
||||
var names []string
|
||||
for name, own := range m.OwnSecrets.Plain() {
|
||||
if own.IssuedBy != catalogue.IssuedOutside {
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
if len(names) > 0 {
|
||||
sort.Strings(names)
|
||||
out[m.Module] = names
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sealingKeyPEM is a machine's sealing key, recorded raw in base64, as the SPKI PEM the vault seals to.
|
||||
func sealingKeyPEM(raw string) (string, error) {
|
||||
b, err := base64.StdEncoding.DecodeString(strings.TrimSpace(raw))
|
||||
if err != nil || len(b) != 32 {
|
||||
return "", errors.New("the machine's sealing key is not a raw X25519 key")
|
||||
}
|
||||
pub, err := ecdh.X25519().NewPublicKey(b)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
der, err := x509.MarshalPKIXPublicKey(pub)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: der})), nil
|
||||
}
|
||||
|
||||
// coreBoxesOf decides each core credential of a machine: on a send, the vault's answer (the kept box while its
|
||||
// generation and key are the same), the kept box where the vault did not answer, or the module left to the controller's
|
||||
// own seal where nothing was kept; on any other question, the kept box, or a stand-in the next send asks for. Pure: what to keep and what to
|
||||
// say is answered, not done.
|
||||
func coreBoxesOf(node, sealingKey string, core map[string][]string, kept map[string]map[string]inventory.CoreBox,
|
||||
choosing Choosing, ask coreAsk) coreBoxes {
|
||||
out := coreBoxes{sealed: map[string]map[string]json.RawMessage{}, notPlaced: map[string]string{}}
|
||||
modules := make([]string, 0, len(core))
|
||||
for m := range core {
|
||||
modules = append(modules, m)
|
||||
}
|
||||
sort.Strings(modules)
|
||||
for _, module := range modules {
|
||||
if sealingKey == "" {
|
||||
out.notPlaced[module] = fmt.Sprintf("%s has no sealing key, so the vault can seal %s's credentials to "+
|
||||
"nothing there (novox/hq ADR 0349 §16)", node, module)
|
||||
continue
|
||||
}
|
||||
key, err := sealingKeyPEM(sealingKey)
|
||||
if err != nil {
|
||||
out.notPlaced[module] = fmt.Sprintf("%s's credentials cannot be asked of the vault: %v", module, err)
|
||||
continue
|
||||
}
|
||||
boxes := map[string]json.RawMessage{}
|
||||
for _, name := range core[module] {
|
||||
k, has := kept[module][name]
|
||||
valid := has && k.SealedTo == sealingKey && len(k.Box) > 0
|
||||
if choosing != Allocating {
|
||||
if valid {
|
||||
boxes[name] = k.Box
|
||||
} else {
|
||||
boxes[name] = catalogue.ForeseenVaultBox
|
||||
}
|
||||
continue
|
||||
}
|
||||
secret := "own/" + node + "/" + module + "/" + name
|
||||
got, err := ask(secret, key)
|
||||
if err == nil && (got.Generation <= 0 || canonicalBox(got.Sealed) == nil) {
|
||||
err = errors.New("it answered no box")
|
||||
}
|
||||
if err != nil {
|
||||
out.silent = append(out.silent, fmt.Sprintf("%s: %v", secret, err))
|
||||
if valid {
|
||||
boxes[name] = k.Box
|
||||
out.held = append(out.held, module+"/"+name)
|
||||
continue
|
||||
}
|
||||
out.byController = append(out.byController, module)
|
||||
break
|
||||
}
|
||||
if valid && k.Generation == got.Generation {
|
||||
boxes[name] = k.Box
|
||||
continue
|
||||
}
|
||||
box := canonicalBox(got.Sealed)
|
||||
boxes[name] = box
|
||||
out.keep = append(out.keep, inventory.CoreBox{Module: module, Name: name, Generation: got.Generation,
|
||||
SealedTo: sealingKey, Box: box})
|
||||
}
|
||||
if _, left := out.notPlaced[module]; !left && !slices.Contains(out.byController, module) {
|
||||
out.sealed[module] = boxes
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// canonicalBox is the vault's box in the one form the declaration carries — the sealed box's fields, in their order,
|
||||
// and nothing else — or nil when it is not one.
|
||||
func canonicalBox(raw json.RawMessage) json.RawMessage {
|
||||
var b struct {
|
||||
V int `json:"v"`
|
||||
Eph string `json:"eph"`
|
||||
IV string `json:"iv"`
|
||||
Tag string `json:"tag"`
|
||||
Ct string `json:"ct"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &b); err != nil || b.V <= 0 || b.Eph == "" || b.Ct == "" {
|
||||
return nil
|
||||
}
|
||||
out, _ := json.Marshal(b)
|
||||
return out
|
||||
}
|
||||
|
||||
// coreVault is how a send asks the vault for a core credential: its `get`, in the controller's name. A test stands
|
||||
// in for it.
|
||||
var coreVault = func(ctx context.Context) coreAsk {
|
||||
vault := theVault(ctx)
|
||||
return func(name, publicKey string) (coreAnswer, error) {
|
||||
raw, err := vault("get", map[string]any{"name": name, "public_key": publicKey})
|
||||
if err != nil {
|
||||
return coreAnswer{}, err
|
||||
}
|
||||
var a coreAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return coreAnswer{}, fmt.Errorf("the vault's answer is not one: %w", err)
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
}
|
||||
|
||||
// coreBoxesFor is coreBoxesOf for one machine, read from and kept in the store, and said on a send.
|
||||
func coreBoxesFor(ctx context.Context, inv *inventory.Inventory, node string, modules []catalogue.Manifest,
|
||||
handed func(catalogue.Manifest) bool, choosing Choosing) (coreBoxes, error) {
|
||||
core := coreModulesOf(modules, handed)
|
||||
if len(core) == 0 {
|
||||
if choosing == Allocating {
|
||||
settleVaultSilent(ctx, node)
|
||||
}
|
||||
return coreBoxes{}, nil
|
||||
}
|
||||
key, err := inv.SealingKeyOf(ctx, node)
|
||||
if err != nil {
|
||||
return coreBoxes{}, err
|
||||
}
|
||||
kept, err := inv.KeptCoreBoxes(ctx, node)
|
||||
if err != nil {
|
||||
return coreBoxes{}, err
|
||||
}
|
||||
var ask coreAsk
|
||||
if choosing == Allocating {
|
||||
ask = coreVault(ctx)
|
||||
}
|
||||
out := coreBoxesOf(node, key, core, kept, choosing, ask)
|
||||
for _, b := range out.keep {
|
||||
if err := inv.KeepCoreBox(ctx, node, b); err != nil {
|
||||
return coreBoxes{}, err
|
||||
}
|
||||
}
|
||||
if choosing == Allocating {
|
||||
sayVaultSilent(ctx, node, out)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// kindVaultSilent is a send that did not have the vault's box for a core credential of a machine.
|
||||
const kindVaultSilent = "vault-silent"
|
||||
|
||||
// vaultSilentObservation is that, as a condition: `machine.<machine>.vault-silent`.
|
||||
func vaultSilentObservation(node string, b coreBoxes) conditions.Observation {
|
||||
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: kindVaultSilent, Token: kindVaultSilent,
|
||||
Machine: node, Severity: conditions.Warning, Source: coreDowngradeSource}
|
||||
var said []string
|
||||
if len(b.notPlaced) > 0 {
|
||||
o.Severity = conditions.Urgent
|
||||
var left []string
|
||||
for m := range b.notPlaced {
|
||||
left = append(left, m)
|
||||
}
|
||||
sort.Strings(left)
|
||||
said = append(said, "not placed: "+strings.Join(left, ", "))
|
||||
}
|
||||
if len(b.byController) > 0 {
|
||||
said = append(said, "sealed by the controller until the vault holds it: "+strings.Join(b.byController, ", "))
|
||||
}
|
||||
if len(b.held) > 0 {
|
||||
said = append(said, "kept as last sent: "+strings.Join(b.held, ", "))
|
||||
}
|
||||
o.Summary = fmt.Sprintf("the vault did not answer for the core's credentials on %s; %s", node, strings.Join(said, "; "))
|
||||
o.Said = o.Summary + " — " + strings.Join(b.silent, "; ")
|
||||
return o
|
||||
}
|
||||
|
||||
// sayVaultSilent raises the condition for a send the vault did not answer whole, and clears it for one it did.
|
||||
func sayVaultSilent(ctx context.Context, node string, b coreBoxes) {
|
||||
if len(b.silent) == 0 && len(b.notPlaced) == 0 {
|
||||
settleVaultSilent(ctx, node)
|
||||
return
|
||||
}
|
||||
fmt.Printf("HELD: %s\n", vaultSilentObservation(node, b).Said)
|
||||
if conditionsFrom == nil {
|
||||
return
|
||||
}
|
||||
if _, err := conditionsFrom.Observe(ctx, vaultSilentObservation(node, b)); err != nil {
|
||||
fmt.Printf("the condition for the vault's silence on %s could not be raised: %v\n", node, err)
|
||||
}
|
||||
}
|
||||
|
||||
// settleVaultSilent clears the condition once a send had every core credential of the machine from the vault.
|
||||
func settleVaultSilent(ctx context.Context, node string) {
|
||||
if conditionsFrom == nil {
|
||||
return
|
||||
}
|
||||
key := vaultSilentObservation(node, coreBoxes{}).Key()
|
||||
if _, err := conditionsFrom.Clear(ctx, key, "the vault answered for every core credential of "+node); err != nil {
|
||||
fmt.Printf("the condition %s could not be cleared: %v\n", key, err)
|
||||
}
|
||||
}
|
||||
@@ -1,129 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func aCoreMesh(t *testing.T, contract int) (*stores, *coreStandIn) {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
client := aClientModule()
|
||||
register(t, open, client)
|
||||
if _, err := assign(ctx, open, "laptop", client.Module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := issueProvisionerClient(ctx, open.inventory, client, "laptop", broker.Broker{Fingerprint: "ab"},
|
||||
"10.0.0.1:4222"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.RecordHealth(ctx, inventory.NodeHealth{Node: "laptop", Contract: contract,
|
||||
SaidAt: time.Now()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v := &coreStandIn{generation: 1}
|
||||
withVault(t, v)
|
||||
return open, v
|
||||
}
|
||||
|
||||
func TestTheSendCarriesTheVaultsBoxForACoreCredentialAndKeepsIt(t *testing.T) {
|
||||
open, vault := aCoreMesh(t, link.VaultBoxContract)
|
||||
key, err := open.inventory.SealingKeyOf(t.Context(), "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := clientsBroker(sentTo(t, open, "laptop"))
|
||||
if f == nil || f["sealed"] != nil || f[catalogue.VaultSealedField] == nil {
|
||||
t.Fatalf("the core credential is sent as %v: the vault's box, never the controller's seal", f)
|
||||
}
|
||||
if want := []string{"own/laptop/mesh-provisioner-client/" + broker.ProvisionerSecret}; !reflect.DeepEqual(vault.asked, want) {
|
||||
t.Fatalf("the vault was asked %v", vault.asked)
|
||||
}
|
||||
if want, _ := sealingKeyPEM(key); vault.keys[0] != want {
|
||||
t.Fatalf("the vault was asked to seal to %q, not the machine's key", vault.keys[0])
|
||||
}
|
||||
first, _ := json.Marshal(f[catalogue.VaultSealedField])
|
||||
|
||||
// The vault answers afresh with the same generation: the kept box is carried again, so the declaration is stable.
|
||||
again, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField])
|
||||
if string(again) != string(first) {
|
||||
t.Fatalf("a fresh box of the same generation changed the declaration: %s, then %s", first, again)
|
||||
}
|
||||
// A rotation in the vault (a newer generation) is carried.
|
||||
vault.generation = 2
|
||||
rotated, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField])
|
||||
if string(rotated) == string(first) {
|
||||
t.Fatal("a newer generation was not carried")
|
||||
}
|
||||
if _, found := vaultSilence(t, "laptop"); found {
|
||||
t.Fatal("a send the vault answered raised its silence")
|
||||
}
|
||||
}
|
||||
|
||||
func TestASilentVaultHoldsTheKeptBoxAndSaysSo(t *testing.T) {
|
||||
open, vault := aCoreMesh(t, link.VaultBoxContract)
|
||||
first, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField])
|
||||
vault.silent = errors.New("no responders")
|
||||
held := clientsBroker(sentTo(t, open, "laptop"))
|
||||
got, _ := json.Marshal(held[catalogue.VaultSealedField])
|
||||
if held == nil || held["sealed"] != nil || string(got) != string(first) {
|
||||
t.Fatalf("a silent vault's credential is sent as %v: the box kept, never the controller's seal", held)
|
||||
}
|
||||
c, found := vaultSilence(t, "laptop")
|
||||
if !found || !strings.Contains(c.Summary, "vault") || !strings.Contains(c.Summary, "laptop") {
|
||||
t.Fatalf("no condition names the vault and the machine: %+v", c)
|
||||
}
|
||||
// The vault answers again: the condition clears.
|
||||
vault.silent = nil
|
||||
sentTo(t, open, "laptop")
|
||||
if _, found := vaultSilence(t, "laptop"); found {
|
||||
t.Fatal("the condition stayed after the vault answered")
|
||||
}
|
||||
}
|
||||
|
||||
func TestACoreModuleFirstPlacedWhileTheVaultIsSilentIsSealedByTheController(t *testing.T) {
|
||||
open, vault := aCoreMesh(t, link.VaultBoxContract)
|
||||
vault.silent = errors.New("no responders")
|
||||
plan, settings, err := planFor(t.Context(), open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationFor(t.Context(), open, "laptop", plan, settings)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := clientsBroker(declared.Resources)
|
||||
if f == nil || f["sealed"] == nil || f[catalogue.VaultSealedField] != nil {
|
||||
t.Fatalf("the core credential is composed as %v: the controller's own seal, until the vault holds it", f)
|
||||
}
|
||||
if why, left := declared.leftOutWhy["mesh-provisioner-client"]; left {
|
||||
t.Fatalf("the core module was left out: %v", why)
|
||||
}
|
||||
c, found := vaultSilence(t, "laptop")
|
||||
if !found || !strings.Contains(c.Summary, "sealed by the controller until the vault holds it") {
|
||||
t.Fatalf("the condition does not say why: %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnEngineBelowTheContractIsSentTheControllersSealAsBefore(t *testing.T) {
|
||||
open, vault := aCoreMesh(t, link.HandedContract)
|
||||
f := clientsBroker(sentTo(t, open, "laptop"))
|
||||
if f == nil || f["sealed"] == nil || f[catalogue.VaultSealedField] != nil {
|
||||
t.Fatalf("an older engine is sent %v", f)
|
||||
}
|
||||
if len(vault.asked) != 0 {
|
||||
t.Fatalf("the vault was asked %v for an older engine", vault.asked)
|
||||
}
|
||||
}
|
||||
@@ -1,131 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The core's own credentials travel sealed by the vault to their machine (novox/hq ADR 0349 §16, issue 755; the lead's
|
||||
// decision, issue 755 notes 4 and 5): to an engine that opens the vault's box, the send carries the box the vault
|
||||
// answered the controller and never the controller's own seal; when the vault does not answer, a kept box is carried
|
||||
// again and a module with none is sealed by the controller (ADR 0396, issue 908), with a condition naming the vault and the machine. An engine below the
|
||||
// contract composes exactly as before, and the vault is not asked.
|
||||
|
||||
// coreStandIn answers `get` with a box per name and generation, or its silence.
|
||||
type coreStandIn struct {
|
||||
asked []string
|
||||
keys []string
|
||||
generation int
|
||||
silent error
|
||||
}
|
||||
|
||||
func (v *coreStandIn) ask(name, key string) (coreAnswer, error) {
|
||||
v.asked, v.keys = append(v.asked, name), append(v.keys, key)
|
||||
if v.silent != nil {
|
||||
return coreAnswer{}, v.silent
|
||||
}
|
||||
box := `{"v":1,"eph":"ZQ==","iv":"aQ==","tag":"dA==","ct":"` + strings.Repeat("c", len(v.asked)) + `"}`
|
||||
return coreAnswer{Name: name, Generation: v.generation, Sealed: json.RawMessage(box)}, nil
|
||||
}
|
||||
|
||||
func withVault(t *testing.T, v *coreStandIn) {
|
||||
t.Helper()
|
||||
before := coreVault
|
||||
coreVault = func(context.Context) coreAsk { return v.ask }
|
||||
t.Cleanup(func() { coreVault = before })
|
||||
}
|
||||
|
||||
func clientsBroker(resources []map[string]any) map[string]any {
|
||||
for _, r := range resources {
|
||||
if r["id"] == "mesh-provisioner-client.needs-"+broker.ProvisionerSecret {
|
||||
return r
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func vaultSilence(t *testing.T, node string) (conditions.Condition, bool) {
|
||||
t.Helper()
|
||||
c, found, err := conditionsFrom.Get(t.Context(), vaultSilentObservation(node, coreBoxes{}).Key())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c, found
|
||||
}
|
||||
|
||||
// Issue 908: a machine at the vault-box contract, the vault silent, no kept box: nats is composed with the
|
||||
// controller's own seal, never left out, and the vault-silent condition names it.
|
||||
func TestReplay908(t *testing.T) {
|
||||
key := "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8="
|
||||
core := map[string][]string{"nats": {"password"}}
|
||||
ask := func(name, publicKey string) (coreAnswer, error) { return coreAnswer{}, errors.New("no responders") }
|
||||
got := coreBoxesOf("novox", key, core, nil, Allocating, ask)
|
||||
if _, sealed := got.sealed["nats"]; sealed || len(got.notPlaced) != 0 {
|
||||
t.Fatalf("nats is vault-sealed %v or not placed %v", got.sealed, got.notPlaced)
|
||||
}
|
||||
if !reflect.DeepEqual(got.byController, []string{"nats"}) {
|
||||
t.Fatalf("nats is not left to the controller's seal: %+v", got)
|
||||
}
|
||||
o := vaultSilentObservation("novox", got)
|
||||
if o.Severity != conditions.Warning || !strings.Contains(o.Summary, "sealed by the controller until the vault holds it: nats") {
|
||||
t.Fatalf("the condition does not name nats: %+v", o)
|
||||
}
|
||||
// A kept box is still carried while the vault is silent.
|
||||
kept := map[string]map[string]inventory.CoreBox{"nats": {"password": {SealedTo: key, Box: json.RawMessage(`{"v":1}`)}}}
|
||||
got = coreBoxesOf("novox", key, core, kept, Allocating, ask)
|
||||
if string(got.sealed["nats"]["password"]) != `{"v":1}` || len(got.byController) != 0 {
|
||||
t.Fatalf("the kept box was not held: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// An engine below VaultBoxContract is sent the controller's own seal, as before, and the vault is not asked.
|
||||
|
||||
// Every question but a send reads the kept box and asks nothing; with none kept, a stand-in the next send asks for.
|
||||
func TestOnlyASendAsksTheVault(t *testing.T) {
|
||||
core := map[string][]string{"nats": {"broker"}}
|
||||
key := aPublicKey(t)
|
||||
kept := map[string]map[string]inventory.CoreBox{"nats": {"broker": {Module: "nats", Name: "broker",
|
||||
Generation: 1, SealedTo: key, Box: json.RawMessage(`{"v":1}`)}}}
|
||||
never := func(string, string) (coreAnswer, error) {
|
||||
t.Fatal("the vault was asked by a question that is not a send")
|
||||
return coreAnswer{}, nil
|
||||
}
|
||||
for _, choosing := range []Choosing{Reading, Foreseeing} {
|
||||
if got := coreBoxesOf("anchor", key, core, kept, choosing, never); string(got.sealed["nats"]["broker"]) != `{"v":1}` {
|
||||
t.Errorf("read %s", got.sealed["nats"]["broker"])
|
||||
}
|
||||
if got := coreBoxesOf("anchor", key, core, nil, choosing, never); string(got.sealed["nats"]["broker"]) !=
|
||||
string(catalogue.ForeseenVaultBox) {
|
||||
t.Errorf("with none kept, read %s", got.sealed["nats"]["broker"])
|
||||
}
|
||||
}
|
||||
// A box kept for another key of the machine is never carried.
|
||||
other := coreBoxesOf("anchor", aPublicKey(t), core, kept, Allocating,
|
||||
func(string, string) (coreAnswer, error) { return coreAnswer{}, errors.New("silent") })
|
||||
if _, vaultSealed := other.sealed["nats"]; vaultSealed || !reflect.DeepEqual(other.byController, []string{"nats"}) {
|
||||
t.Fatalf("a box sealed to the machine's old key was carried: %+v", other)
|
||||
}
|
||||
}
|
||||
|
||||
// The core's modules on a machine: on the list, not handed, with their own secrets but one issued outside.
|
||||
func TestTheCoresModulesAreTheListsWithTheirOwnSecrets(t *testing.T) {
|
||||
modules := []catalogue.Manifest{
|
||||
{Module: "nats", OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/a"}, "given": {Path: "/b",
|
||||
IssuedBy: catalogue.IssuedOutside}}},
|
||||
{Module: "postgres", OwnSecrets: catalogue.OwnSecrets{"superuser": {Path: "/c"}}},
|
||||
{Module: "mesh-provisioner", OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/d"}}},
|
||||
}
|
||||
got := coreModulesOf(modules, func(m catalogue.Manifest) bool { return m.Module == "mesh-provisioner" })
|
||||
if want := map[string][]string{"nats": {"broker"}}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("got %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
@@ -1,215 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
)
|
||||
|
||||
// **No send but a person's `push <node>` and a gate's own put-back moves the mesh's own core to a build older
|
||||
// than the one its machine was last sent** (novox/hq issue 201). Every send made through sendToEach keeps it — a
|
||||
// plan's, a release's, a healer's, a rotation's, and the bus step too, which is a person's act but composes under
|
||||
// sendKeeps like the rest. On 2026-10-02 two merges a minute apart made two plans;
|
||||
// the controller's plan rolled the new controller, which widened its seat's row, and the node-engine's plan then
|
||||
// sent the control node the controller build before it. The older binary read the row it could not serve and the
|
||||
// mesh lost its voice for ten minutes — the thing that would have corrected it was the thing that was down. ADR
|
||||
// 0185 took the outage out of it (a controller behind its row serves what it can); the race is closed here,
|
||||
// without needing its trigger: whatever made the mesh's record older, the send keeps the build.
|
||||
//
|
||||
// **Kept, not refused.** The send composes the controller or the node-engine at the build the machine was last
|
||||
// sent, as a recorded module is kept (ADR 0245), and carries everything else: no other module, secret, healer or
|
||||
// walk waits on it, and nothing is retried. The kept move is no move for a gate either: ungatedIn and gatedSend
|
||||
// leave it out (withoutKept), so a gate judges the other moves of its send. A plan whose own build is the older one finds its gate superseded by
|
||||
// the build kept there (issue 352) and ends. The mesh's record stays older, and a warning condition says so until
|
||||
// a send carries the record — a person's `push <node>` accepts the older build — or the record moves forward: a
|
||||
// newer merge, or `rebuild mesh-controller`.
|
||||
//
|
||||
// Older is by the builds' registration order — when each was asked (novox/hq issue 219) — never by the commits'
|
||||
// dates. So a retry or a rebuild of an old commit, asked anew, counts as newer: it is registered as the module's
|
||||
// current build by the same order, and the guard does not stop that move. A build that failed its gate is never
|
||||
// ahead: the machine was sent it and the record was put back before it (gateFailed), and if the put-back's own
|
||||
// send was refused, the next send completes it rather than keeping the failed build there. A gate's own put-back
|
||||
// carries its modules back whatever the order (sendScope.puttingBack); the controller's is held to the store's
|
||||
// schema already (issue 352).
|
||||
|
||||
// kindCoreDowngrade is a core module kept at the build its machine was last sent, over an older record.
|
||||
const kindCoreDowngrade = "core-downgrade-kept"
|
||||
|
||||
// coreDowngradeSource is the condition's source: the sends, which raise and clear it.
|
||||
const coreDowngradeSource = "sends"
|
||||
|
||||
// downgradeGuarded is the core components a send may not move backwards: the controller, which reads the
|
||||
// store and its seat's row, and the node-engine, which reads what the controller declares.
|
||||
var downgradeGuarded = []string{lease.ComponentController, lease.ComponentEngine}
|
||||
|
||||
// askedBuild is what the order needs of one build of a module: when it was asked, and whether it failed its gate.
|
||||
type askedBuild struct {
|
||||
asked time.Time
|
||||
failedGate bool
|
||||
}
|
||||
|
||||
// buildAsked is the newest build of a module from a commit that worked — when it was asked (or recorded, when
|
||||
// the request time is not known), and whether it failed its gate — and false when the records hold none. A
|
||||
// variable, so a test reads its own order.
|
||||
var buildAsked = func(ctx context.Context, inv *inventory.Inventory, module, commit string) (askedBuild, bool, error) {
|
||||
builds, err := inv.Builds(ctx, module, askedLookBack)
|
||||
if err != nil {
|
||||
return askedBuild{}, false, err
|
||||
}
|
||||
for _, b := range builds {
|
||||
if b.Worked() && sameCommit(b.Commit, commit) {
|
||||
failed, err := inv.GateFailed(ctx, b.ID)
|
||||
if err != nil {
|
||||
return askedBuild{}, false, err
|
||||
}
|
||||
return askedBuild{asked: b.AskedOrAt(), failedGate: failed}, true, nil
|
||||
}
|
||||
}
|
||||
return askedBuild{}, false, nil
|
||||
}
|
||||
|
||||
// askedLookBack is how many of a module's newest builds are read for when a build was asked.
|
||||
const askedLookBack = 50
|
||||
|
||||
// coreDowngrade is one core module a machine was last sent a build of that is newer than the mesh's record.
|
||||
type coreDowngrade struct {
|
||||
module, node, component string
|
||||
// sent is the build the machine was last sent, record the one the mesh holds.
|
||||
sent, record string
|
||||
sentAsked, recordAsked time.Time
|
||||
}
|
||||
|
||||
func (d coreDowngrade) String() string {
|
||||
return fmt.Sprintf("%s was last sent %s %s, and the mesh holds the older %s (asked %s, before %s): a send there "+
|
||||
"keeps %s; `push %s` sends the older build, and a newer merge or `rebuild %s` moves the record forward "+
|
||||
"(novox/hq issue 201)", d.node, d.module, short(d.sent), short(d.record),
|
||||
d.recordAsked.UTC().Format(time.RFC3339), d.sentAsked.UTC().Format(time.RFC3339), short(d.sent), d.node,
|
||||
d.module)
|
||||
}
|
||||
|
||||
// coreDowngrades is, for one machine, each guarded core module whose record is older than the build the machine
|
||||
// was last sent, leaving out what a gate's own put-back puts back and a sent build that failed its gate. A move
|
||||
// whose either build the records do not hold is not proved older and is left to the gate that judges it. Pure
|
||||
// but for asked.
|
||||
func coreDowngrades(node string, sent map[string]string, record map[string]string, scope sendScope,
|
||||
asked func(module, commit string) (askedBuild, bool, error)) ([]coreDowngrade, error) {
|
||||
var out []coreDowngrade
|
||||
for module, was := range sent {
|
||||
component := coreComponent(module)
|
||||
now := record[module]
|
||||
if !guarded(component) || was == "" || now == "" || sameCommit(was, now) {
|
||||
continue
|
||||
}
|
||||
if scope.puttingBack && scope.modules[module] {
|
||||
continue
|
||||
}
|
||||
from, known, err := asked(module, was)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", module, short(was), err)
|
||||
}
|
||||
if !known || from.failedGate {
|
||||
continue
|
||||
}
|
||||
to, known, err := asked(module, now)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", module, short(now), err)
|
||||
}
|
||||
if known && to.asked.Before(from.asked) {
|
||||
out = append(out, coreDowngrade{module: module, node: node, component: component, sent: was, record: now,
|
||||
sentAsked: from.asked, recordAsked: to.asked})
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].module < out[j].module })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func guarded(component string) bool {
|
||||
for _, c := range downgradeGuarded {
|
||||
if c == component {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// coreKept is, for a send that keeps (sendKeeps), each guarded core module of a machine to keep at the build it
|
||||
// was last sent, module → commit, each said and raised as a condition.
|
||||
func coreKept(ctx context.Context, inv *inventory.Inventory, node string, sent map[string]string,
|
||||
current map[string]inventory.CurrentBuild) (map[string]string, error) {
|
||||
record := map[string]string{}
|
||||
for m, c := range current {
|
||||
record[m] = c.Commit
|
||||
}
|
||||
found, err := coreDowngrades(node, sent, record, scopeOf(ctx), func(module, commit string) (askedBuild, bool, error) {
|
||||
return buildAsked(ctx, inv, module, commit)
|
||||
})
|
||||
if err != nil || len(found) == 0 {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]string{}
|
||||
for _, d := range found {
|
||||
out[d.module] = d.sent
|
||||
sayCoreDowngrade(ctx, d)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// coreDowngradeObservation is a kept core build as a condition: `core.<component>.<machine>.<kind>`.
|
||||
func coreDowngradeObservation(component, node, summary string) conditions.Observation {
|
||||
return conditions.Observation{Scope: conditions.ScopeCore, ID: component + "." + node, Kind: kindCoreDowngrade,
|
||||
Token: kindCoreDowngrade, Machine: node, Severity: conditions.Warning, Source: coreDowngradeSource,
|
||||
Summary: summary}
|
||||
}
|
||||
|
||||
func sayCoreDowngrade(ctx context.Context, d coreDowngrade) {
|
||||
fmt.Printf("KEPT: %s\n", d)
|
||||
keeper := conditionsFrom
|
||||
if keeper == nil {
|
||||
return
|
||||
}
|
||||
if _, err := keeper.Observe(ctx, coreDowngradeObservation(d.component, d.node, d.String())); err != nil {
|
||||
fmt.Printf("the condition for %s kept on %s could not be raised: %v\n", d.module, d.node, err)
|
||||
}
|
||||
}
|
||||
|
||||
// settleCoreDowngrades clears, after a send to a machine is away, the condition of each guarded core module the
|
||||
// send carried at the mesh's record — a person's push that accepted the older build, or a send after the record
|
||||
// moved forward — or did not carry at all. A send that kept the newer build leaves it open.
|
||||
func settleCoreDowngrades(ctx context.Context, inv *inventory.Inventory, node string, carried map[string]string) {
|
||||
keeper := conditionsFrom
|
||||
if keeper == nil || carried == nil {
|
||||
return
|
||||
}
|
||||
var open []string
|
||||
for _, c := range downgradeGuarded {
|
||||
if _, found, err := keeper.Get(ctx, coreDowngradeObservation(c, node, "").Key()); err == nil && found {
|
||||
open = append(open, c)
|
||||
}
|
||||
}
|
||||
if len(open) == 0 {
|
||||
return
|
||||
}
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("whether %s still keeps a core build over the mesh's record cannot be read: %v\n", node, err)
|
||||
return
|
||||
}
|
||||
for module, now := range current {
|
||||
component := coreComponent(module)
|
||||
if !guarded(component) {
|
||||
continue
|
||||
}
|
||||
if was, has := carried[module]; has && !sameCommit(was, now.Commit) {
|
||||
continue
|
||||
}
|
||||
key := coreDowngradeObservation(component, node, "").Key()
|
||||
if _, err := keeper.Clear(ctx, key, node+" was sent the "+component+" the mesh holds"); err != nil {
|
||||
fmt.Printf("the condition %s could not be cleared: %v\n", key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,3 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
|
||||
@@ -1,148 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
func TestAMovedPathIsNoShrinkAndAShrinkThereStillIs(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
ctx := t.Context()
|
||||
shelf := shelfFor(t, houseManifest)
|
||||
declared := []inventory.DeclaredData{{Module: "house", Item: "config", Class: "irreplaceable", Owned: true}}
|
||||
start := time.Now().Add(-6 * time.Hour)
|
||||
measure := func(at time.Time, path string, size int64) []conditions.Observation {
|
||||
t.Helper()
|
||||
if _, err := inv.RecordData(ctx, "home", declared, map[string]map[string]inventory.Measurement{"house": {
|
||||
"config": {Path: path, Size: bytesOf(size), MeasuredAt: when(at), LastWrite: when(at),
|
||||
LastBackup: when(at)}}}, "", at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
records, err := inv.Data(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
peaks, err := inv.DataPeaks(ctx, at.Add(-shrinkWindow))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return dataFindings(records, peaks, shelf, nil, nil, at)
|
||||
}
|
||||
measure(start, "/home/operator/.claude", 94_700_000)
|
||||
if got := findingsByKind(measure(start.Add(10*time.Minute), "/home/agent/.claude", 490)); got[kindDataShrank].Kind != "" {
|
||||
t.Fatalf("a moved path raised a shrink: %+v", got[kindDataShrank])
|
||||
}
|
||||
measure(start.Add(2*time.Hour), "/home/agent/.claude", 300<<20)
|
||||
got := findingsByKind(measure(start.Add(4*time.Hour), "/home/agent/.claude", 1<<20))[kindDataShrank]
|
||||
if got.Severity != conditions.Urgent || !strings.Contains(got.Summary, "shrank") {
|
||||
t.Fatalf("a genuine shrink at the new path was not raised: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "keeper", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}})
|
||||
house, err := catalogue.ParseManifest([]byte(houseManifest))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, house)
|
||||
if _, err := assign(ctx, open, "laptop", "house"); err == nil {
|
||||
t.Fatal("irreplaceable data was assigned to a machine with nothing to back it up")
|
||||
}
|
||||
for _, node := range []string{"laptop", "anchor"} {
|
||||
if _, err := assign(ctx, open, node, "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "house"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Sent, and applied: a holder is asked only once it has been (novox/hq issue 275).
|
||||
for _, node := range []string{"laptop", "anchor"} {
|
||||
pushedAndApplied(t, open, node)
|
||||
}
|
||||
|
||||
conn := onATestBus(t)
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
laptop, anchor := &fakeHolder{}, &fakeHolder{}
|
||||
laptop.serve(t, conn, "laptop")
|
||||
anchor.serve(t, conn, "anchor")
|
||||
now := time.Now()
|
||||
heard := &watchdogs{last: &signalFacts{now: now, machines: []machineFacts{
|
||||
{name: "laptop", lastHeard: now}, {name: "anchor", lastHeard: now}}}}
|
||||
d := &doctor{open: open, js: js, watchdogs: heard}
|
||||
var d13 probe
|
||||
for _, p := range probeRegistry {
|
||||
if p.ID == probeDataID {
|
||||
d13 = p
|
||||
}
|
||||
}
|
||||
run := func() []conditions.Observation {
|
||||
t.Helper()
|
||||
probing := context.WithValue(ctx, probeAsksKey{}, d13)
|
||||
obs, err := probeData(probing, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return obs
|
||||
}
|
||||
|
||||
laptop.set(measuredHouse("/var/lib/house/config", 2<<30, now))
|
||||
if obs := run(); len(obs) != 0 {
|
||||
t.Fatalf("a measured, backed-up item raised %+v", obs)
|
||||
}
|
||||
r, err := inv.DataOf(ctx, "laptop", "house", "config")
|
||||
if err != nil || r.Path != "/var/lib/house/config" || r.Size == nil || *r.Size != 2<<30 || r.LastBackup == nil {
|
||||
t.Fatalf("what the holder measured was not kept: %+v, %v", r, err)
|
||||
}
|
||||
|
||||
said, err := unassign(ctx, open, "laptop", "house")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(said, "house's config (/var/lib/house/config, 2.0 GB) stays where it is") {
|
||||
t.Fatalf("the unassignment does not say the data stays:\n%s", said)
|
||||
}
|
||||
laptop.set()
|
||||
run()
|
||||
r, err = inv.DataOf(ctx, "laptop", "house", "config")
|
||||
if err != nil || !r.Retired() || r.Path != "/var/lib/house/config" {
|
||||
t.Fatalf("unassigned, the irreplaceable item is not kept retired: %+v, %v", r, err)
|
||||
}
|
||||
records, _ := inv.Data(ctx)
|
||||
if rows := retiredData(records, time.Now()); len(rows) != 1 || rows[0].Path != "/var/lib/house/config" {
|
||||
t.Fatalf("cleanup list: %+v", rows)
|
||||
}
|
||||
|
||||
// Assigned on the anchor, onto an empty directory.
|
||||
if _, err := assign(ctx, open, "anchor", "house"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
anchor.set(measuredHouse("/var/lib/house/config", 300<<10, time.Now()))
|
||||
obs := findingsByKind(run())
|
||||
o, ok := obs[kindEmptyReplacement]
|
||||
if !ok || o.Severity != conditions.Urgent || o.Machine != "anchor" || o.Also[0] != "laptop" {
|
||||
t.Fatalf("an empty replacement of a module's data was not urgent: %+v", obs)
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -9,11 +10,14 @@ import (
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
func bytesOf(n int64) *int64 { return &n }
|
||||
@@ -165,11 +169,6 @@ func TestAShrinkOfMoreThanHalfIsUrgent(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// THE FALSE ALARM (issue 368), replayed through the store D13 reads: an agent's home moved from the
|
||||
// operator's own home (94.7 MB) to the agent account's fresh one (490 B), and `data-shrank` was raised
|
||||
// for data that was never lost. A moved item is read against its new path only, so nothing is raised —
|
||||
// and a genuine shrink at the new path, a week of history later, still is.
|
||||
|
||||
// Data said to be written all the time and not written; data with no backup or an old one — urgent when
|
||||
// irreplaceable, a warning when valuable; and a new item given its bound before it is said.
|
||||
func TestQuietDataAndMissingBackupsAreSaidByClass(t *testing.T) {
|
||||
@@ -276,6 +275,103 @@ func measuredHouse(path string, size int64, at time.Time) map[string]any {
|
||||
// directory is an empty replacement — through the real stores and a real bus. The unassignment says the
|
||||
// data stays; the self-check's next run retires it (kept, listed by cleanup), and when the module comes
|
||||
// up on another machine with an empty directory while the full one waits retired, that is urgent.
|
||||
func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "keeper", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}})
|
||||
house, err := catalogue.ParseManifest([]byte(houseManifest))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, house)
|
||||
if _, err := assign(ctx, open, "laptop", "house"); err == nil {
|
||||
t.Fatal("irreplaceable data was assigned to a machine with nothing to back it up")
|
||||
}
|
||||
for _, node := range []string{"laptop", "anchor"} {
|
||||
if _, err := assign(ctx, open, node, "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "house"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Sent, and applied: a holder is asked only once it has been (novox/hq issue 275).
|
||||
for _, node := range []string{"laptop", "anchor"} {
|
||||
pushedAndApplied(t, open, node)
|
||||
}
|
||||
|
||||
conn := onATestBus(t)
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
laptop, anchor := &fakeHolder{}, &fakeHolder{}
|
||||
laptop.serve(t, conn, "laptop")
|
||||
anchor.serve(t, conn, "anchor")
|
||||
now := time.Now()
|
||||
heard := &watchdogs{last: &signalFacts{now: now, machines: []machineFacts{
|
||||
{name: "laptop", lastHeard: now}, {name: "anchor", lastHeard: now}}}}
|
||||
d := &doctor{open: open, js: js, watchdogs: heard}
|
||||
var d13 probe
|
||||
for _, p := range probeRegistry {
|
||||
if p.ID == probeDataID {
|
||||
d13 = p
|
||||
}
|
||||
}
|
||||
run := func() []conditions.Observation {
|
||||
t.Helper()
|
||||
probing := context.WithValue(ctx, probeAsksKey{}, d13)
|
||||
obs, err := probeData(probing, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return obs
|
||||
}
|
||||
|
||||
laptop.set(measuredHouse("/var/lib/house/config", 2<<30, now))
|
||||
if obs := run(); len(obs) != 0 {
|
||||
t.Fatalf("a measured, backed-up item raised %+v", obs)
|
||||
}
|
||||
r, err := inv.DataOf(ctx, "laptop", "house", "config")
|
||||
if err != nil || r.Path != "/var/lib/house/config" || r.Size == nil || *r.Size != 2<<30 || r.LastBackup == nil {
|
||||
t.Fatalf("what the holder measured was not kept: %+v, %v", r, err)
|
||||
}
|
||||
|
||||
said, err := unassign(ctx, open, "laptop", "house")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(said, "house's config (/var/lib/house/config, 2.0 GB) stays where it is") {
|
||||
t.Fatalf("the unassignment does not say the data stays:\n%s", said)
|
||||
}
|
||||
laptop.set()
|
||||
run()
|
||||
r, err = inv.DataOf(ctx, "laptop", "house", "config")
|
||||
if err != nil || !r.Retired() || r.Path != "/var/lib/house/config" {
|
||||
t.Fatalf("unassigned, the irreplaceable item is not kept retired: %+v, %v", r, err)
|
||||
}
|
||||
records, _ := inv.Data(ctx)
|
||||
if rows := retiredData(records, time.Now()); len(rows) != 1 || rows[0].Path != "/var/lib/house/config" {
|
||||
t.Fatalf("cleanup list: %+v", rows)
|
||||
}
|
||||
|
||||
// Assigned on the anchor, onto an empty directory.
|
||||
if _, err := assign(ctx, open, "anchor", "house"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
anchor.set(measuredHouse("/var/lib/house/config", 300<<10, time.Now()))
|
||||
obs := findingsByKind(run())
|
||||
o, ok := obs[kindEmptyReplacement]
|
||||
if !ok || o.Severity != conditions.Urgent || o.Machine != "anchor" || o.Also[0] != "laptop" {
|
||||
t.Fatalf("an empty replacement of a module's data was not urgent: %+v", obs)
|
||||
}
|
||||
}
|
||||
|
||||
// Data on redundant storage: the array it is on is watched, one condition per array as loud as the most
|
||||
// precious item on it; an item said to be on redundancy and found on plain storage is said; an item
|
||||
|
||||
@@ -144,15 +144,11 @@ func actOnDeadLetter(ctx context.Context, on *busHandles, act string, id uint64,
|
||||
}
|
||||
switch act {
|
||||
case "deliver":
|
||||
delivered, to, err := link.DeliverAgain(on.js, id)
|
||||
_, to, err := link.DeliverAgain(on.js, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
answer["delivered_on"] = to
|
||||
if delivered.Original != "" {
|
||||
// What became of the ask in its seat's queue (novox/hq issue 334): removed, or left, and why.
|
||||
answer["original"] = delivered.Original
|
||||
}
|
||||
answer["done"] = fmt.Sprintf("dead letter %d was delivered again to %s, and nobody else; it is no longer kept",
|
||||
id, consumerWho(d.Stream, d.Consumer))
|
||||
case "drop":
|
||||
|
||||
@@ -1,117 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A dead letter from a restricted stream ages out with its stream (novox/hq ADR 0376 §3, §6).
|
||||
//
|
||||
// DEAD_LETTERS lets such a message go when its stream's maximum age runs out, so a consumer's `max-deliveries`
|
||||
// condition can clear with nobody having delivered it again or dropped it. That is said: the watchdogs remember,
|
||||
// from one look to the next, when each restricted dead letter ages out, count those gone once their time passed,
|
||||
// say each in the controller's log as it goes, and clear the condition with words saying how many aged out
|
||||
// undelivered and from which stream.
|
||||
|
||||
// agingTally is what the watchdogs remember of restricted dead letters between looks.
|
||||
type agingTally struct {
|
||||
// seen is each consumer's restricted dead letters at the last look, by id, with when each ages out.
|
||||
seen map[string]map[uint64]time.Time
|
||||
// agedOut is how many of each consumer's aged out undelivered while its condition was open.
|
||||
agedOut map[string]int
|
||||
}
|
||||
|
||||
// look compares this look's restricted dead letters with the last: newly is how many of each consumer's aged out
|
||||
// since, and cleared how many aged out in all for each consumer DEAD_LETTERS no longer holds anything of, when any
|
||||
// did. held is every consumer's count this look; aging its restricted dead letters.
|
||||
func (t *agingTally) look(now time.Time, held map[string]int, aging map[string][]link.Aging) (newly, cleared map[string]int) {
|
||||
if t.seen == nil {
|
||||
t.seen, t.agedOut = map[string]map[uint64]time.Time{}, map[string]int{}
|
||||
}
|
||||
newly, cleared = map[string]int{}, map[string]int{}
|
||||
for key, before := range t.seen {
|
||||
still := map[uint64]bool{}
|
||||
for _, a := range aging[key] {
|
||||
still[a.ID] = true
|
||||
}
|
||||
for id, agesOut := range before {
|
||||
if !still[id] && !agesOut.After(now) {
|
||||
newly[key]++
|
||||
}
|
||||
}
|
||||
if newly[key] == 0 {
|
||||
delete(newly, key)
|
||||
}
|
||||
t.agedOut[key] += newly[key]
|
||||
}
|
||||
for key, n := range t.agedOut {
|
||||
if held[key] > 0 {
|
||||
continue
|
||||
}
|
||||
if n > 0 {
|
||||
cleared[key] = n
|
||||
}
|
||||
delete(t.agedOut, key)
|
||||
}
|
||||
t.seen = map[string]map[uint64]time.Time{}
|
||||
for key, letters := range aging {
|
||||
t.seen[key] = map[uint64]time.Time{}
|
||||
for _, a := range letters {
|
||||
t.seen[key][a.ID] = a.AgesOut
|
||||
}
|
||||
}
|
||||
return newly, cleared
|
||||
}
|
||||
|
||||
// agedOutResolved is the line a consumer's condition clears with when messages it gave up on aged out undelivered.
|
||||
func agedOutResolved(key string, n int) string {
|
||||
stream, _, _ := strings.Cut(key, ".")
|
||||
messages := "a message"
|
||||
if n > 1 {
|
||||
messages = fmt.Sprintf("%d messages", n)
|
||||
}
|
||||
return fmt.Sprintf("Resolved: %s of the stream %s aged out undelivered", messages, stream)
|
||||
}
|
||||
|
||||
// sayAgedOut says, in the log, what aged out since the last look, and clears the condition of every consumer whose
|
||||
// dead letters are all gone with the words saying how many aged out. Before the rows are reconciled, so the
|
||||
// condition is cleared with these words and not its own.
|
||||
func (w *watchdogs) sayAgedOut(ctx context.Context, f *signalFacts) []string {
|
||||
if f.advisoriesErr != nil || f.deadLettersAging == nil {
|
||||
return nil
|
||||
}
|
||||
newly, cleared := w.aging.look(f.now, f.deadLetters, f.deadLettersAging)
|
||||
var problems []string
|
||||
for _, key := range countedKeys(newly) {
|
||||
stream, consumer, _ := strings.Cut(key, ".")
|
||||
fmt.Printf("%d message(s) %s gave up on aged out of %s undelivered: %s keeps them only for its maximum age "+
|
||||
"(novox/hq ADR 0376)\n", newly[key], link.ConsumerInWords(stream, consumer), "DEAD_LETTERS", stream)
|
||||
}
|
||||
for _, key := range countedKeys(cleared) {
|
||||
stream, consumer, _ := strings.Cut(key, ".")
|
||||
fmt.Printf("%s no longer has a message kept: %d aged out of %s undelivered, so its condition clears "+
|
||||
"(novox/hq ADR 0376)\n", link.ConsumerInWords(stream, consumer), cleared[key], stream)
|
||||
if _, err := w.keeper.ClearSaying(ctx, conditions.Key(conditions.ScopeBus, key, link.AdvisoryMaxDeliveries),
|
||||
fmt.Sprintf("its %d dead letter(s) of %s aged out undelivered", cleared[key], stream),
|
||||
agedOutResolved(key, cleared[key])); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// countedKeys is a count's keys in order.
|
||||
func countedKeys(m map[string]int) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A dead letter that ages out clears its condition and says so** (novox/hq ADR 0376 §6), on a fake clock: while
|
||||
// DEAD_LETTERS holds a restricted stream's messages for a consumer its max-deliveries condition is open; once they
|
||||
// have aged out it clears, with words saying how many aged out undelivered and from which stream. One dropped before
|
||||
// its age is not counted as aged out, and a consumer of EVENTS clears as before.
|
||||
func TestADeadLetterThatAgesOutClearsItsConditionAndSaysSo(t *testing.T) {
|
||||
now := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)
|
||||
store := conditions.NewInMemory()
|
||||
told := &conditions.Told{}
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: told,
|
||||
Now: func() time.Time { return now }})
|
||||
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
||||
|
||||
look := func(at time.Time, held map[string]int, aging map[string][]link.Aging) {
|
||||
f := calm(at)
|
||||
f.deadLetters, f.deadLettersAging = held, aging
|
||||
w.see(t.Context(), f)
|
||||
}
|
||||
mail := "MAIL.ace_mailer"
|
||||
look(now, map[string]int{mail: 3, "EVENTS.media_sonarr": 1}, map[string][]link.Aging{mail: {
|
||||
{ID: 1, AgesOut: now.Add(time.Hour)}, {ID: 2, AgesOut: now.Add(2 * time.Hour)}, {ID: 3, AgesOut: now.Add(3 * time.Hour)}}})
|
||||
if open, _ := k.Open(t.Context()); len(open) != 2 {
|
||||
t.Fatalf("open: %+v", open)
|
||||
}
|
||||
// One dropped by a person before its age: not aged out.
|
||||
look(now.Add(30*time.Minute), map[string]int{mail: 2, "EVENTS.media_sonarr": 1}, map[string][]link.Aging{mail: {
|
||||
{ID: 2, AgesOut: now.Add(2 * time.Hour)}, {ID: 3, AgesOut: now.Add(3 * time.Hour)}}})
|
||||
look(now.Add(150*time.Minute), map[string]int{mail: 1}, map[string][]link.Aging{mail: {{ID: 3, AgesOut: now.Add(3 * time.Hour)}}})
|
||||
open, _ := k.Open(t.Context())
|
||||
if len(open) != 1 || open[0].Key != conditions.Key(conditions.ScopeBus, mail, link.AdvisoryMaxDeliveries) {
|
||||
t.Fatalf("after one aged out: %+v", open)
|
||||
}
|
||||
look(now.Add(4*time.Hour), map[string]int{}, map[string][]link.Aging{})
|
||||
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||
t.Fatalf("after all aged out: %+v", open)
|
||||
}
|
||||
k.Close(context.Background()) // everything said is said once it closes
|
||||
var mailCleared, sonarrCleared string
|
||||
for _, e := range told.Said() {
|
||||
if e.Change != conditions.ChangeCleared {
|
||||
continue
|
||||
}
|
||||
switch e.Condition.Key {
|
||||
case conditions.Key(conditions.ScopeBus, mail, link.AdvisoryMaxDeliveries):
|
||||
mailCleared = e.Condition.Resolved
|
||||
case conditions.Key(conditions.ScopeBus, "EVENTS.media_sonarr", link.AdvisoryMaxDeliveries):
|
||||
sonarrCleared = e.Condition.Resolved
|
||||
}
|
||||
}
|
||||
if mailCleared != "Resolved: 2 messages of the stream MAIL aged out undelivered" {
|
||||
t.Fatalf("MAIL's condition cleared with %q", mailCleared)
|
||||
}
|
||||
if sonarrCleared != "Resolved: the messages it gave up on were delivered again or dropped" {
|
||||
t.Fatalf("an EVENTS consumer's condition cleared with %q", sonarrCleared)
|
||||
}
|
||||
if why, ok := conditions.Plain(mailCleared); !ok {
|
||||
t.Fatalf("not plain: %s", why)
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user