Compare commits
95
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
91a7c1eaff | ||
|
|
08e000bb56 | ||
|
|
a39d6ce78d | ||
|
|
c531c78eb5 | ||
|
|
9a20547dbd | ||
|
|
7c9c187ca6 | ||
|
|
2d24fbb716 | ||
|
|
9cdc7d6276 | ||
|
|
1a309a3783 | ||
|
|
b77a298a7e | ||
|
|
64415f8984 | ||
|
|
07b050790e | ||
|
|
8db72edd3c | ||
|
|
66dc501fc4 | ||
|
|
aac4bae5a4 | ||
|
|
8b3a9163d0 | ||
|
|
e43d52c1ea | ||
|
|
0d83c6ffc9 | ||
|
|
b752b1f1df | ||
|
|
d209de58b7 | ||
|
|
14fe2ad736 | ||
|
|
d977f64ec9 | ||
|
|
d5d5449835 | ||
|
|
874a91a3c6 | ||
|
|
4669a8d1ab | ||
|
|
34197afa01 | ||
|
|
0125ccd7d2 | ||
|
|
fe5472bab4 | ||
|
|
48f5a7d3a9 | ||
|
|
4864a43413 | ||
|
|
cd88dbd336 | ||
|
|
65d570f114 | ||
|
|
1586bd1134 | ||
|
|
c43772fd94 | ||
|
|
7ee2b42230 | ||
|
|
e42becb479 | ||
|
|
43b59c5dff | ||
|
|
81715273d3 | ||
|
|
085b661936 | ||
|
|
21414307c8 | ||
|
|
b8c810a349 | ||
|
|
0f21fe7888 | ||
|
|
bd257afbd5 | ||
|
|
357e08a4f6 | ||
|
|
a83b921af0 | ||
|
|
3d09d5235f | ||
|
|
f883830975 | ||
|
|
96325c295c | ||
|
|
9ca2c2bfb3 | ||
|
|
20c8d9454a | ||
|
|
3d192a4857 | ||
|
|
8af77c2c75 | ||
|
|
fd522f4322 | ||
|
|
7d80155d16 | ||
|
|
bafc31bf85 | ||
|
|
fbf5a3f909 | ||
|
|
20e14872ef | ||
|
|
5702187695 | ||
|
|
a79182e53b | ||
|
|
ff16938096 | ||
|
|
e3f22c737f | ||
|
|
2b99d455fc | ||
|
|
9e99f31c23 | ||
|
|
df39a499ef | ||
|
|
11725908ec | ||
|
|
379a52c2e8 | ||
|
|
b0d489a86c | ||
|
|
6355176fda | ||
|
|
300d3c1bf4 | ||
|
|
baf1501d81 | ||
|
|
a5ba47e23c | ||
|
|
6a62a7e552 | ||
|
|
5890eb104d | ||
|
|
262b39ae09 | ||
|
|
2f2a8bb57e | ||
|
|
1cdf12df00 | ||
|
|
f614eeb445 | ||
|
|
53a3c91f53 | ||
|
|
df5f01bfa1 | ||
|
|
32014ac6de | ||
|
|
c7cb9f0a2c | ||
|
|
87410d79ed | ||
|
|
1c3c3edc7d | ||
|
|
b2e2bf0638 | ||
|
|
e231185572 | ||
|
|
b12755c163 | ||
|
|
abf37c7212 | ||
|
|
ead062569e | ||
|
|
f4ddddd59d | ||
|
|
210f7cf5e6 | ||
|
|
9cb08299b3 | ||
|
|
5fa523f608 | ||
|
|
9a5c02bd60 | ||
|
|
9ce77e4e7b | ||
|
|
470c97654b |
@@ -95,7 +95,7 @@ PROXY_IMAGE ?= mesh-route-proxy:$(VERSION)
|
||||
PROXY_DEV_TAG ?= mesh-route-proxy:development
|
||||
|
||||
proxy-image:
|
||||
docker build --build-arg GO_BASE=$(GO_BASE) -f examples/route-proxy/Dockerfile -t $(PROXY_IMAGE) -t $(PROXY_DEV_TAG) .
|
||||
docker build --build-arg GO_BASE=$(GO_BASE) -f modules/route-proxy/Dockerfile -t $(PROXY_IMAGE) -t $(PROXY_DEV_TAG) .
|
||||
@echo
|
||||
@docker image inspect $(PROXY_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||
|
||||
@@ -110,14 +110,18 @@ proxy-image:
|
||||
# after formatting never ran through it. The script is the one list of steps; this target only gives it
|
||||
# a database (MESH_TEST_POSTGRES, exported above).
|
||||
#
|
||||
# **Two levels** (issue 895): the unit level, `go test ./...` untagged, timed and held to a bound; and the integration
|
||||
# level, `-tags integration`, the tests that start a bus, a store or another process. Each is said with its time.
|
||||
#
|
||||
# **Packages in parallel, under the race detector, each test on a bus of its own** (internal/testbus),
|
||||
# as the script runs them: a bus per test, of the release the mesh runs, makes the suite the same in any
|
||||
# order, and the timeout bounds a hang to a failure with a stack, never a stalled gate.
|
||||
check: postgres
|
||||
@sh merge-check.sh ; status=$$? ; docker rm -f $(PG_CONTAINER) >/dev/null 2>&1 || true ; exit $$status
|
||||
|
||||
# Without a database the store's tests skip rather than fail, so this is the honest subset and not
|
||||
# the gate.
|
||||
# The unit level alone (untagged: no bus, no store, no other process), which a developer runs continuously. The tests
|
||||
# that need a bus or the store carry `//go:build integration` and run in `check`, not here; without a database the
|
||||
# store's tests would skip rather than fail anyway.
|
||||
test:
|
||||
go test -timeout 15m ./...
|
||||
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
#!/bin/sh
|
||||
# The unit tests of every Go module of its own below this repository's root, run from the root, for merge-check.sh
|
||||
# and for the test that tries it on a tree (merge_check_nested_test.go). Its own file so that the test needs no
|
||||
# particular merge-check.sh: the check that judges a change to merge-check.sh is main's.
|
||||
set -eu
|
||||
cgo="${cgo:-0}"
|
||||
race="${race:-}"
|
||||
|
||||
# nested_modules vets and tests every Go module of its own below the root, `modules/<name>/go.mod` (novox/hq ADR 0420
|
||||
# §5, issue 1025): `./...` stops at a nested go.mod, so a module that lives whole in this repository would have its
|
||||
# unit tests run by nobody. None is a pass. Each runs in its own directory, outside the vendor directory this
|
||||
# repository's own packages are built from; a nested module vendors what the forge holds (its own `vendor/`, so the
|
||||
# check runs offline), as this repository's root module does (issue 1019). A failing module fails the check; the others are still run.
|
||||
nested_modules() {
|
||||
nested=0
|
||||
for gomod in modules/*/go.mod; do
|
||||
[ -e "$gomod" ] || continue
|
||||
dir=$(dirname "$gomod")
|
||||
echo "nested module $dir:"
|
||||
if ! (cd "$dir" && GOFLAGS= CGO_ENABLED=0 go vet ./... &&
|
||||
GOFLAGS= CGO_ENABLED="${cgo:-0}" go test ${race:-} -count=1 -timeout 30m ./...); then
|
||||
echo "nested module $dir FAILED"
|
||||
nested=1
|
||||
fi
|
||||
done
|
||||
return $nested
|
||||
}
|
||||
|
||||
nested_modules
|
||||
@@ -7,16 +7,8 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A pull request into a release branch reaches the check as one** (novox/hq ADR 0358): the build seat hands the
|
||||
// check the release branch and, beside it, the trunk each repository is read at when it has no branch of that name;
|
||||
// and it hands the check the judge cache the build agent declares (MESH_JUDGE_CACHE), never a path of its own.
|
||||
func TestTheBuildSeatHandsTheCheckItsReleaseBranch(t *testing.T) {
|
||||
spec := checkSpecOf(link.BuildRequest{ID: "b", Repository: "r", Ref: "abc", Check: &link.CheckRequest{
|
||||
Owner: "novox", Repo: "mesh-catalog", Base: "feat/57", Release: "feat/57",
|
||||
Beside: map[string]link.CheckedOut{"mesh-host": {Repository: "h", Ref: "feat/57", Else: "main"}}}})
|
||||
if spec.Release != "feat/57" || spec.Beside["mesh-host"].Ref != "feat/57" || spec.Beside["mesh-host"].Else != "main" {
|
||||
t.Fatalf("the check was handed release %q and mesh-host %+v", spec.Release, spec.Beside["mesh-host"])
|
||||
}
|
||||
// The build seat hands the check the judge cache the build agent declares (MESH_JUDGE_CACHE), never a path of its own.
|
||||
func TestTheBuildSeatHandsTheCheckTheJudgeCacheItDeclares(t *testing.T) {
|
||||
t.Setenv("MESH_JUDGE_CACHE", "/var/lib/mesh/build-agent/judge-cache")
|
||||
if got := judgeCache(); got != "/var/lib/mesh/build-agent/judge-cache" {
|
||||
t.Fatalf("the judge cache the manifest declares is read as %q", got)
|
||||
@@ -27,6 +27,7 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -290,6 +291,26 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
if err := work.Began(ctx); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "cannot say a build started: %v\n", err)
|
||||
}
|
||||
// **A running check watches its pull request's key** (novox/hq ADR 0406 §4, issue 974): when the head or the trunk
|
||||
// commit in it moves, the check is for a state nobody wants any more, and ends itself — nothing is asked of the forge
|
||||
// (issue 726). A group's check, a try and a build watch nothing.
|
||||
var superseded atomic.Bool
|
||||
if w, ok := work.(interface {
|
||||
WatchPull(context.Context, func(link.PullKey)) (func(), error)
|
||||
}); ok && request.Check != nil {
|
||||
stopWatching, err := w.WatchPull(building, func(k link.PullKey) {
|
||||
superseded.Store(true)
|
||||
say("superseded", fmt.Sprintf("the pull request's key moved to %.12s on %.12s while this checked %.12s on %.12s: stopped",
|
||||
k.Head, k.Trunk, request.Ref, request.Check.Trunk))
|
||||
cancel()
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "the key of %s/%s#%d could not be watched, so its check runs on if the key moves: %v\n",
|
||||
request.Check.Owner, request.Check.Repo, request.Check.Number, err)
|
||||
} else {
|
||||
defer stopWatching()
|
||||
}
|
||||
}
|
||||
|
||||
result := link.BuildResult{
|
||||
ID: request.ID, Repository: request.Repository, Path: request.Path,
|
||||
@@ -337,8 +358,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
result.Check = &link.CheckOutcome{Verdict: v.Verdict, Summary: v.Summary, Report: v.Report,
|
||||
Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo),
|
||||
Head: v.Head, Trunk: v.Trunk, Beside: v.Beside, Built: checkBuiltOf(v.Built),
|
||||
Release: v.Release, JudgedBy: v.JudgedBy, BesideAt: v.BesideAt, Validator: v.Validator,
|
||||
Siblings: siblingsOut(v.Siblings), SiblingsUnknown: v.SiblingsUnknown, JudgeFrom: v.JudgeFrom}
|
||||
Validator: v.Validator, Siblings: siblingsOut(v.Siblings), SiblingsUnknown: v.SiblingsUnknown, JudgeFrom: v.JudgeFrom}
|
||||
// What each of a group's heads was judged as (novox/hq ADR 0323 §6).
|
||||
for _, m := range v.Members {
|
||||
result.Check.Members = append(result.Check.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo,
|
||||
@@ -382,7 +402,11 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
if mine != nil {
|
||||
killed = h.returned(mine) && err != nil
|
||||
}
|
||||
if killed {
|
||||
if superseded.Load() && err != nil {
|
||||
// Ended by its own key moving on (ADR 0406 §4): said as the check's failure, as a kill is.
|
||||
result.Failed = link.SupersededByKey
|
||||
say("failed", link.SupersededByKey)
|
||||
} else if killed {
|
||||
// **Killed by hand is the outcome, whatever the build was doing** (novox/hq ADR 0219): the
|
||||
// error it ended with is the kill's consequence, not a fault of the source.
|
||||
result.Failed = link.KilledByHand
|
||||
@@ -473,13 +497,13 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
|
||||
Owner: c.Owner, Repo: c.Repo, Number: c.Number, Paths: c.Paths, Beside: map[string]builder.Beside{},
|
||||
Modules: c.Modules, New: c.New, Manifests: c.Manifests, Judge: c.Judge, Base: c.Base,
|
||||
Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held),
|
||||
Tests: c.Tests, StoreOnly: c.StoreOnly, Release: c.Release, SiblingsUnknown: c.SiblingsUnknown}
|
||||
Tests: c.Tests, StoreOnly: c.StoreOnly, SiblingsUnknown: c.SiblingsUnknown}
|
||||
for _, s := range c.Siblings {
|
||||
spec.Siblings = append(spec.Siblings, builder.Sibling{Repository: s.Repository, Number: s.Number, Head: s.Head,
|
||||
Base: s.Base, Dir: s.Dir})
|
||||
}
|
||||
for dir, b := range c.Beside {
|
||||
spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref, Else: b.Else}
|
||||
spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref}
|
||||
}
|
||||
for _, b := range c.Builds {
|
||||
spec.Builds = append(spec.Builds, builder.CheckBuildSpec{Module: b.Module, Repository: b.Repository, Path: b.Path,
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/licences"
|
||||
)
|
||||
|
||||
func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aStore())
|
||||
if _, err := assign(ctx, open, "anchor", "distribution"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "distribution",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5101}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A module the mesh built, recorded by digest and path, running on the other machine.
|
||||
if err := open.inventory.RecordBuild(ctx, inventory.Build{
|
||||
ID: "b1", Repository: "r", Module: "app", Commit: "abc",
|
||||
Made: []inventory.Artifact{{Name: "server", Kind: "image",
|
||||
Reference: catalogue.ArtifactStoreScheme + "app/server@" + aDigest}},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "app", Version: "1",
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-app",
|
||||
"image": catalogue.ArtifactStoreScheme + "app/server@" + aDigest}}})
|
||||
if _, err := assign(ctx, open, "laptop", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The runtime's trust is the runtime's module's to write (novox/hq ADR 0222): a stand-in for it
|
||||
// asks where this machine reaches the store, as the docker module does.
|
||||
register(t, open, catalogue.Manifest{Module: "runtime", Version: "1",
|
||||
Resources: []map[string]any{{"id": "daemon", "type": "file", "path": "/etc/docker/daemon.json",
|
||||
"into": "json", "content": `{"insecure-registries": ["${seat:mesh-artifact-store:reach}"]}` + "\n"}}})
|
||||
if _, err := assign(ctx, open, "laptop", "runtime"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
on := map[string]bool{"anchor": true, "laptop": true}
|
||||
node, port, found, err := artifactStoreOnNetwork(ctx, open.inventory, on)
|
||||
if err != nil || !found || node != "anchor" || port != "5101" {
|
||||
t.Fatalf("the store is found on %q:%q (%v, %v); the node put it on 5101", node, port, found, err)
|
||||
}
|
||||
|
||||
var trust string
|
||||
for _, r := range composed(t, open, "laptop").Resources {
|
||||
if r["path"] == "/etc/docker/daemon.json" {
|
||||
trust, _ = r["content"].(string)
|
||||
}
|
||||
if r["id"] == "app.server" && r["image"] != "anchor.internal:5101/app/server@"+aDigest {
|
||||
t.Errorf("the image the mesh built is fetched as %v", r["image"])
|
||||
}
|
||||
}
|
||||
if !strings.Contains(trust, "anchor.internal:5101") || strings.Contains(trust, ":5000") {
|
||||
t.Fatalf("the runtime is told to trust %q; the node put the store on 5101", trust)
|
||||
}
|
||||
|
||||
// And a replay to the catalogue says where the store is now.
|
||||
announced, err := following{open: open}.Announceable(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(announced) != 1 || announced[0].Made[0].Reference != "anchor.internal:5101/app/server@"+aDigest {
|
||||
t.Fatalf("the replay announces %+v", announced)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
control, err := withSeatPorts(m).Resolve([]catalogue.Built{{Name: "controller", Kind: catalogue.ArtifactBundle,
|
||||
Reference: "https://registry.example/mesh-controller/controller.tar.gz", Digest: aDigest}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, control)
|
||||
register(t, open, catalogue.Manifest{Module: "postgres", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}},
|
||||
Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store",
|
||||
"ports": []any{"5432:5432"}, "image": "pg@" + aDigest}}})
|
||||
register(t, open, catalogue.Manifest{Module: "lavinmq", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}},
|
||||
Listens: []catalogue.Listening{{Port: 5671, From: catalogue.FromMesh},
|
||||
{Port: 5672, From: catalogue.FromMesh}},
|
||||
Guards: []int{15672},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-broker",
|
||||
"ports": []any{"5671:5671", "5672:5672", "127.0.0.1:15672:15672"}, "image": "mq@" + aDigest}}})
|
||||
// The control plane's own bus user is the installer's, seeded at genesis before the controller
|
||||
// runs (SeedBusUser); without it a push now refuses the credential nobody issued (issue 203).
|
||||
if err := open.inventory.SeedBusUser(ctx, inventory.BusUser{Username: "anchor.mesh-controller",
|
||||
Kind: inventory.BusController, Node: "anchor", Module: "mesh-controller"}, "bootstrap"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "anchor", "mesh-controller"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The store's module is registered and given its port, and NOT assigned: the state genesis
|
||||
// leaves a given-port node in before the foundation is adopted as modules (04-ISSUES/085).
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "postgres",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5432": 6852}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "anchor", "lavinmq"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "lavinmq",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5672": 5679}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var env map[string]any
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
if r["id"] == "mesh-controller.controller" {
|
||||
env, _ = r["env"].(map[string]any)
|
||||
}
|
||||
}
|
||||
if env == nil {
|
||||
t.Fatal("the control plane's process is not in its own node's declaration")
|
||||
}
|
||||
for key, want := range map[string]string{
|
||||
"MESH_STORE_INVENTORY_PORT": "6852",
|
||||
"MESH_STORE_IDENTITY_PORT": "6852",
|
||||
"MESH_STORE_LICENCES_PORT": "6852",
|
||||
"MESH_BROKER_AMQP_PORT": "5679",
|
||||
// Neither given nor assigned by the mesh: the manifest's own number is NOT the answer,
|
||||
// because the sealed value beside it carries the port genesis wrote (finding F3).
|
||||
"MESH_BROKER_ADDRESS_PORT": "",
|
||||
"MESH_BROKER_MANAGEMENT_PORT": "",
|
||||
} {
|
||||
if env[key] != want {
|
||||
t.Errorf("the control plane is told %s=%v; the node says %q", key, env[key], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func aLoneNode(t *testing.T) *stores {
|
||||
t.Helper()
|
||||
inventory.ForTest(t)
|
||||
licences.ForTest(t)
|
||||
open, err := openStores(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(open.Close)
|
||||
for _, m := range provided {
|
||||
if err := open.inventory.Provide(t.Context(), m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
record, err := open.inventory.AddNode(t.Context(), "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reported, _ := json.Marshal(map[string]any{"capabilities": []map[string]any{
|
||||
{"name": "container-runtime", "present": true}}})
|
||||
var profile map[string]any
|
||||
_ = json.Unmarshal(reported, &profile)
|
||||
if err := open.inventory.RecordProfile(t.Context(), record.ID, profile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSealingKey(t.Context(), record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return open
|
||||
}
|
||||
|
||||
func TestOnANodeWithNoNetworkTheStoreIsReachedByLoopback(t *testing.T) {
|
||||
open := aLoneNode(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aStore())
|
||||
register(t, open, catalogue.Manifest{Module: "builder", Version: "1",
|
||||
Requires: []string{catalogue.ArtifactStoreProvision},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-builder",
|
||||
"image": "registry.example/mesh-builder@" + aDigest}}})
|
||||
if err := open.inventory.RecordBuild(ctx, inventory.Build{
|
||||
ID: "b1", Repository: "r", Module: "postgres", Commit: "abc",
|
||||
Made: []inventory.Artifact{{Name: "runtime", Kind: "image",
|
||||
Reference: catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "postgres", Version: "1",
|
||||
Resources: []map[string]any{{"id": "runtime", "type": "container", "name": "mesh-postgres",
|
||||
"image": catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}}})
|
||||
for _, module := range []string{"distribution", "builder", "postgres"} {
|
||||
if _, err := assign(ctx, open, "anchor", module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "distribution",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5100}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var image any
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
if r["id"] == "postgres.runtime" {
|
||||
image = r["image"]
|
||||
}
|
||||
}
|
||||
if image != "127.0.0.1:5100/postgres/runtime@"+aDigest {
|
||||
t.Fatalf("on the store's own node, off any network, the image is fetched as %v", image)
|
||||
}
|
||||
held := heldBy(ctx)
|
||||
if got := held["postgres/runtime"]; got != "127.0.0.1:5100/postgres/runtime@"+aDigest {
|
||||
t.Fatalf("a builder beside the store is handed the base %q", got)
|
||||
}
|
||||
}
|
||||
@@ -2,13 +2,10 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/licences"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
@@ -86,150 +83,12 @@ func aStore() catalogue.Manifest {
|
||||
|
||||
// **The trust a machine writes for the store, and the address every built image is fetched
|
||||
// through, say the port the node gave the store** — not the catalogue's number.
|
||||
func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aStore())
|
||||
if _, err := assign(ctx, open, "anchor", "distribution"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "distribution",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5101}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A module the mesh built, recorded by digest and path, running on the other machine.
|
||||
if err := open.inventory.RecordBuild(ctx, inventory.Build{
|
||||
ID: "b1", Repository: "r", Module: "app", Commit: "abc",
|
||||
Made: []inventory.Artifact{{Name: "server", Kind: "image",
|
||||
Reference: catalogue.ArtifactStoreScheme + "app/server@" + aDigest}},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "app", Version: "1",
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-app",
|
||||
"image": catalogue.ArtifactStoreScheme + "app/server@" + aDigest}}})
|
||||
if _, err := assign(ctx, open, "laptop", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The runtime's trust is the runtime's module's to write (novox/hq ADR 0222): a stand-in for it
|
||||
// asks where this machine reaches the store, as the docker module does.
|
||||
register(t, open, catalogue.Manifest{Module: "runtime", Version: "1",
|
||||
Resources: []map[string]any{{"id": "daemon", "type": "file", "path": "/etc/docker/daemon.json",
|
||||
"into": "json", "content": `{"insecure-registries": ["${seat:mesh-artifact-store:reach}"]}` + "\n"}}})
|
||||
if _, err := assign(ctx, open, "laptop", "runtime"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
on := map[string]bool{"anchor": true, "laptop": true}
|
||||
node, port, found, err := artifactStoreOnNetwork(ctx, open.inventory, on)
|
||||
if err != nil || !found || node != "anchor" || port != "5101" {
|
||||
t.Fatalf("the store is found on %q:%q (%v, %v); the node put it on 5101", node, port, found, err)
|
||||
}
|
||||
|
||||
var trust string
|
||||
for _, r := range composed(t, open, "laptop").Resources {
|
||||
if r["path"] == "/etc/docker/daemon.json" {
|
||||
trust, _ = r["content"].(string)
|
||||
}
|
||||
if r["id"] == "app.server" && r["image"] != "anchor.internal:5101/app/server@"+aDigest {
|
||||
t.Errorf("the image the mesh built is fetched as %v", r["image"])
|
||||
}
|
||||
}
|
||||
if !strings.Contains(trust, "anchor.internal:5101") || strings.Contains(trust, ":5000") {
|
||||
t.Fatalf("the runtime is told to trust %q; the node put the store on 5101", trust)
|
||||
}
|
||||
|
||||
// And a replay to the catalogue says where the store is now.
|
||||
announced, err := following{open: open}.Announceable(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(announced) != 1 || announced[0].Made[0].Reference != "anchor.internal:5101/app/server@"+aDigest {
|
||||
t.Fatalf("the replay announces %+v", announced)
|
||||
}
|
||||
}
|
||||
|
||||
// **The control plane's own connections say the port the node gave the store and the broker.**
|
||||
//
|
||||
// Composed from the control plane's own manifest against a real inventory: the store's module is
|
||||
// given 6852 on this node the way genesis or an operator gives it, and the control plane's
|
||||
// process is told so beside the sealed connection genesis wrote.
|
||||
func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
control, err := withSeatPorts(m).Resolve([]catalogue.Built{{Name: "controller", Kind: catalogue.ArtifactBundle,
|
||||
Reference: "https://registry.example/mesh-controller/controller.tar.gz", Digest: aDigest}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, control)
|
||||
register(t, open, catalogue.Manifest{Module: "postgres", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}},
|
||||
Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store",
|
||||
"ports": []any{"5432:5432"}, "image": "pg@" + aDigest}}})
|
||||
register(t, open, catalogue.Manifest{Module: "lavinmq", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}},
|
||||
Listens: []catalogue.Listening{{Port: 5671, From: catalogue.FromMesh},
|
||||
{Port: 5672, From: catalogue.FromMesh}},
|
||||
Guards: []int{15672},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-broker",
|
||||
"ports": []any{"5671:5671", "5672:5672", "127.0.0.1:15672:15672"}, "image": "mq@" + aDigest}}})
|
||||
// The control plane's own bus user is the installer's, seeded at genesis before the controller
|
||||
// runs (SeedBusUser); without it a push now refuses the credential nobody issued (issue 203).
|
||||
if err := open.inventory.SeedBusUser(ctx, inventory.BusUser{Username: "anchor.mesh-controller",
|
||||
Kind: inventory.BusController, Node: "anchor", Module: "mesh-controller"}, "bootstrap"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "anchor", "mesh-controller"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The store's module is registered and given its port, and NOT assigned: the state genesis
|
||||
// leaves a given-port node in before the foundation is adopted as modules (04-ISSUES/085).
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "postgres",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5432": 6852}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "anchor", "lavinmq"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "lavinmq",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5672": 5679}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var env map[string]any
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
if r["id"] == "mesh-controller.controller" {
|
||||
env, _ = r["env"].(map[string]any)
|
||||
}
|
||||
}
|
||||
if env == nil {
|
||||
t.Fatal("the control plane's process is not in its own node's declaration")
|
||||
}
|
||||
for key, want := range map[string]string{
|
||||
"MESH_STORE_INVENTORY_PORT": "6852",
|
||||
"MESH_STORE_IDENTITY_PORT": "6852",
|
||||
"MESH_STORE_LICENCES_PORT": "6852",
|
||||
"MESH_BROKER_AMQP_PORT": "5679",
|
||||
// Neither given nor assigned by the mesh: the manifest's own number is NOT the answer,
|
||||
// because the sealed value beside it carries the port genesis wrote (finding F3).
|
||||
"MESH_BROKER_ADDRESS_PORT": "",
|
||||
"MESH_BROKER_MANAGEMENT_PORT": "",
|
||||
} {
|
||||
if env[key] != want {
|
||||
t.Errorf("the control plane is told %s=%v; the node says %q", key, env[key], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// withSeatPorts is the control plane's manifest with the seat placeholders in its environment —
|
||||
// added here until module.json carries them (the manifest lands one commit after the code that
|
||||
@@ -274,79 +133,7 @@ func withSeatPorts(m catalogue.Manifest) catalogue.Manifest {
|
||||
// aLoneNode is one capable machine with nothing placed on any network — the control-node during
|
||||
// genesis, before the "network" step, which is after the store, the broker, the vault and the
|
||||
// catalogue have each been built and pushed (finding F2).
|
||||
func aLoneNode(t *testing.T) *stores {
|
||||
t.Helper()
|
||||
inventory.ForTest(t)
|
||||
licences.ForTest(t)
|
||||
open, err := openStores(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(open.Close)
|
||||
for _, m := range provided {
|
||||
if err := open.inventory.Provide(t.Context(), m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
record, err := open.inventory.AddNode(t.Context(), "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reported, _ := json.Marshal(map[string]any{"capabilities": []map[string]any{
|
||||
{"name": "container-runtime", "present": true}}})
|
||||
var profile map[string]any
|
||||
_ = json.Unmarshal(reported, &profile)
|
||||
if err := open.inventory.RecordProfile(t.Context(), record.ID, profile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSealingKey(t.Context(), record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return open
|
||||
}
|
||||
|
||||
// **Before the network exists, the store's own node reaches it by loopback** — never refused,
|
||||
// never handed the scheme: a genesis pushes the store, the broker, the vault and the catalogue to
|
||||
// a node on no network, and builds the catalogue on a base it must be able to pull.
|
||||
func TestOnANodeWithNoNetworkTheStoreIsReachedByLoopback(t *testing.T) {
|
||||
open := aLoneNode(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aStore())
|
||||
register(t, open, catalogue.Manifest{Module: "builder", Version: "1",
|
||||
Requires: []string{catalogue.ArtifactStoreProvision},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-builder",
|
||||
"image": "registry.example/mesh-builder@" + aDigest}}})
|
||||
if err := open.inventory.RecordBuild(ctx, inventory.Build{
|
||||
ID: "b1", Repository: "r", Module: "postgres", Commit: "abc",
|
||||
Made: []inventory.Artifact{{Name: "runtime", Kind: "image",
|
||||
Reference: catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "postgres", Version: "1",
|
||||
Resources: []map[string]any{{"id": "runtime", "type": "container", "name": "mesh-postgres",
|
||||
"image": catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}}})
|
||||
for _, module := range []string{"distribution", "builder", "postgres"} {
|
||||
if _, err := assign(ctx, open, "anchor", module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "distribution",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5100}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var image any
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
if r["id"] == "postgres.runtime" {
|
||||
image = r["image"]
|
||||
}
|
||||
}
|
||||
if image != "127.0.0.1:5100/postgres/runtime@"+aDigest {
|
||||
t.Fatalf("on the store's own node, off any network, the image is fetched as %v", image)
|
||||
}
|
||||
held := heldBy(ctx)
|
||||
if got := held["postgres/runtime"]; got != "127.0.0.1:5100/postgres/runtime@"+aDigest {
|
||||
t.Fatalf("a builder beside the store is handed the base %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
snapshot "github.com/novox/mesh-controller/internal/facts"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-host/rootsearch"
|
||||
)
|
||||
|
||||
func TestTheSelfCheckSaysAnAgentAccountThatCanBecomeRoot(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.NodeByName(ctx, n); err != nil {
|
||||
if _, err := inv.AddNode(ctx, n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.SetAccount(ctx, n, "ops", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.SetAgentAccount(ctx, "anchor", "agent", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &doctor{open: open}
|
||||
found, err := probeAgentAccounts(ctx, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found = onlyMachine(found, "anchor")
|
||||
if len(found) != 1 || found[0].Machine != "anchor" || found[0].Severity != conditions.Urgent ||
|
||||
!strings.Contains(found[0].Said, "not judged") {
|
||||
t.Fatalf("a named agent account with no verdict: %+v", found)
|
||||
}
|
||||
w := plainWordings[kindAgentCanBecomeRoot](conditions.Observation{Kind: kindAgentCanBecomeRoot, Machine: "anchor"})
|
||||
if w.Headline == "" || w.Needs == "" || w.Resolved == "" {
|
||||
t.Errorf("the condition has no plain words: %+v", w)
|
||||
}
|
||||
|
||||
healthy := inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account",
|
||||
Kind: link.KindAccount, Target: "agent", State: link.StateHealthy, Root: link.RootNever, Account: "agent"}
|
||||
if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.RootContract,
|
||||
SaidAt: time.Now(), HeardAt: time.Now(), Resources: []inventory.ResourceHealth{healthy}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if found, err = probeAgentAccounts(ctx, d); err != nil || len(onlyMachine(found, "anchor")) != 0 {
|
||||
t.Fatalf("a judged agent account still fails: %+v %v", found, err)
|
||||
}
|
||||
if named, confined, why, err := agentConfined(ctx, inv, "anchor", time.Now()); err != nil || !named || !confined {
|
||||
t.Fatalf("agentConfined on anchor: %v %v %q %v", named, confined, why, err)
|
||||
}
|
||||
if named, _, why, err := agentConfined(ctx, inv, "laptop", time.Now()); err != nil || named ||
|
||||
!strings.Contains(why, "operator account") {
|
||||
t.Fatalf("agentConfined on a machine naming none: %v %q %v", named, why, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheFactsCarryTheAgentAccountAsAPseudonym(t *testing.T) {
|
||||
open, _ := aMeshWithSecrets(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.SetAccount(ctx, "anchor", "keeper", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetAgentAccount(ctx, "anchor", "warden", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := gatherFacts(ctx, open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, _ := f.Encode()
|
||||
if strings.Contains(string(body), "warden") {
|
||||
t.Error("the agent account's name is in the snapshot")
|
||||
}
|
||||
m, ok := f.Machine(snapshot.Pseudonym("machine", "anchor"))
|
||||
if !ok || m.AgentAccount != snapshot.Pseudonym("account", "warden") || m.Account == m.AgentAccount {
|
||||
t.Fatalf("the anchor's agent account reads as %q (operator %q)", m.AgentAccount, m.Account)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASearchStillRunningAfterARestartIsNotUrgent(t *testing.T) {
|
||||
if searchQuietFor != rootsearch.Quiet {
|
||||
t.Fatalf("the quiet is %s and the node-engine's %s: they are one value", searchQuietFor, rootsearch.Quiet)
|
||||
}
|
||||
// A daily search that finishes within the quiet never leaves the account unjudged between two of them.
|
||||
if rootsearch.FreshFor < rootsearch.Every+rootsearch.Quiet {
|
||||
t.Fatalf("a complete search judges for %s, less than a day's search (%s) and the quiet (%s)",
|
||||
rootsearch.FreshFor, rootsearch.Every, rootsearch.Quiet)
|
||||
}
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if _, err := inv.NodeByName(ctx, "anchor"); err != nil {
|
||||
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.SetAgentAccount(ctx, "anchor", "agent", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &doctor{open: open}
|
||||
say := func(state, reason string) []conditions.Observation {
|
||||
t.Helper()
|
||||
// The engine's since is always now: it was just restarted.
|
||||
v := inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account",
|
||||
Kind: link.KindAccount, Target: "agent", State: state, Reason: reason, Root: link.RootNever,
|
||||
Account: "agent", Since: time.Now()}
|
||||
if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.RootContract,
|
||||
SaidAt: time.Now(), HeardAt: time.Now(), Resources: []inventory.ResourceHealth{v}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found, err := probeAgentAccounts(ctx, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return onlyMachine(found, "anchor")
|
||||
}
|
||||
running := link.ReasonRootPending + ": the search for setuid programs, started at 19:21, has not finished yet"
|
||||
healthy := func() {
|
||||
t.Helper()
|
||||
if found := say(link.StateHealthy, ""); len(found) != 0 {
|
||||
t.Fatalf("a healthy verdict raised: %+v", found)
|
||||
}
|
||||
}
|
||||
if found := say(link.StateUnknown, running); len(found) != 0 {
|
||||
t.Fatalf("a search first seen now was raised: %+v", found)
|
||||
}
|
||||
if _, confined, why, _ := agentConfined(ctx, inv, "anchor", time.Now()); confined || !strings.Contains(why, "not judged") {
|
||||
t.Fatalf("an account whose search runs was read as confined: %q", why)
|
||||
}
|
||||
// The engine restarted again and again, each statement's own since fresh: the controller's clock runs on.
|
||||
if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
healthy() // a complete verdict forgets when the waiting began …
|
||||
if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil {
|
||||
t.Fatal(err) // … and this restart loop began past the bound
|
||||
}
|
||||
if found := say(link.StateUnknown, running); len(found) != 1 || found[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("a search pending past the bound, by the controller's clock, was not urgent: %+v", found)
|
||||
}
|
||||
healthy()
|
||||
incomplete := rootsearch.ReasonIncomplete + ": the search for setuid programs did not finish (last tried at " +
|
||||
"19:23: timeout); it is tried again later"
|
||||
if found := say(link.StateUnknown, incomplete); len(found) != 1 || found[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("a search that did not finish was not urgent: %+v", found)
|
||||
}
|
||||
if found := say(link.StateUnhealthy, link.ReasonRoot+": in the group docker; "+running); len(found) != 1 ||
|
||||
found[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("a way to root found while the search runs was not urgent: %+v", found)
|
||||
}
|
||||
}
|
||||
@@ -1,14 +1,12 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"github.com/novox/mesh-host/rootsearch"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
snapshot "github.com/novox/mesh-controller/internal/facts"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
@@ -65,55 +63,6 @@ func TestAnAgentAccountIsConfinedOnlyOnAHealthyVerdictJudgedForRoot(t *testing.T
|
||||
|
||||
// DA raises an urgent condition, with plain words, on a machine whose agent account is not judged unable to
|
||||
// become root; a machine that names none is not its to judge.
|
||||
func TestTheSelfCheckSaysAnAgentAccountThatCanBecomeRoot(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.NodeByName(ctx, n); err != nil {
|
||||
if _, err := inv.AddNode(ctx, n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.SetAccount(ctx, n, "ops", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.SetAgentAccount(ctx, "anchor", "agent", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &doctor{open: open}
|
||||
found, err := probeAgentAccounts(ctx, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found = onlyMachine(found, "anchor")
|
||||
if len(found) != 1 || found[0].Machine != "anchor" || found[0].Severity != conditions.Urgent ||
|
||||
!strings.Contains(found[0].Said, "not judged") {
|
||||
t.Fatalf("a named agent account with no verdict: %+v", found)
|
||||
}
|
||||
w := plainWordings[kindAgentCanBecomeRoot](conditions.Observation{Kind: kindAgentCanBecomeRoot, Machine: "anchor"})
|
||||
if w.Headline == "" || w.Needs == "" || w.Resolved == "" {
|
||||
t.Errorf("the condition has no plain words: %+v", w)
|
||||
}
|
||||
|
||||
healthy := inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account",
|
||||
Kind: link.KindAccount, Target: "agent", State: link.StateHealthy, Root: link.RootNever, Account: "agent"}
|
||||
if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.RootContract,
|
||||
SaidAt: time.Now(), HeardAt: time.Now(), Resources: []inventory.ResourceHealth{healthy}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if found, err = probeAgentAccounts(ctx, d); err != nil || len(onlyMachine(found, "anchor")) != 0 {
|
||||
t.Fatalf("a judged agent account still fails: %+v %v", found, err)
|
||||
}
|
||||
if named, confined, why, err := agentConfined(ctx, inv, "anchor", time.Now()); err != nil || !named || !confined {
|
||||
t.Fatalf("agentConfined on anchor: %v %v %q %v", named, confined, why, err)
|
||||
}
|
||||
if named, _, why, err := agentConfined(ctx, inv, "laptop", time.Now()); err != nil || named ||
|
||||
!strings.Contains(why, "operator account") {
|
||||
t.Fatalf("agentConfined on a machine naming none: %v %q %v", named, why, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheAgentRootWordsArePlain(t *testing.T) {
|
||||
w := plainWordings[kindAgentCanBecomeRoot](conditions.Observation{Kind: kindAgentCanBecomeRoot, Machine: "anchor"})
|
||||
@@ -134,28 +83,6 @@ func onlyMachine(obs []conditions.Observation, machine string) []conditions.Obse
|
||||
|
||||
// The snapshot a merge check composes from carries the agent account as a pseudonym (novox/hq ADR 0266),
|
||||
// so a change is judged against machines that name one, and the name never leaves.
|
||||
func TestTheFactsCarryTheAgentAccountAsAPseudonym(t *testing.T) {
|
||||
open, _ := aMeshWithSecrets(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.SetAccount(ctx, "anchor", "keeper", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetAgentAccount(ctx, "anchor", "warden", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := gatherFacts(ctx, open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, _ := f.Encode()
|
||||
if strings.Contains(string(body), "warden") {
|
||||
t.Error("the agent account's name is in the snapshot")
|
||||
}
|
||||
m, ok := f.Machine(snapshot.Pseudonym("machine", "anchor"))
|
||||
if !ok || m.AgentAccount != snapshot.Pseudonym("account", "warden") || m.Account == m.AgentAccount {
|
||||
t.Fatalf("the anchor's agent account reads as %q (operator %q)", m.AgentAccount, m.Account)
|
||||
}
|
||||
}
|
||||
|
||||
// No verb runs a `node` command that sets something: through the generic `command` verb, `node account`,
|
||||
// `node agent-account` and every other `node` subcommand but list and show are refused, naming the terminal.
|
||||
@@ -205,75 +132,3 @@ func TestTheNodeVerbOnlyShows(t *testing.T) {
|
||||
// controller first saw it waiting — never from the engine's own "since", which a restart resets, so an engine
|
||||
// restarted in a loop does not keep it quiet. The account is still not confined, and `node show` still says not
|
||||
// judged; a search that failed, or a way to root found, is urgent at once.
|
||||
func TestASearchStillRunningAfterARestartIsNotUrgent(t *testing.T) {
|
||||
if searchQuietFor != rootsearch.Quiet {
|
||||
t.Fatalf("the quiet is %s and the node-engine's %s: they are one value", searchQuietFor, rootsearch.Quiet)
|
||||
}
|
||||
// A daily search that finishes within the quiet never leaves the account unjudged between two of them.
|
||||
if rootsearch.FreshFor < rootsearch.Every+rootsearch.Quiet {
|
||||
t.Fatalf("a complete search judges for %s, less than a day's search (%s) and the quiet (%s)",
|
||||
rootsearch.FreshFor, rootsearch.Every, rootsearch.Quiet)
|
||||
}
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if _, err := inv.NodeByName(ctx, "anchor"); err != nil {
|
||||
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.SetAgentAccount(ctx, "anchor", "agent", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &doctor{open: open}
|
||||
say := func(state, reason string) []conditions.Observation {
|
||||
t.Helper()
|
||||
// The engine's since is always now: it was just restarted.
|
||||
v := inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account",
|
||||
Kind: link.KindAccount, Target: "agent", State: state, Reason: reason, Root: link.RootNever,
|
||||
Account: "agent", Since: time.Now()}
|
||||
if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.RootContract,
|
||||
SaidAt: time.Now(), HeardAt: time.Now(), Resources: []inventory.ResourceHealth{v}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found, err := probeAgentAccounts(ctx, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return onlyMachine(found, "anchor")
|
||||
}
|
||||
running := link.ReasonRootPending + ": the search for setuid programs, started at 19:21, has not finished yet"
|
||||
healthy := func() {
|
||||
t.Helper()
|
||||
if found := say(link.StateHealthy, ""); len(found) != 0 {
|
||||
t.Fatalf("a healthy verdict raised: %+v", found)
|
||||
}
|
||||
}
|
||||
if found := say(link.StateUnknown, running); len(found) != 0 {
|
||||
t.Fatalf("a search first seen now was raised: %+v", found)
|
||||
}
|
||||
if _, confined, why, _ := agentConfined(ctx, inv, "anchor", time.Now()); confined || !strings.Contains(why, "not judged") {
|
||||
t.Fatalf("an account whose search runs was read as confined: %q", why)
|
||||
}
|
||||
// The engine restarted again and again, each statement's own since fresh: the controller's clock runs on.
|
||||
if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
healthy() // a complete verdict forgets when the waiting began …
|
||||
if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil {
|
||||
t.Fatal(err) // … and this restart loop began past the bound
|
||||
}
|
||||
if found := say(link.StateUnknown, running); len(found) != 1 || found[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("a search pending past the bound, by the controller's clock, was not urgent: %+v", found)
|
||||
}
|
||||
healthy()
|
||||
incomplete := rootsearch.ReasonIncomplete + ": the search for setuid programs did not finish (last tried at " +
|
||||
"19:23: timeout); it is tried again later"
|
||||
if found := say(link.StateUnknown, incomplete); len(found) != 1 || found[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("a search that did not finish was not urgent: %+v", found)
|
||||
}
|
||||
if found := say(link.StateUnhealthy, link.ReasonRoot+": in the group docker; "+running); len(found) != 1 ||
|
||||
found[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("a way to root found while the search runs was not urgent: %+v", found)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
func TestAModuleAssignedAndNotPushedIsAwaitingAPushNotUncomposable(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop") // the machine was pushed before; then the module is assigned
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The read the rest of the mesh asks still refuses it, with the composer's typed error: nothing
|
||||
// can be compared about a secret that does not exist (status), and nothing here string-matches.
|
||||
plan, settings, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = declarationWith(ctx, open, "laptop", plan, settings, gens, Reading)
|
||||
var notMade *catalogue.NotMadeError
|
||||
if !errors.As(err, ¬Made) || notMade.Module != "keeper" || notMade.Name != "repository" {
|
||||
t.Fatalf("a read composition did not say which secret is not made, typed: %v", err)
|
||||
}
|
||||
// Foreseen, it composes, names what the push makes, and made nothing.
|
||||
foreseen, err := declarationWith(ctx, open, "laptop", plan, settings, gens, Foreseeing)
|
||||
if err != nil || len(foreseen.foreseen) != 1 || foreseen.foreseen[0] != "keeper/repository" {
|
||||
t.Fatalf("foreseen: %v %v", foreseen.foreseen, err)
|
||||
}
|
||||
if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "keeper", "repository"); err != nil || held {
|
||||
t.Fatalf("asking ahead of the push made the secret (held %v, %v)", held, err)
|
||||
}
|
||||
|
||||
// Within the bound: nothing at all — no urgent, no warning, nobody notified.
|
||||
if got := d1(t, open); len(got) != 0 {
|
||||
t.Fatalf("a machine waiting for a push raised %+v", got)
|
||||
}
|
||||
|
||||
// Past the bound: a warning, not urgent, saying what the push will make.
|
||||
before := awaitingPushBound
|
||||
awaitingPushBound = -time.Minute
|
||||
t.Cleanup(func() { awaitingPushBound = before })
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.awaiting-push" || got[0].Severity != conditions.Warning ||
|
||||
!strings.Contains(got[0].Summary, "keeper/repository") || !strings.Contains(got[0].Summary, "push laptop") {
|
||||
t.Fatalf("a machine left un-pushed past the bound: %+v", got)
|
||||
}
|
||||
|
||||
// Pushed: the secret is made and D1 says nothing.
|
||||
pushedBy(t, open, "laptop")
|
||||
if got := d1(t, open); len(got) != 0 {
|
||||
t.Fatalf("a pushed machine still raised %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASecretThePushCannotMakeIsStillUncomposable(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aTalker())
|
||||
if _, err := assign(ctx, open, "laptop", "talker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || got[0].Severity != conditions.Urgent ||
|
||||
!strings.Contains(got[0].Summary, "module issue talker --node laptop") {
|
||||
t.Fatalf("a push that will be refused was not said urgently: %+v", got)
|
||||
}
|
||||
|
||||
// And a machine whose composition fails for anything else, with a secret waiting beside it, is
|
||||
// uncomposable for that — the stand-in hides nothing.
|
||||
register(t, open, aKeeper())
|
||||
if _, err := assign(ctx, open, "anchor", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
one, two := rivals()
|
||||
register(t, open, one)
|
||||
register(t, open, two)
|
||||
_, _ = assign(ctx, open, "anchor", "rival-one")
|
||||
_, _ = assign(ctx, open, "anchor", "rival-two")
|
||||
keys := map[string]conditions.Severity{}
|
||||
for _, o := range d1(t, open) {
|
||||
keys[o.Key()] = o.Severity
|
||||
}
|
||||
if keys["machine.anchor.uncomposable"] != conditions.Urgent {
|
||||
t.Fatalf("a real failure beside a waiting secret: %v", keys)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAGivenSecretUnderAnOldKeyIsNotForeseen(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "keeper", "repository", "given"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, err := open.inventory.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "issue it again") {
|
||||
t.Fatalf("a given secret under an old key: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMachineAwaitsAPushSinceItsOldestAssignmentSinceTheLastSend(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop")
|
||||
sent := time.Now()
|
||||
since, err := open.inventory.AwaitingSince(ctx, "laptop")
|
||||
if err != nil || since.After(sent) {
|
||||
t.Fatalf("nothing assigned since the send: waiting since %v (%v), the send was before %v", since, err, sent)
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
since, err = open.inventory.AwaitingSince(ctx, "laptop")
|
||||
if err != nil || since.Before(sent.Add(-time.Second)) {
|
||||
t.Fatalf("assigned after the send: waiting since %v (%v), not from the assignment", since, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestALastSendIsReadFromTheSendAndTheReport(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop")
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sends, err := readDeliveries(ctx, open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatal("a holder assigned and not pushed is expected to answer")
|
||||
}
|
||||
if !sends["laptop"].settled(overlay.Name, time.Now().Add(time.Hour)) {
|
||||
t.Fatal("a module the machine was sent long ago is not expected to answer")
|
||||
}
|
||||
pushedBy(t, open, "laptop")
|
||||
sends, err = readDeliveries(ctx, open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatal("pushed a moment ago and not reported, a holder is already expected")
|
||||
}
|
||||
if !sends["laptop"].settled("keeper", time.Now().Add(reportGrace+time.Minute)) {
|
||||
t.Fatal("pushed past the grace, a holder is not expected")
|
||||
}
|
||||
if _, ok := sends["laptop"].carried["keeper"]; !ok {
|
||||
t.Fatalf("the send's builds do not carry keeper: %v", sends["laptop"].carried)
|
||||
}
|
||||
pushedAndApplied(t, open, "laptop")
|
||||
if sends, err = readDeliveries(ctx, open.inventory); err != nil || !sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatalf("pushed and reported applied, a holder is not expected to answer (%v)", err)
|
||||
}
|
||||
}
|
||||
@@ -1,15 +1,12 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// Between `assign` and `push` a module's own secrets are not made yet: the push makes them. D1 composed
|
||||
@@ -81,140 +78,14 @@ func d1(t *testing.T, open *stores) []conditions.Observation {
|
||||
|
||||
// **THE WINDOW, REPRODUCED**: assigned and not pushed, a module whose own secret the push makes is
|
||||
// waiting, not uncomposable; past the bound it is a warning naming what the push makes; pushed, nothing.
|
||||
func TestAModuleAssignedAndNotPushedIsAwaitingAPushNotUncomposable(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop") // the machine was pushed before; then the module is assigned
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The read the rest of the mesh asks still refuses it, with the composer's typed error: nothing
|
||||
// can be compared about a secret that does not exist (status), and nothing here string-matches.
|
||||
plan, settings, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = declarationWith(ctx, open, "laptop", plan, settings, gens, Reading)
|
||||
var notMade *catalogue.NotMadeError
|
||||
if !errors.As(err, ¬Made) || notMade.Module != "keeper" || notMade.Name != "repository" {
|
||||
t.Fatalf("a read composition did not say which secret is not made, typed: %v", err)
|
||||
}
|
||||
// Foreseen, it composes, names what the push makes, and made nothing.
|
||||
foreseen, err := declarationWith(ctx, open, "laptop", plan, settings, gens, Foreseeing)
|
||||
if err != nil || len(foreseen.foreseen) != 1 || foreseen.foreseen[0] != "keeper/repository" {
|
||||
t.Fatalf("foreseen: %v %v", foreseen.foreseen, err)
|
||||
}
|
||||
if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "keeper", "repository"); err != nil || held {
|
||||
t.Fatalf("asking ahead of the push made the secret (held %v, %v)", held, err)
|
||||
}
|
||||
|
||||
// Within the bound: nothing at all — no urgent, no warning, nobody notified.
|
||||
if got := d1(t, open); len(got) != 0 {
|
||||
t.Fatalf("a machine waiting for a push raised %+v", got)
|
||||
}
|
||||
|
||||
// Past the bound: a warning, not urgent, saying what the push will make.
|
||||
before := awaitingPushBound
|
||||
awaitingPushBound = -time.Minute
|
||||
t.Cleanup(func() { awaitingPushBound = before })
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.awaiting-push" || got[0].Severity != conditions.Warning ||
|
||||
!strings.Contains(got[0].Summary, "keeper/repository") || !strings.Contains(got[0].Summary, "push laptop") {
|
||||
t.Fatalf("a machine left un-pushed past the bound: %+v", got)
|
||||
}
|
||||
|
||||
// Pushed: the secret is made and D1 says nothing.
|
||||
pushedBy(t, open, "laptop")
|
||||
if got := d1(t, open); len(got) != 0 {
|
||||
t.Fatalf("a pushed machine still raised %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **A REAL FAILURE IS STILL URGENT**: a secret the push would be refused on is not one it will make.
|
||||
// A bus credential nobody issued (issue 203) fails the push, so D1 says it — urgent, in the push's words.
|
||||
func TestASecretThePushCannotMakeIsStillUncomposable(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aTalker())
|
||||
if _, err := assign(ctx, open, "laptop", "talker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || got[0].Severity != conditions.Urgent ||
|
||||
!strings.Contains(got[0].Summary, "module issue talker --node laptop") {
|
||||
t.Fatalf("a push that will be refused was not said urgently: %+v", got)
|
||||
}
|
||||
|
||||
// And a machine whose composition fails for anything else, with a secret waiting beside it, is
|
||||
// uncomposable for that — the stand-in hides nothing.
|
||||
register(t, open, aKeeper())
|
||||
if _, err := assign(ctx, open, "anchor", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
one, two := rivals()
|
||||
register(t, open, one)
|
||||
register(t, open, two)
|
||||
_, _ = assign(ctx, open, "anchor", "rival-one")
|
||||
_, _ = assign(ctx, open, "anchor", "rival-two")
|
||||
keys := map[string]conditions.Severity{}
|
||||
for _, o := range d1(t, open) {
|
||||
keys[o.Key()] = o.Severity
|
||||
}
|
||||
if keys["machine.anchor.uncomposable"] != conditions.Urgent {
|
||||
t.Fatalf("a real failure beside a waiting secret: %v", keys)
|
||||
}
|
||||
}
|
||||
|
||||
// A given secret sealed to a key the machine no longer has is not the mesh's to make again: the push is
|
||||
// refused on it, so D1 is too.
|
||||
func TestAGivenSecretUnderAnOldKeyIsNotForeseen(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "keeper", "repository", "given"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, err := open.inventory.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "issue it again") {
|
||||
t.Fatalf("a given secret under an old key: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The bound is read from when the machine began waiting: the oldest assignment since its last send.
|
||||
func TestAMachineAwaitsAPushSinceItsOldestAssignmentSinceTheLastSend(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop")
|
||||
sent := time.Now()
|
||||
since, err := open.inventory.AwaitingSince(ctx, "laptop")
|
||||
if err != nil || since.After(sent) {
|
||||
t.Fatalf("nothing assigned since the send: waiting since %v (%v), the send was before %v", since, err, sent)
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
since, err = open.inventory.AwaitingSince(ctx, "laptop")
|
||||
if err != nil || since.Before(sent.Add(-time.Second)) {
|
||||
t.Fatalf("assigned after the send: waiting since %v (%v), not from the assignment", since, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **D3 AND D13 WAIT FOR THE SEND**: a holder is expected to answer on a machine once the machine was sent
|
||||
// it and had time to report — never between assign and push.
|
||||
@@ -242,40 +113,3 @@ func TestAHolderIsExpectedOnlyOnceSentAndReported(t *testing.T) {
|
||||
}
|
||||
|
||||
// And through the stores: assigned, not in the last send; pushed and reported, carried and settled.
|
||||
func TestALastSendIsReadFromTheSendAndTheReport(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop")
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sends, err := readDeliveries(ctx, open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatal("a holder assigned and not pushed is expected to answer")
|
||||
}
|
||||
if !sends["laptop"].settled(overlay.Name, time.Now().Add(time.Hour)) {
|
||||
t.Fatal("a module the machine was sent long ago is not expected to answer")
|
||||
}
|
||||
pushedBy(t, open, "laptop")
|
||||
sends, err = readDeliveries(ctx, open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatal("pushed a moment ago and not reported, a holder is already expected")
|
||||
}
|
||||
if !sends["laptop"].settled("keeper", time.Now().Add(reportGrace+time.Minute)) {
|
||||
t.Fatal("pushed past the grace, a holder is not expected")
|
||||
}
|
||||
if _, ok := sends["laptop"].carried["keeper"]; !ok {
|
||||
t.Fatalf("the send's builds do not carry keeper: %v", sends["laptop"].carried)
|
||||
}
|
||||
pushedAndApplied(t, open, "laptop")
|
||||
if sends, err = readDeliveries(ctx, open.inventory); err != nil || !sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatalf("pushed and reported applied, a holder is not expected to answer (%v)", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// aBackupSeatOnTheBus is one machine's node-backup holder as a test makes it: it counts the `now` asks it hears and
|
||||
// answers `backed-up` with the last night it is told (novox/hq ADR 0404).
|
||||
type aBackupSeatOnTheBus struct {
|
||||
mu sync.Mutex
|
||||
nows []string
|
||||
night map[string]any // module → lastNight
|
||||
}
|
||||
|
||||
func (f *aBackupSeatOnTheBus) serve(t *testing.T, conn *nats.Conn, node string) {
|
||||
t.Helper()
|
||||
reply := func(m *nats.Msg, result any) {
|
||||
body, _ := json.Marshal(map[string]any{"result": result, "error": "", "node": node})
|
||||
_ = m.Respond(body)
|
||||
}
|
||||
now, err := conn.Subscribe(link.NodeSeatToolSubject(catalogue.BackupSeat, "now", node), func(m *nats.Msg) {
|
||||
var a struct {
|
||||
Module string `json:"module"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Data, &a)
|
||||
f.mu.Lock()
|
||||
f.nows = append(f.nows, a.Module)
|
||||
f.mu.Unlock()
|
||||
reply(m, map[string]any{"started": a.Module})
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
read, err := conn.Subscribe(link.NodeSeatToolSubject(catalogue.BackupSeat, "backed-up", node), func(m *nats.Msg) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var out []map[string]any
|
||||
for module, n := range f.night {
|
||||
out = append(out, map[string]any{"module": module, "lastNight": n})
|
||||
}
|
||||
reply(m, out)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = now.Unsubscribe(); _ = read.Unsubscribe() })
|
||||
if err := conn.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (f *aBackupSeatOnTheBus) answers(module string, ok bool, words string) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.night = map[string]any{module: map[string]any{"ok": ok, "at": time.Now().UTC().Add(time.Second), "error": words}}
|
||||
}
|
||||
|
||||
func (f *aBackupSeatOnTheBus) asks() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return len(f.nows)
|
||||
}
|
||||
|
||||
// Acceptance test 3 of design 47, on the controller's side: a delivery plan that moves a module keeping data waits
|
||||
// for its backups on both machines before it asks a build; a backup that fails holds it, with the machine, the
|
||||
// module and the backup's words, and a condition says the same; the verb the delivery's release calls asks the
|
||||
// backups again, and the plan goes on. One bus and one store for the scenarios.
|
||||
func TestADeliveryThatMovesABackedUpModuleWaitsForItsBackupAndAFailedOneHoldsItAndIsSaid(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asked := asksRecorded(t)
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
before := handActConn
|
||||
handActConn = js.Conn()
|
||||
t.Cleanup(func() { handActConn = before })
|
||||
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "keeper", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}})
|
||||
notes, err := catalogue.ParseManifest([]byte(`{"module":"notes","version":"1","data":{"own":[` +
|
||||
`{"id":"files","path":"${dir:files}","class":"valuable","why":"people's notes"}]},` +
|
||||
`"resources":[{"id":"files","type":"directory"}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, notes, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
|
||||
Path: "modules/notes", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
if _, err := assign(ctx, open, node, "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, node, "notes"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
seats := map[string]*aBackupSeatOnTheBus{"anchor": {}, "laptop": {}}
|
||||
for node, s := range seats {
|
||||
s.serve(t, js.Conn(), node)
|
||||
}
|
||||
// Short waits: the bound and the look are variables, so nothing here waits for the minutes of a real backup.
|
||||
wasWithin, wasEvery := backupsWithin, backupsPollEvery
|
||||
backupsWithin, backupsPollEvery = time.Minute, 10*time.Millisecond
|
||||
t.Cleanup(func() { backupsWithin, backupsPollEvery = wasWithin, wasEvery })
|
||||
|
||||
newPlan := func(id string) {
|
||||
t.Helper()
|
||||
p := inventory.Plan{ID: id, Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1", Created: time.Now().UTC(),
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"notes"}}, Modules: map[string]*inventory.PlanModule{"notes": {}}}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
read := func(id string) inventory.Plan {
|
||||
t.Helper()
|
||||
p, err := inv.PlanByID(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// 1. Both machines are asked; while one has not answered, no build is asked.
|
||||
newPlan("plan-one")
|
||||
advancePlans(ctx, open)
|
||||
if seats["anchor"].asks() != 1 || seats["laptop"].asks() != 1 || len(*asked) != 0 {
|
||||
t.Fatalf("asks anchor=%d laptop=%d builds=%v", seats["anchor"].asks(), seats["laptop"].asks(), *asked)
|
||||
}
|
||||
seats["anchor"].answers("notes", true, "")
|
||||
advancePlans(ctx, open)
|
||||
if len(*asked) != 0 || read("plan-one").Backups.State != inventory.BackupPending {
|
||||
t.Fatalf("a tier was asked while a backup was pending: %v %+v", *asked, read("plan-one").Backups)
|
||||
}
|
||||
seats["laptop"].answers("notes", true, "")
|
||||
advancePlans(ctx, open)
|
||||
if len(*asked) != 1 || read("plan-one").Backups.State != inventory.BackupDone {
|
||||
t.Fatalf("the tier was not asked once every backup was done: %v %+v", *asked, read("plan-one").Backups)
|
||||
}
|
||||
finish := read("plan-one")
|
||||
finish.State = inventory.PlanDone
|
||||
if err := inv.SavePlan(ctx, &finish); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// 2. A backup that fails holds the plan: not failed, nothing asked, the words kept, a condition raised.
|
||||
seats["anchor"].night, seats["laptop"].night = nil, nil
|
||||
newPlan("plan-two")
|
||||
advancePlans(ctx, open)
|
||||
seats["anchor"].answers("notes", true, "")
|
||||
seats["laptop"].answers("notes", false, "no space left on device")
|
||||
advancePlans(ctx, open)
|
||||
held := read("plan-two")
|
||||
if held.State != inventory.PlanBuilding || held.Backups.State != inventory.BackupFailed || len(*asked) != 1 {
|
||||
t.Fatalf("the plan was not held: %s %+v builds=%v", held.State, held.Backups, *asked)
|
||||
}
|
||||
var failed *inventory.PlanBackup
|
||||
for i := range held.Backups.Asked {
|
||||
if held.Backups.Asked[i].State == inventory.BackupFailed {
|
||||
failed = &held.Backups.Asked[i]
|
||||
}
|
||||
}
|
||||
if failed == nil || failed.Machine != "laptop" || failed.Module != "notes" || failed.Words != "no space left on device" ||
|
||||
strings.Contains(held.Note, "tried again") {
|
||||
t.Fatalf("the failed backup: %+v note=%q", failed, held.Note)
|
||||
}
|
||||
advancePlans(ctx, open) // not retried on the next look
|
||||
if again := read("plan-two"); again.Backups.State != inventory.BackupFailed || len(*asked) != 1 ||
|
||||
seats["laptop"].asks() != 2 {
|
||||
t.Fatalf("a held plan was asked again by itself: %+v builds=%v asks=%d", again.Backups, *asked, seats["laptop"].asks())
|
||||
}
|
||||
said, err := probeBackupHeld(ctx, &doctor{open: open})
|
||||
if err != nil || len(said) != 1 || said[0].Kind != kindBackupHeld || !strings.Contains(said[0].Said, "no space left on device") {
|
||||
t.Fatalf("the condition: %+v %v", said, err)
|
||||
}
|
||||
|
||||
// 3. The verb the release calls: the failed backup is asked again, and only it, and the plan goes on.
|
||||
seats["laptop"].answers("notes", false, "no space left on device")
|
||||
if err := deliveryCommand(ctx, []string{"backups-again", "plan-two", "--why", "a person released it", "--by",
|
||||
"mesh-delivery for jochen"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := deliveryCommand(ctx, []string{"backups-again", "plan-two", "--why", "again"}); err == nil {
|
||||
t.Fatal("a plan with no failed backup was asked again")
|
||||
}
|
||||
seats["laptop"].night = nil
|
||||
advancePlans(ctx, open)
|
||||
if seats["laptop"].asks() != 3 || seats["anchor"].asks() != 2 {
|
||||
t.Fatalf("asked again: laptop=%d anchor=%d", seats["laptop"].asks(), seats["anchor"].asks())
|
||||
}
|
||||
seats["laptop"].answers("notes", true, "")
|
||||
advancePlans(ctx, open)
|
||||
if got := read("plan-two"); got.Backups.State != inventory.BackupDone || len(*asked) != 2 {
|
||||
t.Fatalf("the plan did not go on: %+v builds=%v", got.Backups, *asked)
|
||||
}
|
||||
if said, _ := probeBackupHeld(ctx, &doctor{open: open}); len(said) != 0 {
|
||||
t.Fatalf("the condition stays: %+v", said)
|
||||
}
|
||||
}
|
||||
+315
-146
@@ -5,7 +5,6 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
@@ -27,9 +26,10 @@ import (
|
||||
// commit in it, with at most one commit per repository, the latest of its branch, which contains the earlier
|
||||
// ones. The walk names every merge it answers.
|
||||
//
|
||||
// - One walk at a time. Merges that come while a walk runs join the next batch. A walk that started is never
|
||||
// superseded; one still waiting for its delivery's word is folded into the next batch's walk, which
|
||||
// answers every merge it answered — the only supersession left.
|
||||
// - One walk at a time beside a related one (ADR 0407 decision 4, issue 973). Merges that come while a walk
|
||||
// runs join the next batch; a closed batch that shares nothing with an open walk is cut beside it. A walk
|
||||
// that started is never superseded; one still waiting for its delivery's word is folded into the walk of a
|
||||
// batch related to it, which answers every merge it answered — the only supersession left.
|
||||
// - A merge heard after a later merge of its branch was put in a walk is answered by that walk: a batch
|
||||
// still assembling, an open walk that builds everything it moves, or a walk done that did, which then
|
||||
// names it at once. Otherwise it joins the next batch.
|
||||
@@ -39,8 +39,9 @@ import (
|
||||
//
|
||||
// **A merge waits only for a batch it shares something with** (novox/hq ADR 0304 decision 1, ADR 0306 decisions
|
||||
// 2 and 3, issue 525): it joins a batch it is related to, and one that is related to none and whose delivery
|
||||
// group has no member still to merge is a batch whose window is closed at once (batchToJoin). Walks stay one at
|
||||
// a time: such a batch is cut when no walk is open, and queued behind the one that is otherwise.
|
||||
// group has no member still to merge is a batch whose window is closed at once (batchToJoin). Such a batch is cut
|
||||
// at once, beside any open walk it shares nothing with, and queued behind the first one it is related to
|
||||
// (cutUnrelated).
|
||||
//
|
||||
// A merge on the controller's own path (a module whose walk waits for nobody's word) never shares a batch with
|
||||
// one that waits for mesh-delivery's: each kind has a batch of its own, an own-path batch is cut first and folds
|
||||
@@ -61,16 +62,17 @@ var (
|
||||
// mergeWindowFileVar names the file the controller's window settings are written to.
|
||||
const mergeWindowFileVar = "MESH_MERGE_WINDOW_FILE"
|
||||
|
||||
// batchCutEvery is how often the controller looks whether a batch's window closed.
|
||||
const batchCutEvery = 5 * time.Second
|
||||
|
||||
// batchLateAfter is how long past its maximum a batch may still be assembling, with no walk open, before
|
||||
// the controller is said to have failed to cut it (ADR 0276 decision 9).
|
||||
const batchLateAfter = time.Minute
|
||||
|
||||
// edgeGroupOrder is a delivery group's order made an edge of a walk's tiers (ADR 0276 decision 5): the
|
||||
// modules of the member after depend on the modules of the member before, and are gated on them.
|
||||
const edgeGroupOrder = "group-order"
|
||||
// mergeGrace is how long before a batch's first merge a merge may have been made and still be told apart from one
|
||||
// made before the controller began keeping merges (owedLate).
|
||||
const mergeGrace = 10 * time.Minute
|
||||
|
||||
// edgeChangeOrder is the order the rules of a plan draw among its modules (novox/hq ADR 0419 clauses 3 and 4, delivery_edges.go)
|
||||
// made an edge of its tiers: the module after depends on the module before, and is gated on it.
|
||||
const edgeChangeOrder = "change-order"
|
||||
|
||||
// mergeWindowOf is the window and its maximum, from the controller's settings; a value unreadable is its
|
||||
// default, and said.
|
||||
@@ -125,8 +127,7 @@ func durationSetting(v any) (time.Duration, bool) {
|
||||
// hearMerge puts a merge the forge announced into the batch that answers it, at the moment it was heard,
|
||||
// and cuts what is due.
|
||||
func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.Time) error {
|
||||
// One merge acted on at a time, whoever hands it over: the bus, or the catch-up that reads back
|
||||
// what the bus did not hand over (novox/hq issue 266).
|
||||
// One merge acted on at a time, whoever hands it over (novox/hq issue 266).
|
||||
actingOnMerges.Lock()
|
||||
defer actingOnMerges.Unlock()
|
||||
|
||||
@@ -175,12 +176,12 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
|
||||
repository, m.Base, m.Commit, already)
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("%s merged into %s (%.8s); nothing the mesh holds reads it\n", repository, m.Base, m.Commit)
|
||||
fmt.Print(nothingReadsIt(repository, m))
|
||||
return nil
|
||||
}
|
||||
// **Its delivery group's word, before the plans are held** (ADR 0306 decision 2): a slow answer holds up no walk.
|
||||
own := ownPath(touches)
|
||||
waits, err := groupWait(ctx, inv, m, deliverySeatHeld(entries), own)
|
||||
waits, holding, err := groupWaitHolding(ctx, inv, m, deliverySeatHeld(entries))
|
||||
if err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
@@ -236,6 +237,7 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
|
||||
if added, err := inv.AddMerge(ctx, heard); err != nil || !added {
|
||||
return notNow(err)
|
||||
}
|
||||
holdFor(&batch, m.Head, waits, holding, own, now)
|
||||
if err := keepBatch(ctx, inv, &batch, now, ""); err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
@@ -244,9 +246,46 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
|
||||
return cutBatchesHeld(ctx, f.open, now)
|
||||
}
|
||||
|
||||
// holdFor keeps in a batch that a merge of a group joined whether that group still has a member to merge (ADR 0418
|
||||
// clause 3): held while it has, the window running again from the moment the last hold is cleared. A group with a
|
||||
// merge of the controller's own path makes its batch an own-path one: such a plan waits for nobody's word, and is
|
||||
// started by the controller when the group's batch closes (clause 6).
|
||||
func holdFor(batch *inventory.Plan, head, why string, holding, own bool, now time.Time) {
|
||||
if batch.Delivery == nil {
|
||||
batch.Delivery = &inventory.PlanDelivery{}
|
||||
}
|
||||
if batch.Delivery.Batch == nil {
|
||||
batch.Delivery.Batch = &inventory.PlanBatch{Own: own}
|
||||
}
|
||||
w := batch.Delivery.Batch
|
||||
w.Own = w.Own || own
|
||||
if head == "" {
|
||||
return
|
||||
}
|
||||
if holding {
|
||||
if w.Holds == nil {
|
||||
w.Holds = map[string]string{}
|
||||
}
|
||||
w.Holds[head] = why
|
||||
return
|
||||
}
|
||||
if _, held := w.Holds[head]; held {
|
||||
holds := map[string]string{}
|
||||
for h, y := range w.Holds {
|
||||
if h != head {
|
||||
holds[h] = y
|
||||
}
|
||||
}
|
||||
w.Holds = holds
|
||||
if len(holds) == 0 {
|
||||
w.Holds, w.HeldAt = nil, now
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// owedLate says a merge that reads as history is still the record's to answer (review of this change): a cut
|
||||
// marks every module it moved seen at that moment, so a merge of that branch the bus lost and the catch-up
|
||||
// hands over later — made before the cut, heard after it — reads as history and was dropped. One that touches
|
||||
// marks every module it moved seen at that moment, so a merge of that branch
|
||||
// handed over late — made before the cut, heard after it — reads as history and was dropped. One that touches
|
||||
// a module the mesh holds, of a branch whose merges the controller keeps, made since it began keeping them,
|
||||
// is kept; one made before it began is old news, as before.
|
||||
func owedLate(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, merged time.Time,
|
||||
@@ -340,14 +379,23 @@ func ownPath(touches []inventory.Entry) bool {
|
||||
// names why): a new batch, which waits for the window, beside the ones it shares nothing with; its words say
|
||||
// why.
|
||||
// - none, and no member of its group is still to merge: a new batch whose window is closed at once. Cut at
|
||||
// once means only that: walks stay one at a time (ADR 0276 decision 4), so with a walk open the batch is
|
||||
// queued behind it as any closed batch is.
|
||||
// once means only that: with a walk open that it is related to the batch is queued behind it as any closed
|
||||
// batch is (ADR 0276 decision 4, ADR 0407 decision 4); beside one it shares nothing with it is cut.
|
||||
func batchToJoin(ctx context.Context, inv *inventory.Inventory, heard inventory.BatchedMerge, entries []inventory.Entry,
|
||||
read map[string][]inventory.ReadRepository, now time.Time, own bool, waits string) (inventory.Plan, string, error) {
|
||||
edges, err := inv.Dependencies(ctx)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, "", err
|
||||
}
|
||||
// **A group is one batch, whatever kinds its members are** (novox/hq ADR 0418 clause 6): a merge joins the batch
|
||||
// of the other kind that holds a merge from its head.
|
||||
if group, ok, err := batchOfGroup(ctx, inv, announcedOf(heard).Head, heard.Repository); err != nil {
|
||||
return inventory.Plan{}, "", err
|
||||
} else if ok && group.OwnPath() != own && group.Delivery != nil && group.Delivery.Batch != nil {
|
||||
group.Delivery.Batch.CutAtOnce = false
|
||||
return group, "its group " + announcedOf(heard).Head + " has a merge in " + group.ID + ", a batch of the other kind: " +
|
||||
"a group is one batch", nil
|
||||
}
|
||||
joins, whys, _, err := relatedBatches(ctx, inv, []inventory.BatchedMerge{heard}, own, "", entries, read, edges)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, "", err
|
||||
@@ -406,7 +454,9 @@ func relatedBatches(ctx context.Context, inv *inventory.Inventory, merges []inve
|
||||
}
|
||||
carry := newFootprint()
|
||||
if !own {
|
||||
// An own-path batch folds no waiting walk (cutBatchesHeld); a catalogue batch may fold every one.
|
||||
// An own-path batch folds no waiting walk (cutBatchesHeld); a catalogue batch folds those it is related to
|
||||
// (cutUnrelated). Every one is counted here, which only makes a merge join a batch more readily: it waits
|
||||
// for the window, never walks beside what it shares something with (issue 973).
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
@@ -544,6 +594,11 @@ func newBatch(ctx context.Context, inv *inventory.Inventory, first inventory.Bat
|
||||
// keepBatch writes a batch as its merges and the clock say it now: the commits it holds, the merges they
|
||||
// answer, its window, and assembling or queued behind a walk. Saved only when that changed.
|
||||
func keepBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan, now time.Time, behind string) error {
|
||||
return keepBatchBecause(ctx, inv, b, now, behind, "")
|
||||
}
|
||||
|
||||
// keepBatchBecause is keepBatch for a batch queued behind a walk it is related to, which its words say (issue 973).
|
||||
func keepBatchBecause(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan, now time.Time, behind, why string) error {
|
||||
merges, err := inv.MergesOf(ctx, b.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -564,29 +619,42 @@ func keepBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan,
|
||||
if b.Delivery == nil {
|
||||
b.Delivery = &inventory.PlanDelivery{}
|
||||
}
|
||||
closes, latest := windowOf(merges, b.Created, window, atMost)
|
||||
anchor := b.Created
|
||||
if was := b.Delivery.Batch; was != nil && was.HeldAt.After(anchor) {
|
||||
anchor = was.HeldAt // the maximum counts from the last hold's end (ADR 0418 clause 3)
|
||||
}
|
||||
closes, latest := windowOf(merges, anchor, window, atMost)
|
||||
b.Delivery.Merges = named
|
||||
var closed, cutAtOnce bool
|
||||
var withheld []string
|
||||
var waits string
|
||||
var holds map[string]string
|
||||
var heldAt time.Time
|
||||
if was := b.Delivery.Batch; was != nil {
|
||||
holds, heldAt = was.Holds, was.HeldAt
|
||||
closed, withheld = was.Closed, was.Withheld // a stop's, which no look takes back (novox/hq ADR 0299)
|
||||
cutAtOnce = was.CutAtOnce // the merge's, until another joins it (ADR 0304 decision 1)
|
||||
waits = was.Waits // its group's, as said when it was made (ADR 0306 decision 2)
|
||||
}
|
||||
b.Delivery.Batch = &inventory.PlanBatch{ClosesAt: closes, AtMost: latest, Behind: behind, Own: b.OwnPath(),
|
||||
Closed: closed, Withheld: withheld, CutAtOnce: cutAtOnce, Waits: waits}
|
||||
b.Delivery.Batch = &inventory.PlanBatch{ClosesAt: closes, AtMost: latest, Behind: behind, BehindWhy: why, Own: b.OwnPath(),
|
||||
Closed: closed, Withheld: withheld, CutAtOnce: cutAtOnce, Waits: waits, Holds: holds, HeldAt: heldAt}
|
||||
b.State = inventory.PlanAssembling
|
||||
if behind != "" && windowClosed(*b, now) {
|
||||
b.State = inventory.PlanQueued
|
||||
} else {
|
||||
b.Delivery.Batch.Behind = ""
|
||||
b.Delivery.Batch.Behind, b.Delivery.Batch.BehindWhy = "", ""
|
||||
}
|
||||
b.Note = batchWords(*b, now)
|
||||
if unNoted(*b) == before {
|
||||
return nil
|
||||
}
|
||||
return inv.SavePlan(ctx, b)
|
||||
if err := inv.SavePlan(ctx, b); err != nil {
|
||||
return err
|
||||
}
|
||||
// The window's clock follows the record, only when the record moved: a merge made the quiet key again, a
|
||||
// batch that opened made its most (ADR 0407 decision 1).
|
||||
armWindow(ctx, *b, now)
|
||||
return nil
|
||||
}
|
||||
|
||||
// windowOf is when a batch's window closes, unless another merge comes, and when at the latest: the last
|
||||
@@ -605,13 +673,20 @@ func windowOf(merges []inventory.BatchedMerge, created time.Time, window, atMost
|
||||
}
|
||||
|
||||
// windowClosed says a batch's window is closed: closed by a stop, closed at once for a merge that shares
|
||||
// nothing (ADR 0304 decision 1), no merge for the window's length, or its maximum reached.
|
||||
// nothing (ADR 0304 decision 1), no merge for the window's length, or its maximum reached. **A batch holding a group
|
||||
// with a member still to merge is closed by none of the last three** (novox/hq ADR 0418 clause 3).
|
||||
func windowClosed(b inventory.Plan, now time.Time) bool {
|
||||
if b.Delivery == nil || b.Delivery.Batch == nil {
|
||||
return true
|
||||
}
|
||||
w := b.Delivery.Batch
|
||||
return w.Closed || w.CutAtOnce || !now.Before(w.ClosesAt) || !now.Before(w.AtMost)
|
||||
if w.Closed {
|
||||
return true
|
||||
}
|
||||
if len(w.Holds) > 0 {
|
||||
return false // a group's last member is still to merge: neither the window nor its maximum closes the batch
|
||||
}
|
||||
return w.CutAtOnce || !now.Before(w.ClosesAt) || !now.Before(w.AtMost)
|
||||
}
|
||||
|
||||
// carriedOf is a set of merges as a walk carries them: the latest merge of each repository's branch, and
|
||||
@@ -739,31 +814,10 @@ func nameMerges(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan
|
||||
return inv.SavePlan(ctx, p)
|
||||
}
|
||||
|
||||
// batchCutter cuts what is due on a timer, until the context ends: a window closes with no merge to say so.
|
||||
func batchCutter(ctx context.Context, open *stores) {
|
||||
tick := time.NewTicker(batchCutEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
release, err := open.inventory.HoldPlans(ctx, false)
|
||||
if err != nil {
|
||||
continue // whoever holds the plans moves them, and cuts at the end
|
||||
}
|
||||
if err := cutBatchesHeld(ctx, open, time.Now().UTC()); err != nil {
|
||||
fmt.Printf("batches: %v\n", err)
|
||||
}
|
||||
release()
|
||||
}
|
||||
}
|
||||
|
||||
// cutBatchesHeld takes the step that is due, for a caller holding the plans: after a failed walk, the search
|
||||
// for the merge that brought the failure; with no walk open, the next merge of that search walked alone, or a
|
||||
// closed batch cut into one walk with the waiting walks folded in; with a walk open, a closed batch said
|
||||
// queued behind it.
|
||||
// for the merge that brought the failure; with no walk open, the next merge of that search walked alone; else
|
||||
// each closed batch cut into a walk with the waiting walks it is related to folded in, unless it is related to a
|
||||
// walk open, behind which it is said queued (cutUnrelated). While the search runs everything waits.
|
||||
func cutBatchesHeld(ctx context.Context, open *stores, now time.Time) error {
|
||||
inv := open.inventory
|
||||
if err := searchAfterFailures(ctx, inv); err != nil {
|
||||
@@ -780,8 +834,7 @@ func cutBatchesHeld(ctx context.Context, open *stores, now time.Time) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var started *inventory.Plan
|
||||
var waiting []inventory.Plan
|
||||
var started, waiting []inventory.Plan
|
||||
for i := range plans {
|
||||
p := plans[i]
|
||||
if p.Release != nil {
|
||||
@@ -789,14 +842,19 @@ func cutBatchesHeld(ctx context.Context, open *stores, now time.Time) error {
|
||||
}
|
||||
if p.Waiting() {
|
||||
waiting = append(waiting, p)
|
||||
} else if started == nil {
|
||||
started = &plans[i]
|
||||
} else {
|
||||
started = append(started, p)
|
||||
}
|
||||
}
|
||||
batches, err := inv.Batches(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// **A hold is looked at again at every cut** (ADR 0418 clause 3): a group whose last member merged or closed
|
||||
// since is released, and its window runs again from now.
|
||||
if batches, err = releaseHolds(ctx, inv, batches, now); err != nil {
|
||||
return err
|
||||
}
|
||||
keepAll := func(behind string) error {
|
||||
for i := range batches {
|
||||
if err := keepBatch(ctx, inv, &batches[i], now, behind); err != nil {
|
||||
@@ -805,46 +863,151 @@ func cutBatchesHeld(ctx context.Context, open *stores, now time.Time) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if started != nil {
|
||||
// **One walk at a time** (ADR 0276 decision 4): every batch waits behind it, whatever its window says.
|
||||
return keepAll(started.ID)
|
||||
}
|
||||
if len(alone) > 0 {
|
||||
// A failed walk's search walks its merges alone, one at a time: everything waits for it (kept as it was,
|
||||
// issue 973).
|
||||
if len(started) > 0 {
|
||||
return keepAll(started[0].ID)
|
||||
}
|
||||
if len(waiting) > 0 {
|
||||
// The waiting walk is the one open; the search walks after it, and the batches wait behind both.
|
||||
return keepAll(waiting[0].ID)
|
||||
}
|
||||
return walkAlone(ctx, open, alone[0], now)
|
||||
}
|
||||
if err := keepAll(""); err != nil {
|
||||
return cutUnrelated(ctx, open, batches, started, waiting, now)
|
||||
}
|
||||
|
||||
// releaseHolds asks mesh-delivery again about each group a batch holds, and clears the holds of those with no member
|
||||
// left to merge. A group mesh-delivery cannot be asked about stays held: the safe error is to wait. It answers the
|
||||
// batches as kept.
|
||||
func releaseHolds(ctx context.Context, inv *inventory.Inventory, batches []inventory.Plan, now time.Time) ([]inventory.Plan, error) {
|
||||
var entries []inventory.Entry
|
||||
for i := range batches {
|
||||
b := &batches[i]
|
||||
if b.Delivery == nil || b.Delivery.Batch == nil || len(b.Delivery.Batch.Holds) == 0 {
|
||||
continue
|
||||
}
|
||||
if entries == nil {
|
||||
var err error
|
||||
if entries, err = inv.Catalogued(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
changed := false
|
||||
for head := range b.Delivery.Batch.Holds {
|
||||
open, isGroup, _, err := openMembersOf(ctx, inv, deliverySeatHeld(entries), head, "")
|
||||
if err != nil || (isGroup && len(open) > 0) {
|
||||
continue
|
||||
}
|
||||
holdFor(b, head, "", false, false, now)
|
||||
changed = true
|
||||
fmt.Printf("%s: group %s has no member left to merge; its window runs again from now\n", b.ID, head)
|
||||
}
|
||||
if changed {
|
||||
if err := keepBatch(ctx, inv, b, now, ""); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return batches, nil
|
||||
}
|
||||
|
||||
// cutUnrelated cuts the closed batches that share nothing with an open walk, each into a walk beside the open
|
||||
// ones, and says the others queued behind the first walk they are related to (novox/hq ADR 0407 decision 4,
|
||||
// issue 973). A batch cut in this look counts as open for the batches after it.
|
||||
//
|
||||
// - An own-path batch goes first and folds no waiting walk; a catalogue batch folds the waiting walks it is
|
||||
// related to, and those it shares nothing with keep waiting.
|
||||
// - **The controller's own path is related to everything** (decided for issue 973): a walk or batch that
|
||||
// replaces the controller, the node-engine, the tool runner, the bus or mesh-delivery never runs beside
|
||||
// another, and nothing runs beside it.
|
||||
// - A batch whose window is still open is assembling; one related to an open walk is queued behind it, and
|
||||
// ADR 0276 decision 4 stands for it: nothing supersedes a walk that started.
|
||||
func cutUnrelated(ctx context.Context, open *stores, batches, started, waiting []inventory.Plan, now time.Time) error {
|
||||
inv := open.inventory
|
||||
sort.SliceStable(batches, func(i, j int) bool { return batches[i].OwnPath() && !batches[j].OwnPath() })
|
||||
var entries []inventory.Entry
|
||||
var read map[string][]inventory.ReadRepository
|
||||
var edges []inventory.Edge
|
||||
judged := false
|
||||
openWalks, err := openWalksOf(ctx, inv, started)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The oldest batch whose window closed is cut, an own-path one before a catalogue one (it goes first, and
|
||||
// the catalogue batch queues behind it rather than being cut and overtaken). Several of a kind may be open
|
||||
// since a merge that shares nothing is cut at once (ADR 0304 decision 1): walks stay one at a time, so one
|
||||
// is cut per look, and once its walk started the others are queued behind it.
|
||||
sort.SliceStable(batches, func(i, j int) bool { return batches[i].OwnPath() && !batches[j].OwnPath() })
|
||||
for i := range batches {
|
||||
b := &batches[i]
|
||||
if !windowClosed(*b, now) {
|
||||
continue
|
||||
}
|
||||
if b.OwnPath() {
|
||||
// A walk on the controller's own path folds no walk waiting for its word (decided during the build,
|
||||
// 2026-10-10): that walk keeps waiting beside it, and starts once this one ended (advanceOnce).
|
||||
waiting = nil
|
||||
}
|
||||
if err := cutBatch(ctx, open, b, waiting, now); err != nil {
|
||||
return err
|
||||
}
|
||||
// The batches left wait behind the walk just cut when it started; the next look does the same.
|
||||
if b.Open() && !b.Waiting() {
|
||||
if batches, err = inv.Batches(ctx); err != nil {
|
||||
if err := keepBatch(ctx, inv, b, now, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
return keepAll(b.ID)
|
||||
continue
|
||||
}
|
||||
if !judged {
|
||||
var err error
|
||||
if entries, err = inv.Catalogued(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if read, err = readForPlanning(ctx, inv); err != nil {
|
||||
return err
|
||||
}
|
||||
if edges, err = inv.Dependencies(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
judged = true
|
||||
}
|
||||
held, err := inv.MergesOf(ctx, b.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fp := footprintAsCut(held, entries, read, edges)
|
||||
var fold, keep []inventory.Plan
|
||||
if !b.OwnPath() {
|
||||
for _, w := range waiting {
|
||||
ww, err := openWalkOf(ctx, inv, w)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ok, _ := related(fp, ww.fp, edges); ok {
|
||||
fold = append(fold, w)
|
||||
fp = unionFootprint(fp, ww.fp)
|
||||
} else {
|
||||
keep = append(keep, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
if behind, why := relatedOpen(fp, b.OwnPath(), openWalks, edges); behind != "" {
|
||||
if err := keepBatchBecause(ctx, inv, b, now, behind, why); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := keepBatch(ctx, inv, b, now, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := cutBatch(ctx, open, b, fold, now); err != nil {
|
||||
return err
|
||||
}
|
||||
waiting = keep
|
||||
if c := windowClockOf(); c != nil {
|
||||
c.Forget(ctx, b.ID) // a batch cut has no window left to close
|
||||
}
|
||||
cut, err := inv.PlanByID(ctx, b.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !cut.Open() {
|
||||
continue
|
||||
}
|
||||
w, err := openWalkOf(ctx, inv, cut)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cut.Waiting() {
|
||||
waiting = append(waiting, cut)
|
||||
} else {
|
||||
openWalks = append(openWalks, w)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -976,9 +1139,13 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
|
||||
}
|
||||
}
|
||||
sort.Strings(also)
|
||||
ordered := groupOrderEdges(members, entries, read, edges)
|
||||
ordered, err := changeOrderEdges(ctx, merges, members, entries, moved, edges)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var withheld []string
|
||||
plan := planOfMoves(moved, append(append([]inventory.Edge{}, edges...), ordered...))
|
||||
placedBy(&plan, ordered)
|
||||
// **Withheld, a stopped merge's modules** (novox/hq ADR 0299): a module is built at its branch, which holds the
|
||||
// merge a stop left out of the walk these merges were walked again from — built here only as a dependent, it
|
||||
// would deliver that merge. One a merge of this batch moves is built: that merge carries the stopped change,
|
||||
@@ -1000,6 +1167,7 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
|
||||
plan.Delivery.Merges = named
|
||||
plan.Delivery.Alone = batch.Delivery != nil && batch.Delivery.Alone
|
||||
plan.Delivery.Withheld = withheld
|
||||
plan.Delivery.Order = ordered
|
||||
// **Left where it is, a module whose program did not change** (novox/hq issue 721): its check found it the same as
|
||||
// the build the mesh holds.
|
||||
for n, why := range leaveSameAsRunning(ctx, inv, &plan, moved, append(append([]inventory.Edge{}, edges...), ordered...)) {
|
||||
@@ -1031,7 +1199,14 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
|
||||
}
|
||||
var tiers []string
|
||||
for i, t := range plan.Tiers {
|
||||
tiers = append(tiers, fmt.Sprintf("%d: %s", i, strings.Join(t, ", ")))
|
||||
var names []string
|
||||
for _, n := range t {
|
||||
if m := plan.Modules[n]; m != nil && m.Placed != "" {
|
||||
n += " " + m.Placed
|
||||
}
|
||||
names = append(names, n)
|
||||
}
|
||||
tiers = append(tiers, fmt.Sprintf("%d: %s", i, strings.Join(names, ", ")))
|
||||
}
|
||||
var answers []string
|
||||
for _, n := range named {
|
||||
@@ -1059,6 +1234,23 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
|
||||
return nil
|
||||
}
|
||||
|
||||
// placedBy writes on each module of a plan the edges of its order that put it after another module of the plan, with the
|
||||
// rule that drew each (novox/hq ADR 0419 clause 5).
|
||||
func placedBy(plan *inventory.Plan, ordered []inventory.Edge) {
|
||||
for _, e := range ordered {
|
||||
m, in := plan.Modules[e.From]
|
||||
if _, before := plan.Modules[e.To]; !in || !before {
|
||||
continue
|
||||
}
|
||||
said := "after " + e.To + " (" + e.Why + ")"
|
||||
if m.Placed == "" {
|
||||
m.Placed = said
|
||||
} else if !strings.Contains(m.Placed, said) {
|
||||
m.Placed += "; " + said
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// unchangedSaid names the modules a walk says matched what runs, for its note.
|
||||
func unchangedSaid(unchanged map[string]string) string {
|
||||
names := make([]string, 0, len(unchanged))
|
||||
@@ -1247,73 +1439,46 @@ type batchMember struct {
|
||||
moved []string
|
||||
}
|
||||
|
||||
// afterLines are the `after: <repository>` lines of a pull request's description, as mesh-delivery reads them.
|
||||
var afterLines = regexp.MustCompile(`(?im)^\s*after:\s*([A-Za-z0-9_.\-/]+)\s*$`)
|
||||
|
||||
// groupOrderEdges is the order of every delivery group among a batch's merges as edges of its walk's tiers
|
||||
// (ADR 0276 decision 5, ADR 0249 unchanged): merges of two repositories or more from one head branch name are
|
||||
// a group; each pair its order rules give puts the modules of the member after in a later tier than the
|
||||
// member before's. A pair in a contradiction orders nothing: its group was refused before it merged.
|
||||
func groupOrderEdges(members []batchMember, entries []inventory.Entry, read map[string][]inventory.ReadRepository,
|
||||
edges []inventory.Edge) []inventory.Edge {
|
||||
byHead := map[string][]batchMember{}
|
||||
// changeOrderEdges is the order of a batch's moved modules as edges of its plan's tiers (novox/hq ADR 0419): drawn
|
||||
// per module from the module graph and from what the merges say of the manifests they touch (delivery_edges.go),
|
||||
// whatever repository the merges are in and whether or not they form a delivery group. No `after:` line is read.
|
||||
// A cycle is said, never lost: the plan puts the modules left in its last tier, and the composed check fails the
|
||||
// group before it merges.
|
||||
func changeOrderEdges(ctx context.Context, merges []inventory.BatchedMerge, members []batchMember,
|
||||
entries []inventory.Entry, moved []string, edges []inventory.Edge) ([]inventory.Edge, error) {
|
||||
// What each repository's branch held before the batch: the first parent of its earliest merge in it.
|
||||
earliest := map[string]inventory.BatchedMerge{}
|
||||
for _, k := range merges {
|
||||
key := strings.ToLower(k.Repository) + "\x00" + k.Branch
|
||||
if e, ok := earliest[key]; !ok || laterMerge(e, k) {
|
||||
earliest[key] = k
|
||||
}
|
||||
}
|
||||
var sources []changeSource
|
||||
for _, m := range members {
|
||||
if m.merge.Head != "" {
|
||||
byHead[m.merge.Head] = append(byHead[m.merge.Head], m)
|
||||
s := changeSource{Owner: m.merge.Owner, Repo: m.merge.Repo, New: m.merge.Commit,
|
||||
Paths: m.merge.Paths, Truncated: m.merge.PathsTruncated, Removed: m.merge.Removed}
|
||||
if e, ok := earliest[strings.ToLower(m.merge.Owner+"/"+m.merge.Repo)+"\x00"+m.merge.Base]; ok {
|
||||
s.Since = e.Commit
|
||||
}
|
||||
if s.Truncated {
|
||||
s.Paths, s.Truncated = truncatedPaths(entries, s.Owner, s.Repo, moved), false
|
||||
}
|
||||
sources = append(sources, s)
|
||||
}
|
||||
var out []inventory.Edge
|
||||
heads := make([]string, 0, len(byHead))
|
||||
for h := range byHead {
|
||||
heads = append(heads, h)
|
||||
changes, err := changesOf(ctx, sources)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("what the merges say of their manifests cannot be read, so the order of their modules "+
|
||||
"cannot be drawn: %w", err)
|
||||
}
|
||||
sort.Strings(heads)
|
||||
for _, h := range heads {
|
||||
group := byHead[h]
|
||||
repos := map[string]bool{}
|
||||
for _, m := range group {
|
||||
repos[strings.ToLower(m.merge.Owner+"/"+m.merge.Repo)] = true
|
||||
}
|
||||
if len(repos) < 2 {
|
||||
continue
|
||||
}
|
||||
var om []orderMember
|
||||
moved := map[string][]string{}
|
||||
for _, m := range group {
|
||||
repository := m.merge.Owner + "/" + m.merge.Repo
|
||||
id := repository + "@" + m.merge.Commit
|
||||
var after []string
|
||||
for _, l := range afterLines.FindAllStringSubmatch(m.merge.Body, -1) {
|
||||
after = append(after, strings.TrimSuffix(l[1], ".git"))
|
||||
}
|
||||
om = append(om, orderMember{ID: id, Repository: repository, Base: m.merge.Base, Head: m.merge.Commit,
|
||||
Number: m.merge.Number, Paths: m.merge.Paths, PathsTruncated: m.merge.PathsTruncated,
|
||||
Removed: m.merge.Removed, ModuleDirs: m.merge.ModuleDirs, ModuleDirsSaid: m.merge.ModuleDirsSaid,
|
||||
CloneURL: m.merge.CloneURL, After: after})
|
||||
moved[id] = m.moved
|
||||
}
|
||||
order := orderOf(om, reachOfMembers(om, entries, read, edges), edges)
|
||||
contradicted := map[string]bool{}
|
||||
for _, c := range order.Contradictions {
|
||||
for _, id := range c.Members {
|
||||
contradicted[id] = true
|
||||
}
|
||||
}
|
||||
for _, p := range order.Pairs {
|
||||
if contradicted[p.Before] && contradicted[p.After] {
|
||||
continue
|
||||
}
|
||||
for _, before := range moved[p.Before] {
|
||||
for _, after := range moved[p.After] {
|
||||
if before != after {
|
||||
out = append(out, inventory.Edge{From: after, To: before, Kind: edgeGroupOrder})
|
||||
}
|
||||
}
|
||||
}
|
||||
fmt.Printf(" group %s: %s before %s (%s) inside the walk\n", h, p.Before, p.After, p.Why)
|
||||
}
|
||||
order := orderEdges(moved, changes, edges)
|
||||
for _, e := range order.Edges {
|
||||
fmt.Printf(" order: %s\n", edgeSaid(e))
|
||||
}
|
||||
return out
|
||||
if said := order.CycleSaid(); said != "" {
|
||||
fmt.Printf(" %s\n", said)
|
||||
}
|
||||
return order.Edges, nil
|
||||
}
|
||||
|
||||
// searchAfterFailures starts the search a failed walk leaves (ADR 0276 decision 3): every merge it answered
|
||||
@@ -1396,7 +1561,11 @@ func batchWords(b inventory.Plan, now time.Time) string {
|
||||
grouped += " (on the controller's own path)"
|
||||
}
|
||||
if b.State == inventory.PlanQueued {
|
||||
return fmt.Sprintf("queued behind %s; grouped: %s; plan not yet calculated", w.Behind, grouped)
|
||||
because := ""
|
||||
if w.BehindWhy != "" {
|
||||
because = " (" + w.BehindWhy + ")"
|
||||
}
|
||||
return fmt.Sprintf("queued behind %s%s; grouped: %s; plan not yet calculated", w.Behind, because, grouped)
|
||||
}
|
||||
if w.CutAtOnce {
|
||||
return fmt.Sprintf("cut at once, sharing nothing with a batch assembling; grouped: %s; plan not yet calculated",
|
||||
|
||||
@@ -0,0 +1,698 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func windowed(t *testing.T) *stores {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
settings := t.TempDir() + "/merge-window.json"
|
||||
if err := os.WriteFile(settings, []byte(`{"merge-window": "90s", "merge-window-at-most": "10m"}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv(mergeWindowFileVar, settings)
|
||||
noGroups(t)
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Ref: "main", BuiltFrom: "c0",
|
||||
Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range []struct{ module, repository, path string }{
|
||||
{"app", "novox/mesh-catalog", "modules/app"},
|
||||
{"notes", "novox/mesh-catalog", "modules/notes"},
|
||||
{"one", "novox/one", ""},
|
||||
{"two", "novox/two", ""},
|
||||
{"three", "novox/three", ""},
|
||||
} {
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1"},
|
||||
inventory.Source{Repository: m.repository, Seat: "git", Path: m.path, Ref: "main", BuiltFrom: "c0",
|
||||
Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return open
|
||||
}
|
||||
|
||||
func hear(t *testing.T, open *stores, m link.SourceMoved, now time.Time) {
|
||||
t.Helper()
|
||||
if err := (following{open: open}).hearMerge(t.Context(), m, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func cutAt(t *testing.T, open *stores, now time.Time) {
|
||||
t.Helper()
|
||||
if err := cutBatchesHeld(t.Context(), open, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTwoMergesSecondsApartAreOneWalkAtTheLaterCommit(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asked := asksWithPaths(t)
|
||||
claude := catalogueMerge("553b7191claude", "app", t0)
|
||||
dunst := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second))
|
||||
hear(t, open, claude, t0.Add(time.Second))
|
||||
hear(t, open, dunst, t0.Add(18*time.Second))
|
||||
if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 1 {
|
||||
t.Fatalf("within the window: %d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
cutAt(t, open, t0.Add(18*time.Second+89*time.Second))
|
||||
if ws, _ := walks(t, open); len(ws) != 0 {
|
||||
t.Fatalf("cut before the window closed: %+v", ws)
|
||||
}
|
||||
cutAt(t, open, t0.Add(18*time.Second+90*time.Second))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 0 {
|
||||
t.Fatalf("after the window: %d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
w := ws[0]
|
||||
if w.Commit != dunst.Commit || len(w.Commits) != 1 || w.Commits[0].Commit != dunst.Commit {
|
||||
t.Fatalf("the walk is at %s %+v, not the later commit", w.Commit, w.Commits)
|
||||
}
|
||||
for _, m := range []string{"app", "notes"} {
|
||||
if _, in := w.Modules[m]; !in {
|
||||
t.Fatalf("the walk does not build %s, which one of its merges moved: %v", m, w.Modules)
|
||||
}
|
||||
}
|
||||
want := []inventory.PlanMerge{{Repository: "novox/mesh-catalog", Commit: claude.Commit, Carried: dunst.Commit},
|
||||
{Repository: "novox/mesh-catalog", Commit: dunst.Commit}}
|
||||
if w.Delivery == nil || !slices.EqualFunc(w.Delivery.Merges, want, sameMerge) {
|
||||
t.Fatalf("the walk answers %+v, want %+v", w.Delivery, want)
|
||||
}
|
||||
if len(*asked) != 2 || (*asked)[0][2] != "main" || (*asked)[1][2] != "main" {
|
||||
t.Fatalf("the walk did not ask its modules at the branch, which holds the commit it carries: %v", *asked)
|
||||
}
|
||||
// Heard again, as the bus may hand it over twice: never a second merge, nor a second walk.
|
||||
hear(t, open, claude, t0.Add(5*time.Minute))
|
||||
if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 {
|
||||
t.Fatalf("a merge heard twice made %d walk(s) and %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestThreeRepositoriesInOneWindowAreOneWalk(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "feat/three")
|
||||
asksRecorded(t)
|
||||
for i, r := range []string{"one", "two", "three"} {
|
||||
hear(t, open, fromHead(repoMerge(r, "c-"+r, t0.Add(time.Duration(i)*20*time.Second)), "feat/three"),
|
||||
t0.Add(time.Duration(i)*20*time.Second+time.Second))
|
||||
}
|
||||
cutAt(t, open, t0.Add(41*time.Second+90*time.Second))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 0 {
|
||||
t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
w := ws[0]
|
||||
if len(w.Commits) != 3 || len(w.Delivery.Merges) != 3 {
|
||||
t.Fatalf("the walk carries %+v and answers %+v", w.Commits, w.Delivery.Merges)
|
||||
}
|
||||
for _, r := range []string{"one", "two", "three"} {
|
||||
if w.CommitOf("novox/"+r) != "c-"+r {
|
||||
t.Fatalf("the walk carries %s at %q", r, w.CommitOf("novox/"+r))
|
||||
}
|
||||
if _, in := w.Modules[r]; !in {
|
||||
t.Fatalf("the walk does not build %s: %v", r, w.Modules)
|
||||
}
|
||||
}
|
||||
// Each module's ask is recorded at its own repository's commit.
|
||||
for _, r := range []string{"one", "two", "three"} {
|
||||
q, found, err := open.inventory.BuildRequestByID(t.Context(), w.Modules[r].Build)
|
||||
if err != nil || !found || q.Commit != "c-"+r {
|
||||
t.Fatalf("%s's build was not recorded at its own commit: %+v %v %v", r, q, found, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMergeRestartsTheWindowAndTheMaximumClosesIt(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "feat/both", "feat/busy")
|
||||
asksWithPaths(t)
|
||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0)
|
||||
hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(80*time.Second)), "feat/both"), t0.Add(80*time.Second))
|
||||
cutAt(t, open, t0.Add(100*time.Second))
|
||||
if ws, _ := walks(t, open); len(ws) != 0 {
|
||||
t.Fatal("the window closed 90 seconds after its first merge, not after its last")
|
||||
}
|
||||
cutAt(t, open, t0.Add(170*time.Second))
|
||||
if ws, _ := walks(t, open); len(ws) != 1 || len(ws[0].Commits) != 2 {
|
||||
t.Fatalf("the window did not close 90 seconds after its last merge: %+v", ws)
|
||||
}
|
||||
|
||||
busy := windowed(t)
|
||||
groupsWaiting(t, "feat/both", "feat/busy")
|
||||
asksWithPaths(t)
|
||||
var last time.Time
|
||||
for i := 0; i < 12; i++ {
|
||||
last = t0.Add(time.Duration(i) * time.Minute)
|
||||
repo := []string{"one", "two", "three"}[i%3]
|
||||
hear(t, busy, fromHead(repoMerge(repo, "c"+string(rune('a'+i)), last), "feat/busy"), last)
|
||||
if ws, _ := walks(t, busy); len(ws) > 0 {
|
||||
if i != 10 {
|
||||
t.Fatalf("a busy window closed at merge %d (minute %d), not at its maximum", i, i)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
ws, _ := walks(t, busy)
|
||||
if len(ws) != 1 {
|
||||
t.Fatalf("ten minutes of merges a minute apart were never cut: %d walk(s)", len(ws))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheAssemblingBatchIsShown(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "pr/553b7191claude")
|
||||
asksWithPaths(t)
|
||||
var said []inventory.Plan
|
||||
was := inventory.PlanSaved
|
||||
inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { said = append(said, p) }
|
||||
t.Cleanup(func() { inventory.PlanSaved = was })
|
||||
now := time.Now().UTC()
|
||||
claude := catalogueMerge("553b7191claude", "app", now.Add(-20*time.Second))
|
||||
claude.Number, claude.Title = 174, "claude-code: an agent proposes a section"
|
||||
dunst := catalogueMerge("48bda475dunst", "notes", now.Add(-2*time.Second))
|
||||
dunst.Number, dunst.Title = 175, "dunst: the font the operator chose"
|
||||
hear(t, open, claude, now.Add(-19*time.Second))
|
||||
hear(t, open, dunst, now.Add(-time.Second))
|
||||
hear(t, open, fromHead(repoMerge("one", "a6bc0931one", now), dunst.Head), now)
|
||||
out := captured(t, func() error { return plansCommand(t.Context(), nil) })
|
||||
lines := strings.Split(out, "\n")
|
||||
first := lines[0]
|
||||
for _, want := range []string{"assembling: ", " s left (at the latest ", "grouped: novox/mesh-catalog@48bda475 " +
|
||||
"(answers 553b7191), novox/one@a6bc0931; plan not yet calculated"} {
|
||||
if !strings.Contains(first, want) {
|
||||
t.Fatalf("plans' first line %q does not say %q", first, want)
|
||||
}
|
||||
}
|
||||
// Under it, one line per repository: the pull request, what it answers, what it moves.
|
||||
if len(lines) < 3 || strings.TrimSpace(lines[1]) != "mesh-catalog #175 dunst: the font the operator chose (answers "+
|
||||
"#174 claude-code: an agent proposes a section) · app, notes" ||
|
||||
strings.TrimSpace(lines[2]) != "one a6bc0931 · one" {
|
||||
t.Fatalf("the grouped list reads %q", lines[1:4])
|
||||
}
|
||||
if len(said) == 0 || said[len(said)-1].State != inventory.PlanAssembling || said[len(said)-1].Delivery.Batch == nil ||
|
||||
len(said[len(said)-1].Delivery.Merges) != 3 {
|
||||
t.Fatalf("the batch was not said as assembling with its merges: %+v", said)
|
||||
}
|
||||
}
|
||||
|
||||
func captured(t *testing.T, run func() error) string {
|
||||
t.Helper()
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
was := os.Stdout
|
||||
os.Stdout = w
|
||||
runErr := run()
|
||||
os.Stdout = was
|
||||
_ = w.Close()
|
||||
var b bytes.Buffer
|
||||
_, _ = io.Copy(&b, r)
|
||||
if runErr != nil {
|
||||
t.Fatal(runErr)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func TestAMergeDuringAWalkJoinsTheNextBatch(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 || !ws[0].Open() {
|
||||
t.Fatalf("the first batch was not cut: %+v", ws)
|
||||
}
|
||||
first := ws[0]
|
||||
hear(t, open, repoMerge("one", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
|
||||
cutAt(t, open, t0.Add(5*time.Minute))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != first.ID {
|
||||
t.Fatalf("the merge during the walk: %d walk(s), batches %+v", len(ws), bs)
|
||||
}
|
||||
if got, _ := open.inventory.PlanByID(t.Context(), first.ID); !got.Open() {
|
||||
t.Fatalf("the started walk was %s by the next merge", got.State)
|
||||
}
|
||||
if !strings.HasPrefix(batchWords(bs[0], t0.Add(5*time.Minute)), "queued behind "+first.ID) {
|
||||
t.Fatalf("a queued batch reads %q", batchWords(bs[0], t0.Add(5*time.Minute)))
|
||||
}
|
||||
first.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(t.Context(), &first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(6*time.Minute))
|
||||
ws, bs = walks(t, open)
|
||||
if len(ws) != 2 || len(bs) != 0 || ws[0].Commit != "c2" {
|
||||
t.Fatalf("the queued batch was not cut when the walk ended: %+v %+v", ws, bs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWaitingWalkIsFoldedIntoTheNextBatch(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 || !ws[0].Waiting() {
|
||||
t.Fatalf("the walk does not wait for its word: %+v", ws)
|
||||
}
|
||||
waiting := ws[0]
|
||||
hear(t, open, repoMerge("one", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
|
||||
cutAt(t, open, t0.Add(2*time.Minute+90*time.Second))
|
||||
ws, bs := walks(t, open)
|
||||
if len(bs) != 0 || len(ws) != 2 {
|
||||
t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
folded, err := open.inventory.PlanByID(ctx, waiting.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
newer := ws[0]
|
||||
if folded.State != inventory.PlanSuperseded || folded.Delivery.TakenOverBy != newer.ID {
|
||||
t.Fatalf("the waiting walk is %s, taken over by %q", folded.State, folded.Delivery.TakenOverBy)
|
||||
}
|
||||
if len(newer.Delivery.Merges) != 2 || newer.CommitOf("novox/one") != "c2" {
|
||||
t.Fatalf("the newer walk answers %+v", newer.Delivery.Merges)
|
||||
}
|
||||
if _, in := newer.Modules["one"]; !in {
|
||||
t.Fatalf("what the folded walk was to build is not built: %v", newer.Modules)
|
||||
}
|
||||
}
|
||||
|
||||
func TestALateMergeIsAnsweredByTheWalkOfTheLaterOne(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
later := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second))
|
||||
hear(t, open, later, t0.Add(18*time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
w := ws[0]
|
||||
w.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Moves only notes, which the done walk built from the branch after it: answered there, at once.
|
||||
hear(t, open, catalogueMerge("553b7191early", "notes", t0), t0.Add(15*time.Minute))
|
||||
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
||||
if got.State != inventory.PlanDone || len(got.Delivery.Merges) != 2 ||
|
||||
!sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: "553b7191early",
|
||||
Carried: later.Commit}) {
|
||||
t.Fatalf("the late merge is not named by the walk that carried it: %+v", got.Delivery.Merges)
|
||||
}
|
||||
// Moves app, which that walk never built: the next batch walks it.
|
||||
hear(t, open, catalogueMerge("1111aaaaapp", "app", t0.Add(time.Second)), t0.Add(16*time.Minute))
|
||||
_, bs := walks(t, open)
|
||||
if len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "1111aaaaapp" {
|
||||
t.Fatalf("a late merge moving what the walk never built did not join the next batch: %+v", bs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailedWalkWalksItsEarlierMergesAlone(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asked := asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
// Made before the cut marks their modules seen, as merges are: walked again, they are news all the same.
|
||||
oldest := catalogueMerge("aaaa0001", "app", t0.Add(-3*time.Minute))
|
||||
middle := catalogueMerge("bbbb0002", "app", t0.Add(-3*time.Minute+10*time.Second))
|
||||
newest := catalogueMerge("cccc0003", "notes", t0.Add(-3*time.Minute+20*time.Second))
|
||||
for i, m := range []link.SourceMoved{oldest, middle, newest} {
|
||||
hear(t, open, m, t0.Add(time.Duration(i)*10*time.Second+time.Second))
|
||||
}
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
failed := ws[0]
|
||||
failed.State, failed.Note = inventory.PlanFailed, "notes failed to build in tier 0"
|
||||
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(3*time.Minute))
|
||||
got, _ := open.inventory.PlanByID(ctx, failed.ID)
|
||||
if got.State != inventory.PlanFailed || len(got.Delivery.Merges) != 3 {
|
||||
t.Fatalf("the failed walk's record changed: %s %+v", got.State, got.Delivery.Merges)
|
||||
}
|
||||
ws, _ = walks(t, open)
|
||||
alone := ws[0]
|
||||
if alone.ID == failed.ID || alone.Commit != middle.Commit || len(alone.Delivery.Merges) != 1 ||
|
||||
!sameMerge(alone.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: middle.Commit}) {
|
||||
t.Fatalf("the newest earlier merge was not walked alone on its own commit: %+v", alone)
|
||||
}
|
||||
if _, in := alone.Modules["app"]; !in || (*asked)[len(*asked)-1] != [3]string{"novox/mesh-catalog", "modules/app",
|
||||
middle.Commit} {
|
||||
t.Fatalf("the merge walked alone does not build app at its own commit: %v, asked %v", alone.Modules, *asked)
|
||||
}
|
||||
// Retried, the failed walk would name merges the search answers now.
|
||||
if _, err := retryPlan(ctx, open, failed.ID); err == nil || !strings.Contains(err.Error(), "walked alone") {
|
||||
t.Fatalf("a failed walk whose merges are searched was retried: %v", err)
|
||||
}
|
||||
alone.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &alone); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(4*time.Minute))
|
||||
ws, bs := walks(t, open)
|
||||
if ws[0].ID != alone.ID || len(bs) != 0 {
|
||||
t.Fatalf("the search went on after a merge was delivered: %+v", ws[0])
|
||||
}
|
||||
done, _ := open.inventory.PlanByID(ctx, alone.ID)
|
||||
if len(done.Delivery.Merges) != 2 || !sameMerge(done.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog",
|
||||
Commit: oldest.Commit, Carried: middle.Commit}) {
|
||||
t.Fatalf("the oldest merge is not answered by the walk that delivered the one after it: %+v", done.Delivery.Merges)
|
||||
}
|
||||
// A stopped walk starts no search: a person ended it.
|
||||
hear(t, open, catalogueMerge("dddd0004", "app", t0.Add(5*time.Minute)), t0.Add(5*time.Minute))
|
||||
hear(t, open, catalogueMerge("eeee0005", "notes", t0.Add(5*time.Minute+time.Second)), t0.Add(5*time.Minute+time.Second))
|
||||
cutAt(t, open, t0.Add(8*time.Minute))
|
||||
ws, _ = walks(t, open)
|
||||
if _, err := stopWalk(ctx, open.inventory, ws[0].ID, "mesh-delivery for jochen", "not now"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(9*time.Minute))
|
||||
if again, _ := walks(t, open); again[0].ID != ws[0].ID {
|
||||
t.Fatalf("a stopped walk's earlier merge was walked alone: %+v", again[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAGroupsOrderBecomesTiersInsideTheWalk(t *testing.T) {
|
||||
for _, grouped := range []bool{true, false} {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
for _, m := range []string{"mesh-host"} {
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"},
|
||||
inventory.Source{Repository: "novox/" + m, Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
host := repoMerge("mesh-host", "h1", t0)
|
||||
ctl := repoMerge("mesh-controller", "k1", t0.Add(time.Second))
|
||||
host.Head, ctl.Head = "feat/together", "feat/together"
|
||||
if !grouped {
|
||||
ctl.Head = "feat/alone"
|
||||
}
|
||||
hear(t, open, ctl, t0.Add(2*time.Second))
|
||||
hear(t, open, host, t0.Add(3*time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 {
|
||||
t.Fatalf("%d walks", len(ws))
|
||||
}
|
||||
tiers := ws[0].Tiers
|
||||
if grouped {
|
||||
if len(tiers) != 2 || !slices.Equal(tiers[0], []string{"mesh-host"}) ||
|
||||
!slices.Equal(tiers[1], []string{"mesh-controller"}) {
|
||||
t.Fatalf("the group's order is not the walk's tiers: %v", tiers)
|
||||
}
|
||||
} else if len(tiers) != 1 {
|
||||
t.Fatalf("two merges of no group were ordered: %v", tiers)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestABatchSurvivesARestart(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "feat/both")
|
||||
asksWithPaths(t)
|
||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0)
|
||||
hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(30*time.Second)), "feat/both"), t0.Add(30*time.Second))
|
||||
|
||||
// A new controller: nothing of the first one's but the store.
|
||||
again, err := openStores(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(again.Close)
|
||||
hear(t, again, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0.Add(60*time.Second)) // the bus hands it over again
|
||||
cutAt(t, again, t0.Add(119*time.Second))
|
||||
if ws, bs := walks(t, again); len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 {
|
||||
t.Fatalf("the restarted controller's batch: %d walk(s), %+v", len(ws), bs)
|
||||
}
|
||||
cutAt(t, again, t0.Add(120*time.Second))
|
||||
ws, bs := walks(t, again)
|
||||
if len(ws) != 1 || len(bs) != 0 || len(ws[0].Delivery.Merges) != 2 {
|
||||
t.Fatalf("the restarted controller did not cut the batch at its window: %+v %+v", ws, bs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestALateMergeOfAOneModuleRepositoryIsNamed(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
c2 := repoMerge("one", "c2", t0.Add(-50*time.Second))
|
||||
hear(t, open, c2, t0)
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
w := ws[0]
|
||||
w.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, repoMerge("one", "c1", t0.Add(-60*time.Second)), t0.Add(15*time.Minute))
|
||||
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
||||
if len(got.Delivery.Merges) != 2 || !sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/one",
|
||||
Commit: "c1", Carried: "c2"}) {
|
||||
t.Fatalf("the late merge was not named by the walk that carried it: %+v", got.Delivery.Merges)
|
||||
}
|
||||
|
||||
// A walk of two that built only two: a late merge of one is not its to name.
|
||||
hear(t, open, repoMerge("two", "d2", t0.Add(16*time.Minute)), t0.Add(16*time.Minute))
|
||||
cutAt(t, open, t0.Add(18*time.Minute))
|
||||
ws, _ = walks(t, open)
|
||||
w = ws[0]
|
||||
delete(w.Modules, "two")
|
||||
w.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, repoMerge("two", "d1", t0.Add(15*time.Minute)), t0.Add(20*time.Minute))
|
||||
if got, _ := open.inventory.PlanByID(ctx, w.ID); len(got.Delivery.Merges) != 1 {
|
||||
t.Fatalf("a walk that never built what the late merge moves named it: %+v", got.Delivery.Merges)
|
||||
}
|
||||
if _, bs := walks(t, open); len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "d1" {
|
||||
t.Fatalf("the late merge did not join the next batch: %+v", bs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWalkLetGoBesideAStartedOneStartsOnceItEnded(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asked := asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
started, _ := walks(t, open)
|
||||
waiting := inventory.Plan{ID: "plan-waiting", Repository: "novox/one", Branch: "main", Commit: "d1", Created: t0,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"one"}}, Modules: map[string]*inventory.PlanModule{"one": {}},
|
||||
Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}}
|
||||
if err := open.inventory.SavePlan(ctx, &waiting); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := letGo(ctx, open.inventory, waiting.ID, catalogue.DeliverySeat, "its turn"); err != nil {
|
||||
t.Fatalf("the word was refused beside %s: %v", started[0].ID, err)
|
||||
}
|
||||
before := len(*asked)
|
||||
advanceHeld(ctx, open)
|
||||
got, _ := open.inventory.PlanByID(ctx, waiting.ID)
|
||||
if len(*asked) != before || !strings.Contains(got.Note, "starts once "+started[0].ID) {
|
||||
t.Fatalf("a walk let go beside a started one asked (%d → %d) or does not say it waits: %q", before, len(*asked), got.Note)
|
||||
}
|
||||
// Read as a wait, an hour on: its note on the line, never LATE, and no tier of it late for S3.
|
||||
later := time.Now().Add(time.Hour)
|
||||
if line := planLine(got, later); !strings.Contains(line, "starts once "+started[0].ID) || strings.Contains(line, "LATE") {
|
||||
t.Fatalf("a deferred walk reads %q", line)
|
||||
}
|
||||
facts, _, err := gatherPlans(ctx, open.inventory, later, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range facts {
|
||||
if f.id == got.ID {
|
||||
t.Fatalf("a deferred walk is watched as a tier running late: %+v", f)
|
||||
}
|
||||
}
|
||||
done := started[0]
|
||||
done.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) != before+1 {
|
||||
t.Fatalf("the walk did not start once the started one ended: asked %v", *asked)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMergeOnTheControllersPathNeverSharesABatch(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asked := asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, catalogueMerge("aaaa0001", "app", t0), t0)
|
||||
hear(t, open, repoMerge("mesh-controller", "k1", t0.Add(10*time.Second)), t0.Add(10*time.Second))
|
||||
hear(t, open, catalogueMerge("bbbb0002", "notes", t0.Add(20*time.Second)), t0.Add(20*time.Second))
|
||||
_, bs := walks(t, open)
|
||||
var ownBatch, catalogueBatch inventory.Plan
|
||||
for _, b := range bs {
|
||||
if b.OwnPath() {
|
||||
ownBatch = b
|
||||
} else {
|
||||
catalogueBatch = b
|
||||
}
|
||||
}
|
||||
if len(bs) != 2 || ownBatch.ID == "" || catalogueBatch.ID == "" || len(catalogueBatch.Delivery.Merges) != 2 ||
|
||||
len(ownBatch.Delivery.Merges) != 1 {
|
||||
t.Fatalf("a controller merge and two catalogue merges in one window: %+v", bs)
|
||||
}
|
||||
if !strings.Contains(batchWords(ownBatch, t0.Add(30*time.Second)), "own path") {
|
||||
t.Fatalf("the own-path batch does not say so: %q", batchWords(ownBatch, t0.Add(30*time.Second)))
|
||||
}
|
||||
// Both windows closed, the controller's batch is cut first and starts; the catalogue batch queues behind it,
|
||||
// is cut when the controller's walk ended, and waits for its word with both merges.
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || ws[0].Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k1" {
|
||||
t.Fatalf("the controller's batch was not cut first into a started walk: %+v", ws)
|
||||
}
|
||||
ownWalk := ws[0]
|
||||
for _, m := range []string{"app", "notes"} {
|
||||
if _, in := ownWalk.Modules[m]; in {
|
||||
t.Fatalf("the controller's walk builds %s, a catalogue module: it skipped its turn", m)
|
||||
}
|
||||
}
|
||||
if len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != ownWalk.ID || bs[0].OwnPath() {
|
||||
t.Fatalf("the catalogue batch does not queue behind the controller's walk: %+v", bs)
|
||||
}
|
||||
ownWalk.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &ownWalk); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(3*time.Minute))
|
||||
ws, bs = walks(t, open)
|
||||
if len(bs) != 0 || ws[0].ID != catalogueBatch.ID || !ws[0].Waiting() || len(ws[0].Delivery.Merges) != 2 {
|
||||
t.Fatalf("the catalogue batch was not cut into a waiting walk once the controller's ended: %+v %+v", ws, bs)
|
||||
}
|
||||
for _, m := range []string{"app", "notes"} {
|
||||
if _, in := ws[0].Modules[m]; !in {
|
||||
t.Fatalf("the catalogue walk does not build %s: %v", m, ws[0].Modules)
|
||||
}
|
||||
}
|
||||
for _, a := range *asked {
|
||||
if a[1] == "modules/app" || a[1] == "modules/notes" {
|
||||
t.Fatalf("a catalogue module was asked without the word: %v", *asked)
|
||||
}
|
||||
}
|
||||
|
||||
// A catalogue walk waiting for its word when an own-path batch is cut keeps waiting beside the own-path
|
||||
// walk, is not folded into it, and its word is taken meanwhile: it starts once the own-path walk ended.
|
||||
catalogueWalk := ws[0]
|
||||
hear(t, open, repoMerge("mesh-controller", "k2", t0.Add(4*time.Minute)), t0.Add(4*time.Minute))
|
||||
cutAt(t, open, t0.Add(6*time.Minute))
|
||||
ws, _ = walks(t, open)
|
||||
kept, _ := open.inventory.PlanByID(ctx, catalogueWalk.ID)
|
||||
if !kept.Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k2" || ws[0].Waiting() {
|
||||
t.Fatalf("the waiting catalogue walk was not kept waiting beside the controller's walk: %s %+v", kept.State, ws)
|
||||
}
|
||||
if _, in := ws[0].Modules["app"]; in {
|
||||
t.Fatalf("the controller's walk folded the waiting catalogue walk in: %v", ws[0].Modules)
|
||||
}
|
||||
if _, err := letGo(ctx, open.inventory, catalogueWalk.ID, catalogue.DeliverySeat, "its turn"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := len(*asked)
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) != before {
|
||||
t.Fatalf("the catalogue walk started beside the controller's: asked %v", (*asked)[before:])
|
||||
}
|
||||
own2 := ws[0]
|
||||
own2.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &own2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) < before+1 || (*asked)[before][1] != "modules/app" {
|
||||
t.Fatalf("the catalogue walk did not start once the controller's ended: %v", (*asked)[before:])
|
||||
}
|
||||
}
|
||||
|
||||
func TestALateCatalogueMergeIsNotAnsweredByAnOwnPathWalk(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
for _, m := range []struct {
|
||||
module string
|
||||
claims []catalogue.Claim
|
||||
}{
|
||||
{"nats", nil},
|
||||
{"mesh-delivery", []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
||||
} {
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1", Claims: m.claims},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + m.module, Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mixed := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "m1xed", MergedAt: t0.Format(time.RFC3339Nano),
|
||||
Paths: []string{"modules/nats/module.json", "modules/app/module.json"}, ModuleDirs: []string{"modules/nats", "modules/app"},
|
||||
ModuleDirsSaid: true}
|
||||
hear(t, open, mixed, t0.Add(time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 || ws[0].Waiting() {
|
||||
t.Fatalf("the mixed merge's walk waited, or was not cut: %+v", ws)
|
||||
}
|
||||
done := ws[0]
|
||||
done.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, catalogueMerge("ear1ier", "app", t0.Add(-30*time.Second)), t0.Add(3*time.Minute))
|
||||
got, _ := open.inventory.PlanByID(ctx, done.ID)
|
||||
_, bs := walks(t, open)
|
||||
if len(got.Delivery.Merges) != 1 || len(bs) != 1 || bs[0].OwnPath() || bs[0].Delivery.Merges[0].Commit != "ear1ier" {
|
||||
t.Fatalf("the late catalogue merge was answered by the own-path walk (%+v) rather than the next catalogue batch (%+v)",
|
||||
got.Delivery.Merges, bs)
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,9 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"errors"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
@@ -19,42 +18,19 @@ import (
|
||||
// The suite's tests of a merge, but for the merge window's own (this file's), plan the merge as it is heard: a
|
||||
// window of nothing closes at once, so a merge with no walk open is cut into its walk at once, as every merge
|
||||
// was planned before novox/hq ADR 0276. The window's tests set it through the controller's settings.
|
||||
func init() { mergeWindowDefault = 0 }
|
||||
func init() {
|
||||
mergeWindowDefault = 0
|
||||
// No forge in a test: no manifest is found at any commit, so a merge draws no edge of the manifests' change
|
||||
// (delivery_edges.go). A test that wants some replaces these (forgeHolding).
|
||||
manifestAt = func(context.Context, string, string, string, string) (catalogue.Manifest, bool, error) {
|
||||
return catalogue.Manifest{}, false, nil
|
||||
}
|
||||
parentOf = func(context.Context, string, string, string) (string, error) { return "", nil }
|
||||
}
|
||||
|
||||
// windowed is a mesh whose controller's settings give a merge window of 90 seconds and at most 10 minutes, as its
|
||||
// settings file says them, and
|
||||
// modules built from the catalogue (app, notes) and from three repositories of their own.
|
||||
func windowed(t *testing.T) *stores {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
settings := t.TempDir() + "/merge-window.json"
|
||||
if err := os.WriteFile(settings, []byte(`{"merge-window": "90s", "merge-window-at-most": "10m"}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv(mergeWindowFileVar, settings)
|
||||
noGroups(t)
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Ref: "main", BuiltFrom: "c0",
|
||||
Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range []struct{ module, repository, path string }{
|
||||
{"app", "novox/mesh-catalog", "modules/app"},
|
||||
{"notes", "novox/mesh-catalog", "modules/notes"},
|
||||
{"one", "novox/one", ""},
|
||||
{"two", "novox/two", ""},
|
||||
{"three", "novox/three", ""},
|
||||
} {
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1"},
|
||||
inventory.Source{Repository: m.repository, Seat: "git", Path: m.path, Ref: "main", BuiltFrom: "c0",
|
||||
Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return open
|
||||
}
|
||||
|
||||
// t0 is the moment a test's clock starts: merges are dated by it, and heard after it.
|
||||
var t0 = time.Now().UTC().Truncate(time.Second)
|
||||
@@ -83,15 +59,16 @@ func noGroups(t *testing.T) {
|
||||
t.Cleanup(func() { deliveryGroupOf = was })
|
||||
}
|
||||
|
||||
// groupsWaiting stubs mesh-delivery's word on groups for a test: each head named is a group with a member of another
|
||||
// repository whose pull request is open (novox/elsewhere, ready), so a merge from it waits for the window (ADR 0306
|
||||
// decision 2); every other head names no group.
|
||||
// groupsWaiting stubs mesh-delivery's word on groups for a test of the merge window's own timing: each head named
|
||||
// is one whose group mesh-delivery cannot say, so a merge from it waits for the window (ADR 0306 decision 2) and
|
||||
// nothing holds the batch open; every other head names no group. A group with a member still to merge, which holds
|
||||
// its batch open however long it takes (ADR 0418 clause 3), is groupStates'.
|
||||
func groupsWaiting(t *testing.T, heads ...string) {
|
||||
t.Helper()
|
||||
was := deliveryGroupOf
|
||||
deliveryGroupOf = func(_ context.Context, _ bool, head string) ([]groupMember, bool, error) {
|
||||
if slices.Contains(heads, head) {
|
||||
return []groupMember{{Repository: "novox/elsewhere", State: "ready"}}, true, nil
|
||||
return nil, false, errors.New("the test's mesh-delivery does not say")
|
||||
}
|
||||
return nil, false, nil
|
||||
}
|
||||
@@ -99,12 +76,37 @@ func groupsWaiting(t *testing.T, heads ...string) {
|
||||
}
|
||||
|
||||
// everyHeadWaits stubs mesh-delivery's word on groups for a test whose merges each wait for the window as they did
|
||||
// before novox/hq ADR 0304: every head is a group with a member of another repository still to merge.
|
||||
// before novox/hq ADR 0304: mesh-delivery cannot say any head's group.
|
||||
func everyHeadWaits(t *testing.T) {
|
||||
t.Helper()
|
||||
was := deliveryGroupOf
|
||||
deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) {
|
||||
return []groupMember{{Repository: "novox/elsewhere", State: "ready"}}, true, nil
|
||||
return nil, false, errors.New("the test's mesh-delivery does not say")
|
||||
}
|
||||
t.Cleanup(func() { deliveryGroupOf = was })
|
||||
}
|
||||
|
||||
// groupStates stubs mesh-delivery's word on one group: its members and their states, which a test changes as the
|
||||
// members merge.
|
||||
func groupStates(t *testing.T, head string, states map[string]string) {
|
||||
t.Helper()
|
||||
groupStatesFrom(t, head, states, true)
|
||||
}
|
||||
|
||||
// groupStatesFrom is groupStates for a mesh-delivery that does (composed) or does not yet say the group's gate is the
|
||||
// composed check's: only the first holds the group's batch open for its last member.
|
||||
func groupStatesFrom(t *testing.T, head string, states map[string]string, composed bool) {
|
||||
t.Helper()
|
||||
was := deliveryGroupOf
|
||||
deliveryGroupOf = func(_ context.Context, _ bool, asked string) ([]groupMember, bool, error) {
|
||||
if asked != head {
|
||||
return nil, false, nil
|
||||
}
|
||||
var members []groupMember
|
||||
for repository, state := range states {
|
||||
members = append(members, groupMember{Repository: repository, State: state, Composed: composed})
|
||||
}
|
||||
return members, true, nil
|
||||
}
|
||||
t.Cleanup(func() { deliveryGroupOf = was })
|
||||
}
|
||||
@@ -124,20 +126,8 @@ func lone(m link.SourceMoved) link.SourceMoved {
|
||||
}
|
||||
|
||||
// hear is a merge heard at a moment.
|
||||
func hear(t *testing.T, open *stores, m link.SourceMoved, now time.Time) {
|
||||
t.Helper()
|
||||
if err := (following{open: open}).hearMerge(t.Context(), m, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// cutAt is the cutter looking at a moment.
|
||||
func cutAt(t *testing.T, open *stores, now time.Time) {
|
||||
t.Helper()
|
||||
if err := cutBatchesHeld(t.Context(), open, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// walks is every plan record that is a walk, newest first, and the batches not yet cut.
|
||||
func walks(t *testing.T, open *stores) (walks, batches []inventory.Plan) {
|
||||
@@ -158,426 +148,34 @@ func walks(t *testing.T, open *stores) (walks, batches []inventory.Plan) {
|
||||
|
||||
// The replay of issue 362: two catalogue merges 18 seconds apart are one batch, cut once into one walk at the
|
||||
// later commit, which names both merges — the earlier carried by the later.
|
||||
func TestTwoMergesSecondsApartAreOneWalkAtTheLaterCommit(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asked := asksWithPaths(t)
|
||||
claude := catalogueMerge("553b7191claude", "app", t0)
|
||||
dunst := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second))
|
||||
hear(t, open, claude, t0.Add(time.Second))
|
||||
hear(t, open, dunst, t0.Add(18*time.Second))
|
||||
if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 1 {
|
||||
t.Fatalf("within the window: %d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
cutAt(t, open, t0.Add(18*time.Second+89*time.Second))
|
||||
if ws, _ := walks(t, open); len(ws) != 0 {
|
||||
t.Fatalf("cut before the window closed: %+v", ws)
|
||||
}
|
||||
cutAt(t, open, t0.Add(18*time.Second+90*time.Second))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 0 {
|
||||
t.Fatalf("after the window: %d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
w := ws[0]
|
||||
if w.Commit != dunst.Commit || len(w.Commits) != 1 || w.Commits[0].Commit != dunst.Commit {
|
||||
t.Fatalf("the walk is at %s %+v, not the later commit", w.Commit, w.Commits)
|
||||
}
|
||||
for _, m := range []string{"app", "notes"} {
|
||||
if _, in := w.Modules[m]; !in {
|
||||
t.Fatalf("the walk does not build %s, which one of its merges moved: %v", m, w.Modules)
|
||||
}
|
||||
}
|
||||
want := []inventory.PlanMerge{{Repository: "novox/mesh-catalog", Commit: claude.Commit, Carried: dunst.Commit},
|
||||
{Repository: "novox/mesh-catalog", Commit: dunst.Commit}}
|
||||
if w.Delivery == nil || !slices.EqualFunc(w.Delivery.Merges, want, sameMerge) {
|
||||
t.Fatalf("the walk answers %+v, want %+v", w.Delivery, want)
|
||||
}
|
||||
if len(*asked) != 2 || (*asked)[0][2] != "main" || (*asked)[1][2] != "main" {
|
||||
t.Fatalf("the walk did not ask its modules at the branch, which holds the commit it carries: %v", *asked)
|
||||
}
|
||||
// Heard again, as the bus may hand it over twice: never a second merge, nor a second walk.
|
||||
hear(t, open, claude, t0.Add(5*time.Minute))
|
||||
if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 {
|
||||
t.Fatalf("a merge heard twice made %d walk(s) and %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
}
|
||||
|
||||
// Merges of three repositories in one window are one walk, one commit for each.
|
||||
func TestThreeRepositoriesInOneWindowAreOneWalk(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "feat/three")
|
||||
asksRecorded(t)
|
||||
for i, r := range []string{"one", "two", "three"} {
|
||||
hear(t, open, fromHead(repoMerge(r, "c-"+r, t0.Add(time.Duration(i)*20*time.Second)), "feat/three"),
|
||||
t0.Add(time.Duration(i)*20*time.Second+time.Second))
|
||||
}
|
||||
cutAt(t, open, t0.Add(41*time.Second+90*time.Second))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 0 {
|
||||
t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
w := ws[0]
|
||||
if len(w.Commits) != 3 || len(w.Delivery.Merges) != 3 {
|
||||
t.Fatalf("the walk carries %+v and answers %+v", w.Commits, w.Delivery.Merges)
|
||||
}
|
||||
for _, r := range []string{"one", "two", "three"} {
|
||||
if w.CommitOf("novox/"+r) != "c-"+r {
|
||||
t.Fatalf("the walk carries %s at %q", r, w.CommitOf("novox/"+r))
|
||||
}
|
||||
if _, in := w.Modules[r]; !in {
|
||||
t.Fatalf("the walk does not build %s: %v", r, w.Modules)
|
||||
}
|
||||
}
|
||||
// Each module's ask is recorded at its own repository's commit.
|
||||
for _, r := range []string{"one", "two", "three"} {
|
||||
q, found, err := open.inventory.BuildRequestByID(t.Context(), w.Modules[r].Build)
|
||||
if err != nil || !found || q.Commit != "c-"+r {
|
||||
t.Fatalf("%s's build was not recorded at its own commit: %+v %v %v", r, q, found, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Each merge restarts the window: a merge at second 80 keeps the batch open until second 170; the window
|
||||
// closes at most ten minutes after its first merge however busy.
|
||||
func TestAMergeRestartsTheWindowAndTheMaximumClosesIt(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "feat/both", "feat/busy")
|
||||
asksWithPaths(t)
|
||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0)
|
||||
hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(80*time.Second)), "feat/both"), t0.Add(80*time.Second))
|
||||
cutAt(t, open, t0.Add(100*time.Second))
|
||||
if ws, _ := walks(t, open); len(ws) != 0 {
|
||||
t.Fatal("the window closed 90 seconds after its first merge, not after its last")
|
||||
}
|
||||
cutAt(t, open, t0.Add(170*time.Second))
|
||||
if ws, _ := walks(t, open); len(ws) != 1 || len(ws[0].Commits) != 2 {
|
||||
t.Fatalf("the window did not close 90 seconds after its last merge: %+v", ws)
|
||||
}
|
||||
|
||||
busy := windowed(t)
|
||||
groupsWaiting(t, "feat/both", "feat/busy")
|
||||
asksWithPaths(t)
|
||||
var last time.Time
|
||||
for i := 0; i < 12; i++ {
|
||||
last = t0.Add(time.Duration(i) * time.Minute)
|
||||
repo := []string{"one", "two", "three"}[i%3]
|
||||
hear(t, busy, fromHead(repoMerge(repo, "c"+string(rune('a'+i)), last), "feat/busy"), last)
|
||||
if ws, _ := walks(t, busy); len(ws) > 0 {
|
||||
if i != 10 {
|
||||
t.Fatalf("a busy window closed at merge %d (minute %d), not at its maximum", i, i)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
ws, _ := walks(t, busy)
|
||||
if len(ws) != 1 {
|
||||
t.Fatalf("ten minutes of merges a minute apart were never cut: %d walk(s)", len(ws))
|
||||
}
|
||||
}
|
||||
|
||||
// `plans` lists the batch being assembled first, in the operator's words: how long is left, when at the latest,
|
||||
// what is grouped, and that the plan is not yet calculated. It says so on the bus as plan-moved, too.
|
||||
func TestTheAssemblingBatchIsShown(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "pr/553b7191claude")
|
||||
asksWithPaths(t)
|
||||
var said []inventory.Plan
|
||||
was := inventory.PlanSaved
|
||||
inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { said = append(said, p) }
|
||||
t.Cleanup(func() { inventory.PlanSaved = was })
|
||||
now := time.Now().UTC()
|
||||
claude := catalogueMerge("553b7191claude", "app", now.Add(-20*time.Second))
|
||||
claude.Number, claude.Title = 174, "claude-code: an agent proposes a section"
|
||||
dunst := catalogueMerge("48bda475dunst", "notes", now.Add(-2*time.Second))
|
||||
dunst.Number, dunst.Title = 175, "dunst: the font the operator chose"
|
||||
hear(t, open, claude, now.Add(-19*time.Second))
|
||||
hear(t, open, dunst, now.Add(-time.Second))
|
||||
hear(t, open, fromHead(repoMerge("one", "a6bc0931one", now), dunst.Head), now)
|
||||
out := captured(t, func() error { return plansCommand(t.Context(), nil) })
|
||||
lines := strings.Split(out, "\n")
|
||||
first := lines[0]
|
||||
for _, want := range []string{"assembling: ", " s left (at the latest ", "grouped: novox/mesh-catalog@48bda475 " +
|
||||
"(answers 553b7191), novox/one@a6bc0931; plan not yet calculated"} {
|
||||
if !strings.Contains(first, want) {
|
||||
t.Fatalf("plans' first line %q does not say %q", first, want)
|
||||
}
|
||||
}
|
||||
// Under it, one line per repository: the pull request, what it answers, what it moves.
|
||||
if len(lines) < 3 || strings.TrimSpace(lines[1]) != "mesh-catalog #175 dunst: the font the operator chose (answers "+
|
||||
"#174 claude-code: an agent proposes a section) · app, notes" ||
|
||||
strings.TrimSpace(lines[2]) != "one a6bc0931 · one" {
|
||||
t.Fatalf("the grouped list reads %q", lines[1:4])
|
||||
}
|
||||
if len(said) == 0 || said[len(said)-1].State != inventory.PlanAssembling || said[len(said)-1].Delivery.Batch == nil ||
|
||||
len(said[len(said)-1].Delivery.Merges) != 3 {
|
||||
t.Fatalf("the batch was not said as assembling with its merges: %+v", said)
|
||||
}
|
||||
}
|
||||
|
||||
// captured is what a command printed.
|
||||
func captured(t *testing.T, run func() error) string {
|
||||
t.Helper()
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
was := os.Stdout
|
||||
os.Stdout = w
|
||||
runErr := run()
|
||||
os.Stdout = was
|
||||
_ = w.Close()
|
||||
var b bytes.Buffer
|
||||
_, _ = io.Copy(&b, r)
|
||||
if runErr != nil {
|
||||
t.Fatal(runErr)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// One walk at a time: a merge heard while a walk runs joins the next batch, which is cut when the walk ends;
|
||||
// the walk that started is never superseded.
|
||||
func TestAMergeDuringAWalkJoinsTheNextBatch(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 || !ws[0].Open() {
|
||||
t.Fatalf("the first batch was not cut: %+v", ws)
|
||||
}
|
||||
first := ws[0]
|
||||
hear(t, open, repoMerge("two", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
|
||||
cutAt(t, open, t0.Add(5*time.Minute))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != first.ID {
|
||||
t.Fatalf("the merge during the walk: %d walk(s), batches %+v", len(ws), bs)
|
||||
}
|
||||
if got, _ := open.inventory.PlanByID(t.Context(), first.ID); !got.Open() {
|
||||
t.Fatalf("the started walk was %s by the next merge", got.State)
|
||||
}
|
||||
if !strings.HasPrefix(batchWords(bs[0], t0.Add(5*time.Minute)), "queued behind "+first.ID) {
|
||||
t.Fatalf("a queued batch reads %q", batchWords(bs[0], t0.Add(5*time.Minute)))
|
||||
}
|
||||
first.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(t.Context(), &first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(6*time.Minute))
|
||||
ws, bs = walks(t, open)
|
||||
if len(ws) != 2 || len(bs) != 0 || ws[0].Commit != "c2" {
|
||||
t.Fatalf("the queued batch was not cut when the walk ended: %+v %+v", ws, bs)
|
||||
}
|
||||
}
|
||||
// A merge heard during a walk it is related to (the same repository, issue 973) joins the next batch, which is
|
||||
// cut when the walk ends; the walk that started is never superseded (ADR 0276 decision 4, ADR 0407 decision 4).
|
||||
|
||||
// A walk waiting for its delivery's word is folded into the next batch's walk, which answers its merges; it
|
||||
// names the walk that took it over.
|
||||
func TestAWaitingWalkIsFoldedIntoTheNextBatch(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 || !ws[0].Waiting() {
|
||||
t.Fatalf("the walk does not wait for its word: %+v", ws)
|
||||
}
|
||||
waiting := ws[0]
|
||||
hear(t, open, repoMerge("two", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
|
||||
cutAt(t, open, t0.Add(2*time.Minute+90*time.Second))
|
||||
ws, bs := walks(t, open)
|
||||
if len(bs) != 0 || len(ws) != 2 {
|
||||
t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
folded, err := open.inventory.PlanByID(ctx, waiting.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
newer := ws[0]
|
||||
if folded.State != inventory.PlanSuperseded || folded.Delivery.TakenOverBy != newer.ID {
|
||||
t.Fatalf("the waiting walk is %s, taken over by %q", folded.State, folded.Delivery.TakenOverBy)
|
||||
}
|
||||
if len(newer.Delivery.Merges) != 2 || newer.CommitOf("novox/one") != "c1" || newer.CommitOf("novox/two") != "c2" {
|
||||
t.Fatalf("the newer walk answers %+v", newer.Delivery.Merges)
|
||||
}
|
||||
if _, in := newer.Modules["one"]; !in {
|
||||
t.Fatalf("what the folded walk was to build is not built: %v", newer.Modules)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge heard after a later merge of its branch was walked is answered by that walk when it builds all it
|
||||
// moves — named at once on a walk done — and joins the next batch when it does not.
|
||||
func TestALateMergeIsAnsweredByTheWalkOfTheLaterOne(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
later := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second))
|
||||
hear(t, open, later, t0.Add(18*time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
w := ws[0]
|
||||
w.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Moves only notes, which the done walk built from the branch after it: answered there, at once.
|
||||
hear(t, open, catalogueMerge("553b7191early", "notes", t0), t0.Add(15*time.Minute))
|
||||
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
||||
if got.State != inventory.PlanDone || len(got.Delivery.Merges) != 2 ||
|
||||
!sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: "553b7191early",
|
||||
Carried: later.Commit}) {
|
||||
t.Fatalf("the late merge is not named by the walk that carried it: %+v", got.Delivery.Merges)
|
||||
}
|
||||
// Moves app, which that walk never built: the next batch walks it.
|
||||
hear(t, open, catalogueMerge("1111aaaaapp", "app", t0.Add(time.Second)), t0.Add(16*time.Minute))
|
||||
_, bs := walks(t, open)
|
||||
if len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "1111aaaaapp" {
|
||||
t.Fatalf("a late merge moving what the walk never built did not join the next batch: %+v", bs)
|
||||
}
|
||||
}
|
||||
|
||||
// A failed walk marks nothing delivered: its earlier merges are walked alone, on their own commit, the newest
|
||||
// first; the first delivered answers the older ones, and the search stops.
|
||||
func TestAFailedWalkWalksItsEarlierMergesAlone(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asked := asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
// Made before the cut marks their modules seen, as merges are: walked again, they are news all the same.
|
||||
oldest := catalogueMerge("aaaa0001", "app", t0.Add(-3*time.Minute))
|
||||
middle := catalogueMerge("bbbb0002", "app", t0.Add(-3*time.Minute+10*time.Second))
|
||||
newest := catalogueMerge("cccc0003", "notes", t0.Add(-3*time.Minute+20*time.Second))
|
||||
for i, m := range []link.SourceMoved{oldest, middle, newest} {
|
||||
hear(t, open, m, t0.Add(time.Duration(i)*10*time.Second+time.Second))
|
||||
}
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
failed := ws[0]
|
||||
failed.State, failed.Note = inventory.PlanFailed, "notes failed to build in tier 0"
|
||||
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(3*time.Minute))
|
||||
got, _ := open.inventory.PlanByID(ctx, failed.ID)
|
||||
if got.State != inventory.PlanFailed || len(got.Delivery.Merges) != 3 {
|
||||
t.Fatalf("the failed walk's record changed: %s %+v", got.State, got.Delivery.Merges)
|
||||
}
|
||||
ws, _ = walks(t, open)
|
||||
alone := ws[0]
|
||||
if alone.ID == failed.ID || alone.Commit != middle.Commit || len(alone.Delivery.Merges) != 1 ||
|
||||
!sameMerge(alone.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: middle.Commit}) {
|
||||
t.Fatalf("the newest earlier merge was not walked alone on its own commit: %+v", alone)
|
||||
}
|
||||
if _, in := alone.Modules["app"]; !in || (*asked)[len(*asked)-1] != [3]string{"novox/mesh-catalog", "modules/app",
|
||||
middle.Commit} {
|
||||
t.Fatalf("the merge walked alone does not build app at its own commit: %v, asked %v", alone.Modules, *asked)
|
||||
}
|
||||
// Retried, the failed walk would name merges the search answers now.
|
||||
if _, err := retryPlan(ctx, open, failed.ID); err == nil || !strings.Contains(err.Error(), "walked alone") {
|
||||
t.Fatalf("a failed walk whose merges are searched was retried: %v", err)
|
||||
}
|
||||
alone.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &alone); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(4*time.Minute))
|
||||
ws, bs := walks(t, open)
|
||||
if ws[0].ID != alone.ID || len(bs) != 0 {
|
||||
t.Fatalf("the search went on after a merge was delivered: %+v", ws[0])
|
||||
}
|
||||
done, _ := open.inventory.PlanByID(ctx, alone.ID)
|
||||
if len(done.Delivery.Merges) != 2 || !sameMerge(done.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog",
|
||||
Commit: oldest.Commit, Carried: middle.Commit}) {
|
||||
t.Fatalf("the oldest merge is not answered by the walk that delivered the one after it: %+v", done.Delivery.Merges)
|
||||
}
|
||||
// A stopped walk starts no search: a person ended it.
|
||||
hear(t, open, catalogueMerge("dddd0004", "app", t0.Add(5*time.Minute)), t0.Add(5*time.Minute))
|
||||
hear(t, open, catalogueMerge("eeee0005", "notes", t0.Add(5*time.Minute+time.Second)), t0.Add(5*time.Minute+time.Second))
|
||||
cutAt(t, open, t0.Add(8*time.Minute))
|
||||
ws, _ = walks(t, open)
|
||||
if _, err := stopWalk(ctx, open.inventory, ws[0].ID, "mesh-delivery for jochen", "not now"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(9*time.Minute))
|
||||
if again, _ := walks(t, open); again[0].ID != ws[0].ID {
|
||||
t.Fatalf("a stopped walk's earlier merge was walked alone: %+v", again[0])
|
||||
}
|
||||
}
|
||||
|
||||
// A delivery group's order holds inside the walk (ADR 0249 unchanged): merges of two repositories from one head
|
||||
// branch name, the node-engine's before the controller's, are tiered so; without the group, one tier.
|
||||
func TestAGroupsOrderBecomesTiersInsideTheWalk(t *testing.T) {
|
||||
for _, grouped := range []bool{true, false} {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
for _, m := range []string{"mesh-host"} {
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"},
|
||||
inventory.Source{Repository: "novox/" + m, Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
host := repoMerge("mesh-host", "h1", t0)
|
||||
ctl := repoMerge("mesh-controller", "k1", t0.Add(time.Second))
|
||||
host.Head, ctl.Head = "feat/together", "feat/together"
|
||||
if !grouped {
|
||||
ctl.Head = "feat/alone"
|
||||
}
|
||||
hear(t, open, ctl, t0.Add(2*time.Second))
|
||||
hear(t, open, host, t0.Add(3*time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 {
|
||||
t.Fatalf("%d walks", len(ws))
|
||||
}
|
||||
tiers := ws[0].Tiers
|
||||
if grouped {
|
||||
if len(tiers) != 2 || !slices.Equal(tiers[0], []string{"mesh-host"}) ||
|
||||
!slices.Equal(tiers[1], []string{"mesh-controller"}) {
|
||||
t.Fatalf("the group's order is not the walk's tiers: %v", tiers)
|
||||
}
|
||||
} else if len(tiers) != 1 {
|
||||
t.Fatalf("two merges of no group were ordered: %v", tiers)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A restarted controller resumes the window where it stood: the batch, its merges and their times are in the
|
||||
// store, a merge handed over again is not doubled, and the batch is cut when its window closes.
|
||||
func TestABatchSurvivesARestart(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "feat/both")
|
||||
asksWithPaths(t)
|
||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0)
|
||||
hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(30*time.Second)), "feat/both"), t0.Add(30*time.Second))
|
||||
|
||||
// A new controller: nothing of the first one's but the store.
|
||||
again, err := openStores(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(again.Close)
|
||||
hear(t, again, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0.Add(60*time.Second)) // the bus hands it over again
|
||||
cutAt(t, again, t0.Add(119*time.Second))
|
||||
if ws, bs := walks(t, again); len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 {
|
||||
t.Fatalf("the restarted controller's batch: %d walk(s), %+v", len(ws), bs)
|
||||
}
|
||||
cutAt(t, again, t0.Add(120*time.Second))
|
||||
ws, bs := walks(t, again)
|
||||
if len(ws) != 1 || len(bs) != 0 || len(ws[0].Delivery.Merges) != 2 {
|
||||
t.Fatalf("the restarted controller did not cut the batch at its window: %+v %+v", ws, bs)
|
||||
}
|
||||
}
|
||||
|
||||
// The window's settings reach the controller in its settings file, read at every look; one that says no
|
||||
// duration, or no file, is the default.
|
||||
@@ -622,94 +220,9 @@ func TestAWaitingWalkNamesTheMergesItAnswers(t *testing.T) {
|
||||
// A merge heard after a later merge of its branch was cut is named by that walk however its modules read since
|
||||
// the cut, in a repository of one module (review of this change); a walk that never built what it moves names
|
||||
// nothing, and the merge joins the next batch.
|
||||
func TestALateMergeOfAOneModuleRepositoryIsNamed(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
c2 := repoMerge("one", "c2", t0.Add(-50*time.Second))
|
||||
hear(t, open, c2, t0)
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
w := ws[0]
|
||||
w.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, repoMerge("one", "c1", t0.Add(-60*time.Second)), t0.Add(15*time.Minute))
|
||||
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
||||
if len(got.Delivery.Merges) != 2 || !sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/one",
|
||||
Commit: "c1", Carried: "c2"}) {
|
||||
t.Fatalf("the late merge was not named by the walk that carried it: %+v", got.Delivery.Merges)
|
||||
}
|
||||
|
||||
// A walk of two that built only two: a late merge of one is not its to name.
|
||||
hear(t, open, repoMerge("two", "d2", t0.Add(16*time.Minute)), t0.Add(16*time.Minute))
|
||||
cutAt(t, open, t0.Add(18*time.Minute))
|
||||
ws, _ = walks(t, open)
|
||||
w = ws[0]
|
||||
delete(w.Modules, "two")
|
||||
w.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, repoMerge("two", "d1", t0.Add(15*time.Minute)), t0.Add(20*time.Minute))
|
||||
if got, _ := open.inventory.PlanByID(ctx, w.ID); len(got.Delivery.Merges) != 1 {
|
||||
t.Fatalf("a walk that never built what the late merge moves named it: %+v", got.Delivery.Merges)
|
||||
}
|
||||
if _, bs := walks(t, open); len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "d1" {
|
||||
t.Fatalf("the late merge did not join the next batch: %+v", bs)
|
||||
}
|
||||
}
|
||||
|
||||
// One walk at a time holds against the delivery's word too: a waiting walk let go beside a started one takes the
|
||||
// word and starts once that one ended, asking nothing before.
|
||||
func TestAWalkLetGoBesideAStartedOneStartsOnceItEnded(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asked := asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
started, _ := walks(t, open)
|
||||
waiting := inventory.Plan{ID: "plan-waiting", Repository: "novox/two", Branch: "main", Commit: "d1", Created: t0,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"two"}}, Modules: map[string]*inventory.PlanModule{"two": {}},
|
||||
Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}}
|
||||
if err := open.inventory.SavePlan(ctx, &waiting); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := letGo(ctx, open.inventory, waiting.ID, catalogue.DeliverySeat, "its turn"); err != nil {
|
||||
t.Fatalf("the word was refused beside %s: %v", started[0].ID, err)
|
||||
}
|
||||
before := len(*asked)
|
||||
advanceHeld(ctx, open)
|
||||
got, _ := open.inventory.PlanByID(ctx, waiting.ID)
|
||||
if len(*asked) != before || !strings.Contains(got.Note, "starts once "+started[0].ID) {
|
||||
t.Fatalf("a walk let go beside a started one asked (%d → %d) or does not say it waits: %q", before, len(*asked), got.Note)
|
||||
}
|
||||
// Read as a wait, an hour on: its note on the line, never LATE, and no tier of it late for S3.
|
||||
later := time.Now().Add(time.Hour)
|
||||
if line := planLine(got, later); !strings.Contains(line, "starts once "+started[0].ID) || strings.Contains(line, "LATE") {
|
||||
t.Fatalf("a deferred walk reads %q", line)
|
||||
}
|
||||
facts, _, err := gatherPlans(ctx, open.inventory, later, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range facts {
|
||||
if f.id == got.ID {
|
||||
t.Fatalf("a deferred walk is watched as a tier running late: %+v", f)
|
||||
}
|
||||
}
|
||||
done := started[0]
|
||||
done.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) != before+1 {
|
||||
t.Fatalf("the walk did not start once the started one ended: asked %v", *asked)
|
||||
}
|
||||
}
|
||||
// A related walk waits for the delivery's word too: a waiting walk let go beside a started one it is related to
|
||||
// (the same repository, issue 973) takes the word and starts once that one ended, asking nothing before.
|
||||
|
||||
// A file a merge of the batch removed and a later one brought back is not removed; one removed last is.
|
||||
func TestARemovedFileIsTheLastMergesWord(t *testing.T) {
|
||||
@@ -729,153 +242,10 @@ func TestARemovedFileIsTheLastMergesWord(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The catch-up hands over what the bus lost after its branch's later merges were cut, and the record keeps it:
|
||||
// an earlier merge is named by the walk that carried it; a merge made before a cut and heard after it, which
|
||||
// no later merge carries, joins the next batch — neither reads as history and is dropped (review of this change).
|
||||
func TestTheCatchUpKeepsWhatACutMadeHistory(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
base := time.Now().UTC().Add(-time.Hour).Truncate(time.Second)
|
||||
hear(t, open, repoMerge("one", "c2", base.Add(10*time.Second)), base.Add(11*time.Second))
|
||||
cutAt(t, open, base.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
w := ws[0]
|
||||
w.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lost := []link.AnnouncedMerge{
|
||||
{SourceMoved: repoMerge("one", "c1", base), At: base.Add(time.Second)},
|
||||
{SourceMoved: repoMerge("one", "c3lost", base.Add(20*time.Second)), At: base.Add(21 * time.Second)},
|
||||
}
|
||||
catalogued := func(ctx context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
|
||||
entries, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
read, err := readForPlanning(ctx, open.inventory)
|
||||
return entries, read, err
|
||||
}
|
||||
if err := catchUpOnMerges(ctx, time.Now(), unheardMerges{announcedList(lost), open.inventory}, catalogued,
|
||||
(following{open: open}).SourceMoved, t.Logf); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
||||
if len(got.Delivery.Merges) != 2 || !sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/one",
|
||||
Commit: "c1", Carried: "c2"}) {
|
||||
t.Fatalf("the earlier merge the catch-up handed over is not named by the walk that carried it: %+v",
|
||||
got.Delivery.Merges)
|
||||
}
|
||||
if _, kept, err := open.inventory.MergeOf(ctx, "novox/one", "c3lost"); err != nil || !kept {
|
||||
t.Fatalf("the merge made before the cut and heard after it was dropped: %v %v", kept, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge on the controller's own path never shares a batch with one that waits for mesh-delivery's word
|
||||
// (decided during the build, 2026-10-10): the two are batches of their own, cut one after the other; a walk on
|
||||
// the own path folds no waiting catalogue walk but batches its merges again behind it; the catalogue's walk
|
||||
// still waits for the word, so no catalogue delivery skips its turn.
|
||||
func TestAMergeOnTheControllersPathNeverSharesABatch(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asked := asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, catalogueMerge("aaaa0001", "app", t0), t0)
|
||||
hear(t, open, repoMerge("mesh-controller", "k1", t0.Add(10*time.Second)), t0.Add(10*time.Second))
|
||||
hear(t, open, catalogueMerge("bbbb0002", "notes", t0.Add(20*time.Second)), t0.Add(20*time.Second))
|
||||
_, bs := walks(t, open)
|
||||
var ownBatch, catalogueBatch inventory.Plan
|
||||
for _, b := range bs {
|
||||
if b.OwnPath() {
|
||||
ownBatch = b
|
||||
} else {
|
||||
catalogueBatch = b
|
||||
}
|
||||
}
|
||||
if len(bs) != 2 || ownBatch.ID == "" || catalogueBatch.ID == "" || len(catalogueBatch.Delivery.Merges) != 2 ||
|
||||
len(ownBatch.Delivery.Merges) != 1 {
|
||||
t.Fatalf("a controller merge and two catalogue merges in one window: %+v", bs)
|
||||
}
|
||||
if !strings.Contains(batchWords(ownBatch, t0.Add(30*time.Second)), "own path") {
|
||||
t.Fatalf("the own-path batch does not say so: %q", batchWords(ownBatch, t0.Add(30*time.Second)))
|
||||
}
|
||||
// Both windows closed, the controller's batch is cut first and starts; the catalogue batch queues behind it,
|
||||
// is cut when the controller's walk ended, and waits for its word with both merges.
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || ws[0].Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k1" {
|
||||
t.Fatalf("the controller's batch was not cut first into a started walk: %+v", ws)
|
||||
}
|
||||
ownWalk := ws[0]
|
||||
for _, m := range []string{"app", "notes"} {
|
||||
if _, in := ownWalk.Modules[m]; in {
|
||||
t.Fatalf("the controller's walk builds %s, a catalogue module: it skipped its turn", m)
|
||||
}
|
||||
}
|
||||
if len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != ownWalk.ID || bs[0].OwnPath() {
|
||||
t.Fatalf("the catalogue batch does not queue behind the controller's walk: %+v", bs)
|
||||
}
|
||||
ownWalk.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &ownWalk); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(3*time.Minute))
|
||||
ws, bs = walks(t, open)
|
||||
if len(bs) != 0 || ws[0].ID != catalogueBatch.ID || !ws[0].Waiting() || len(ws[0].Delivery.Merges) != 2 {
|
||||
t.Fatalf("the catalogue batch was not cut into a waiting walk once the controller's ended: %+v %+v", ws, bs)
|
||||
}
|
||||
for _, m := range []string{"app", "notes"} {
|
||||
if _, in := ws[0].Modules[m]; !in {
|
||||
t.Fatalf("the catalogue walk does not build %s: %v", m, ws[0].Modules)
|
||||
}
|
||||
}
|
||||
for _, a := range *asked {
|
||||
if a[1] == "modules/app" || a[1] == "modules/notes" {
|
||||
t.Fatalf("a catalogue module was asked without the word: %v", *asked)
|
||||
}
|
||||
}
|
||||
|
||||
// A catalogue walk waiting for its word when an own-path batch is cut keeps waiting beside the own-path
|
||||
// walk, is not folded into it, and its word is taken meanwhile: it starts once the own-path walk ended.
|
||||
catalogueWalk := ws[0]
|
||||
hear(t, open, repoMerge("mesh-controller", "k2", t0.Add(4*time.Minute)), t0.Add(4*time.Minute))
|
||||
cutAt(t, open, t0.Add(6*time.Minute))
|
||||
ws, _ = walks(t, open)
|
||||
kept, _ := open.inventory.PlanByID(ctx, catalogueWalk.ID)
|
||||
if !kept.Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k2" || ws[0].Waiting() {
|
||||
t.Fatalf("the waiting catalogue walk was not kept waiting beside the controller's walk: %s %+v", kept.State, ws)
|
||||
}
|
||||
if _, in := ws[0].Modules["app"]; in {
|
||||
t.Fatalf("the controller's walk folded the waiting catalogue walk in: %v", ws[0].Modules)
|
||||
}
|
||||
if _, err := letGo(ctx, open.inventory, catalogueWalk.ID, catalogue.DeliverySeat, "its turn"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := len(*asked)
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) != before {
|
||||
t.Fatalf("the catalogue walk started beside the controller's: asked %v", (*asked)[before:])
|
||||
}
|
||||
own2 := ws[0]
|
||||
own2.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &own2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) < before+1 || (*asked)[before][1] != "modules/app" {
|
||||
t.Fatalf("the catalogue walk did not start once the controller's ended: %v", (*asked)[before:])
|
||||
}
|
||||
}
|
||||
|
||||
// `plans` names a walk by what it moves (asked by the operator, 2026-10-10): the repository's pull request and
|
||||
// title, the modules it moves and the machines running them, then the state words; a record naming no pull
|
||||
@@ -921,46 +291,3 @@ func sameMerge(a, b inventory.PlanMerge) bool {
|
||||
// A catalogue merge heard after a later merge of its branch was walked without the word (a merge that touched
|
||||
// the bus too, on the controller's own path) is not answered by that walk while the delivery seat has a holder:
|
||||
// its delivery would skip its turn. It joins the next catalogue batch.
|
||||
func TestALateCatalogueMergeIsNotAnsweredByAnOwnPathWalk(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
for _, m := range []struct {
|
||||
module string
|
||||
claims []catalogue.Claim
|
||||
}{
|
||||
{"nats", nil},
|
||||
{"mesh-delivery", []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
||||
} {
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1", Claims: m.claims},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + m.module, Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mixed := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "m1xed", MergedAt: t0.Format(time.RFC3339Nano),
|
||||
Paths: []string{"modules/nats/module.json", "modules/app/module.json"}, ModuleDirs: []string{"modules/nats", "modules/app"},
|
||||
ModuleDirsSaid: true}
|
||||
hear(t, open, mixed, t0.Add(time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 || ws[0].Waiting() {
|
||||
t.Fatalf("the mixed merge's walk waited, or was not cut: %+v", ws)
|
||||
}
|
||||
done := ws[0]
|
||||
done.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, catalogueMerge("ear1ier", "app", t0.Add(-30*time.Second)), t0.Add(3*time.Minute))
|
||||
got, _ := open.inventory.PlanByID(ctx, done.ID)
|
||||
_, bs := walks(t, open)
|
||||
if len(got.Delivery.Merges) != 1 || len(bs) != 1 || bs[0].OwnPath() || bs[0].Delivery.Merges[0].Commit != "ear1ier" {
|
||||
t.Fatalf("the late catalogue merge was answered by the own-path walk (%+v) rather than the next catalogue batch (%+v)",
|
||||
got.Delivery.Merges, bs)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,8 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
@@ -17,6 +19,9 @@ import (
|
||||
const (
|
||||
kindBatchNotCut = "batch-not-cut"
|
||||
kindBatchBehindWalk = "batch-behind-walk"
|
||||
// kindGroupNotMerged is raised for a group whose last member has not merged by the merge window's maximum after
|
||||
// its first (novox/hq ADR 0418 clause 3).
|
||||
kindGroupNotMerged = "group-not-merged"
|
||||
)
|
||||
|
||||
// batchFacts is one batch not yet cut, as the watchdogs read it.
|
||||
@@ -28,6 +33,11 @@ type batchFacts struct {
|
||||
// tiers.
|
||||
behind string
|
||||
walkBound time.Duration
|
||||
// holds are the groups the batch is held for, by head, with why; created is when the batch was made, the first
|
||||
// merge of the group heard; atMostWindow is the maximum a window may take (the controller's setting).
|
||||
holds map[string]string
|
||||
created time.Time
|
||||
atMostWindow time.Duration
|
||||
}
|
||||
|
||||
// gatherBatches is every batch not yet cut, with the bound of the walk it waits behind.
|
||||
@@ -42,8 +52,10 @@ func gatherBatches(ctx context.Context, inv *inventory.Inventory, now time.Time)
|
||||
}
|
||||
var out []batchFacts
|
||||
for _, b := range batches {
|
||||
f := batchFacts{id: b.ID, state: b.State, grouped: groupedWords(b)}
|
||||
_, atMostWindow := mergeWindowOf()
|
||||
f := batchFacts{id: b.ID, state: b.State, grouped: groupedWords(b), created: b.Created, atMostWindow: atMostWindow}
|
||||
if w := b.Delivery; w != nil && w.Batch != nil {
|
||||
f.holds = w.Batch.Holds
|
||||
f.atMost, f.closed, f.behind = w.Batch.AtMost, w.Batch.ClosesAt, w.Batch.Behind
|
||||
if f.atMost.Before(f.closed) {
|
||||
f.closed = f.atMost
|
||||
@@ -67,7 +79,8 @@ func gatherBatches(ctx context.Context, inv *inventory.Inventory, now time.Time)
|
||||
func watchBatchesNotCut(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, b := range f.batches {
|
||||
if b.state != inventory.PlanAssembling || b.atMost.IsZero() || f.now.Sub(b.atMost) <= batchLateAfter {
|
||||
if b.state != inventory.PlanAssembling || b.atMost.IsZero() || f.now.Sub(b.atMost) <= batchLateAfter ||
|
||||
len(b.holds) > 0 { // a batch held for a group waits on a member, which has its own condition
|
||||
continue
|
||||
}
|
||||
late := f.now.Sub(b.atMost)
|
||||
@@ -105,3 +118,36 @@ func watchBatchesBehind(f *signalFacts) []conditions.Observation {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// watchGroupsNotMerged is the condition of a batch held for a group whose last member has not merged by the merge
|
||||
// window's maximum after the first member merged, naming the member and why (novox/hq ADR 0418 clause 3): nothing of
|
||||
// the group is delivered while it waits.
|
||||
func watchGroupsNotMerged(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, b := range f.batches {
|
||||
if len(b.holds) == 0 || b.created.IsZero() || f.now.Sub(b.created) <= b.atMostWindow {
|
||||
continue
|
||||
}
|
||||
heads := make([]string, 0, len(b.holds))
|
||||
for h := range b.holds {
|
||||
heads = append(heads, h)
|
||||
}
|
||||
sort.Strings(heads)
|
||||
var why []string
|
||||
for _, h := range heads {
|
||||
why = append(why, b.holds[h])
|
||||
}
|
||||
waited := f.now.Sub(b.created)
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id,
|
||||
Kind: kindGroupNotMerged, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s; its first member merged %s ago, past the merge window's maximum (%s), and nothing "+
|
||||
"of the group is delivered while it waits (batch %s)", strings.Join(why, "; "), ago(waited), ago(b.atMostWindow), b.id),
|
||||
Said: "a group waits for a member that has not merged",
|
||||
Headline: "A change waits for the rest of its group to merge",
|
||||
Explanation: fmt.Sprintf("The changes of a group are delivered together, once the last one is merged. "+
|
||||
"One has not merged in %s. Nothing is lost; the merged ones are kept and delivered with it.",
|
||||
humanDuration(waited)),
|
||||
Resolved: "The group's last change merged"})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// An unrelated closed batch is cut into a walk beside an open one, and that walk asks its first tier at once
|
||||
// (issue 973, novox/hq ADR 0407 decision 4): a hotfix is never held by an unrelated delivery.
|
||||
func TestAnUnrelatedBatchWalksBesideAnOpenWalk(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
if ws, _ := walks(t, open); len(ws) != 1 || !ws[0].Open() {
|
||||
t.Fatalf("the first walk is not open: %+v", ws)
|
||||
}
|
||||
heard := t0.Add(100 * time.Second)
|
||||
hear(t, open, lone(repoMerge("two", "c2", heard)), heard)
|
||||
advanceHeld(ctx, open)
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 2 || len(bs) != 0 {
|
||||
t.Fatalf("%d walk(s) and %d batch(es); want the unrelated batch cut beside the open walk", len(ws), len(bs))
|
||||
}
|
||||
for _, w := range ws {
|
||||
if !w.Open() || w.Waiting() || deferred(w) || !askedAny(w) {
|
||||
t.Errorf("%s (%s) is not walking: open %v, deferred %v, asked %v, note %q", w.ID, w.Named(), w.Open(),
|
||||
deferred(w), askedAny(w), w.Note)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A closed batch related to an open walk is queued behind it with the reason, while one that shares nothing is
|
||||
// cut beside it; the walk that started is never superseded (ADR 0276 decision 4, issue 973).
|
||||
func TestARelatedBatchWaitsBehindTheOpenWalk(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
first := func() inventory.Plan { ws, _ := walks(t, open); return ws[len(ws)-1] }()
|
||||
heard := t0.Add(100 * time.Second)
|
||||
hear(t, open, lone(repoMerge("one", "c2", heard)), heard)
|
||||
hear(t, open, lone(repoMerge("three", "c3", heard.Add(time.Second))), heard.Add(time.Second))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 2 || len(bs) != 1 {
|
||||
t.Fatalf("%d walk(s) and %d batch(es); want the unrelated merge walking and the related one queued", len(ws), len(bs))
|
||||
}
|
||||
b := bs[0]
|
||||
words := batchWords(b, heard)
|
||||
if b.State != inventory.PlanQueued || b.Delivery.Batch.Behind != first.ID || b.CommitOf("novox/one") != "c2" ||
|
||||
!strings.Contains(words, "queued behind "+first.ID) || !strings.Contains(words, "both change novox/one") {
|
||||
t.Fatalf("the related batch is %s, behind %q, reading %q", b.State, b.Delivery.Batch.Behind, words)
|
||||
}
|
||||
if got, _ := open.inventory.PlanByID(ctx, first.ID); !got.Open() {
|
||||
t.Fatalf("the started walk was %s by the related merge", got.State)
|
||||
}
|
||||
first.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, heard.Add(2*time.Second))
|
||||
if _, bs := walks(t, open); len(bs) != 0 {
|
||||
t.Fatalf("the queued batch was not cut once the walk it waited for ended: %+v", bs)
|
||||
}
|
||||
}
|
||||
|
||||
// A batch on the controller's own path never walks beside another walk, and nothing walks beside it, whatever
|
||||
// the two share (issue 973): it replaces the controller, so it goes alone.
|
||||
func TestAnOwnPathBatchNeverWalksBeside(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
ws, _ := walks(t, open)
|
||||
first := ws[0]
|
||||
heard := t0.Add(100 * time.Second)
|
||||
hear(t, open, lone(repoMerge("mesh-controller", "k1", heard)), heard)
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 1 || !bs[0].OwnPath() || bs[0].State != inventory.PlanQueued ||
|
||||
bs[0].Delivery.Batch.Behind != first.ID || !strings.Contains(batchWords(bs[0], heard), "own path") {
|
||||
t.Fatalf("an own-path batch beside an open walk: %d walk(s), batches %+v", len(ws), bs)
|
||||
}
|
||||
first.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, heard.Add(time.Second))
|
||||
ws, _ = walks(t, open)
|
||||
if len(ws) != 2 || !ws[0].Open() || ws[0].CommitOf("novox/mesh-controller") != "k1" {
|
||||
t.Fatalf("the own-path batch was not cut once the walk ended: %+v", ws)
|
||||
}
|
||||
own := ws[0]
|
||||
// Unrelated to the own-path walk by every footprint, a merge still waits for it.
|
||||
later := heard.Add(time.Minute)
|
||||
hear(t, open, lone(repoMerge("two", "c2", later)), later)
|
||||
ws, bs = walks(t, open)
|
||||
if len(ws) != 2 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != own.ID {
|
||||
t.Fatalf("a merge beside the own-path walk: %d walk(s), batches %+v", len(ws), bs)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
func TestTheIncidentAConsumerStaysBesideItsDataUntilAPersonPinsIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range storeManifests() {
|
||||
register(t, open, m)
|
||||
}
|
||||
assignAll := func(pairs ...[2]string) {
|
||||
for _, a := range pairs {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
}
|
||||
// The anchor's store and resolver hold the mesh's seats, on record; the laptop runs its own of each.
|
||||
assignAll([2]string{"anchor", "store"}, [2]string{"anchor", "resolver"})
|
||||
for _, seat := range [][2]string{{"mesh-store", "store"}, {"mesh-dns-resolver", "resolver"}} {
|
||||
if err := inv.HoldSeat(ctx, seat[0], catalogue.ScopeMesh, "anchor", seat[1]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
assignAll([2]string{"laptop", "store"}, [2]string{"laptop", "resolver"}, [2]string{"laptop", "network"},
|
||||
[2]string{"laptop", "board"})
|
||||
|
||||
plan, _, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := need(t, plan, "board", "postgres-database"); n.From != "laptop" || n.Module != "store" || !n.KeepsData {
|
||||
t.Fatalf("the consumer was bound to %s/%s (keeps data: %v); its data is beside it", n.From, n.Module, n.KeepsData)
|
||||
}
|
||||
if n := need(t, plan, "network", "wildcard-resolution"); n.From != "anchor" || n.KeepsData {
|
||||
t.Fatalf("the resolver was bound to %s (keeps data: %v); its seat is held on anchor (issue 258)", n.From, n.KeepsData)
|
||||
}
|
||||
|
||||
// Sent, and recorded: only the binding to data.
|
||||
bindings := boundToData(plan, nil)
|
||||
if len(bindings) != 1 || bindings[0].Provider != (catalogue.Chosen{Node: "laptop", Module: "store"}) {
|
||||
t.Fatalf("recorded %+v", bindings)
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, "laptop", bindings); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 {
|
||||
t.Fatalf("a mesh bound where it was sent: %+v, %v", found, err)
|
||||
}
|
||||
|
||||
// The laptop's store taken away: refused, not moved to the anchor's empty one.
|
||||
if err := inv.Unassign(ctx, "laptop", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := planFor(ctx, open, "laptop"); err == nil || !unresolvable(err) ||
|
||||
!strings.Contains(err.Error(), "board on laptop is bound to laptop/store") ||
|
||||
!strings.Contains(err.Error(), "pin laptop postgres-database anchor store") {
|
||||
t.Fatalf("the consumer's store went and it was answered elsewhere: %v", err)
|
||||
}
|
||||
|
||||
// A person pins the anchor's: it moves, said before it is sent, and the record keeps where it was.
|
||||
if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, _, err = planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := need(t, plan, "board", "postgres-database"); n.From != "anchor" {
|
||||
t.Fatalf("pinned to the anchor and bound to %s", n.From)
|
||||
}
|
||||
found, err := probeBindings(ctx, &doctor{open: open})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(found) != 1 || found[0].Kind != kindBindingMoving || found[0].Severity != conditions.Warning ||
|
||||
!strings.Contains(found[0].Summary, "board's postgres-database moves from laptop/store to anchor/store") {
|
||||
t.Fatalf("a pinned move is not said before it is sent: %+v", found)
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
all, err := inv.Bindings(ctx)
|
||||
if err != nil || len(all) != 1 || all[0].MovedFrom != "laptop/store" {
|
||||
t.Fatalf("%+v, %v", all, err)
|
||||
}
|
||||
if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 {
|
||||
t.Fatalf("a move sent is still said: %+v, %v", found, err)
|
||||
}
|
||||
}
|
||||
@@ -39,96 +39,6 @@ func need(t *testing.T, plan catalogue.Resolution, consumer, provision string) c
|
||||
// The incident through the stores: the laptop runs its own store and a consumer of it, the anchor's
|
||||
// store holds the mesh's seat. The consumer stays beside its data, the resolver follows its seat, the
|
||||
// binding is recorded as sent, and a pin — only a pin — moves it, said before the push that carries it.
|
||||
func TestTheIncidentAConsumerStaysBesideItsDataUntilAPersonPinsIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range storeManifests() {
|
||||
register(t, open, m)
|
||||
}
|
||||
assignAll := func(pairs ...[2]string) {
|
||||
for _, a := range pairs {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
}
|
||||
// The anchor's store and resolver hold the mesh's seats, on record; the laptop runs its own of each.
|
||||
assignAll([2]string{"anchor", "store"}, [2]string{"anchor", "resolver"})
|
||||
for _, seat := range [][2]string{{"mesh-store", "store"}, {"mesh-dns-resolver", "resolver"}} {
|
||||
if err := inv.HoldSeat(ctx, seat[0], catalogue.ScopeMesh, "anchor", seat[1]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
assignAll([2]string{"laptop", "store"}, [2]string{"laptop", "resolver"}, [2]string{"laptop", "network"},
|
||||
[2]string{"laptop", "board"})
|
||||
|
||||
plan, _, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := need(t, plan, "board", "postgres-database"); n.From != "laptop" || n.Module != "store" || !n.KeepsData {
|
||||
t.Fatalf("the consumer was bound to %s/%s (keeps data: %v); its data is beside it", n.From, n.Module, n.KeepsData)
|
||||
}
|
||||
if n := need(t, plan, "network", "wildcard-resolution"); n.From != "anchor" || n.KeepsData {
|
||||
t.Fatalf("the resolver was bound to %s (keeps data: %v); its seat is held on anchor (issue 258)", n.From, n.KeepsData)
|
||||
}
|
||||
|
||||
// Sent, and recorded: only the binding to data.
|
||||
bindings := boundToData(plan, nil)
|
||||
if len(bindings) != 1 || bindings[0].Provider != (catalogue.Chosen{Node: "laptop", Module: "store"}) {
|
||||
t.Fatalf("recorded %+v", bindings)
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, "laptop", bindings); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 {
|
||||
t.Fatalf("a mesh bound where it was sent: %+v, %v", found, err)
|
||||
}
|
||||
|
||||
// The laptop's store taken away: refused, not moved to the anchor's empty one.
|
||||
if err := inv.Unassign(ctx, "laptop", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := planFor(ctx, open, "laptop"); err == nil || !unresolvable(err) ||
|
||||
!strings.Contains(err.Error(), "board on laptop is bound to laptop/store") ||
|
||||
!strings.Contains(err.Error(), "pin laptop postgres-database anchor store") {
|
||||
t.Fatalf("the consumer's store went and it was answered elsewhere: %v", err)
|
||||
}
|
||||
|
||||
// A person pins the anchor's: it moves, said before it is sent, and the record keeps where it was.
|
||||
if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, _, err = planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := need(t, plan, "board", "postgres-database"); n.From != "anchor" {
|
||||
t.Fatalf("pinned to the anchor and bound to %s", n.From)
|
||||
}
|
||||
found, err := probeBindings(ctx, &doctor{open: open})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(found) != 1 || found[0].Kind != kindBindingMoving || found[0].Severity != conditions.Warning ||
|
||||
!strings.Contains(found[0].Summary, "board's postgres-database moves from laptop/store to anchor/store") {
|
||||
t.Fatalf("a pinned move is not said before it is sent: %+v", found)
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
all, err := inv.Bindings(ctx)
|
||||
if err != nil || len(all) != 1 || all[0].MovedFrom != "laptop/store" {
|
||||
t.Fatalf("%+v, %v", all, err)
|
||||
}
|
||||
if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 {
|
||||
t.Fatalf("a move sent is still said: %+v, %v", found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// What one machine's resolution says against its record.
|
||||
func TestBindingFindingsSayAKeptMoveAndAMoveNothingAskedFor(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func TestABuildHeardIsRecordedAndRegistered(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
manifest, _ := json.Marshal(map[string]any{"module": "shop", "version": "3"})
|
||||
m, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, link.BuildResult{
|
||||
ID: "b-1", Repository: "http://forge.internal:20000/novox/shop.git", Path: "modules/shop",
|
||||
Ref: "main", On: "anchor", Commit: "abcdef0123", Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/shop"},
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m.Module != "shop" {
|
||||
t.Fatalf("registered %q", m.Module)
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, held := shelf["shop"]; !held {
|
||||
t.Fatal("the module a heard build produced is not in the catalogue")
|
||||
}
|
||||
src, err := open.inventory.SourceOf(ctx, "shop")
|
||||
if err != nil || src.Seat != "git" || src.Repository != "novox/shop" || src.BuiltFrom != "abcdef0123" {
|
||||
t.Fatalf("the source is the seat and the path, never the URL: %+v %v", src, err)
|
||||
}
|
||||
builds, err := open.inventory.Builds(ctx, "shop", 5)
|
||||
if err != nil || len(builds) != 1 || builds[0].ID != "b-1" {
|
||||
t.Fatalf("the build is not recorded once: %v %v", builds, err)
|
||||
}
|
||||
|
||||
named, _ := json.Marshal(map[string]any{"module": "idp", "version": "1", "resources": []any{
|
||||
map[string]any{"id": "server", "type": "container", "image": "x@sha256:aa",
|
||||
"env": map[string]any{"KC_HOSTNAME": "https://login.mesh-one.be"}}}})
|
||||
_, _, err = takeIn(ctx, open.inventory, link.BuildResult{
|
||||
ID: "b-2", Repository: "/r", On: "anchor", Commit: "0123456789", Manifest: named})
|
||||
if err == nil || !strings.Contains(err.Error(), "does not register it") {
|
||||
t.Fatalf("a definition naming an installation was taken in: %v", err)
|
||||
}
|
||||
if shelf, _ := open.inventory.Catalogue(ctx); shelf["idp"].Module != "" {
|
||||
t.Fatal("the refused module was registered anyway")
|
||||
}
|
||||
if builds, _ := open.inventory.Builds(ctx, "idp", 5); len(builds) != 1 {
|
||||
t.Fatalf("the refused build was not recorded: %v", builds)
|
||||
}
|
||||
|
||||
_, _, err = takeIn(ctx, open.inventory, link.BuildResult{ID: "b-3", Repository: "/r", On: "anchor", Failed: "no compiler"})
|
||||
if err == nil || !strings.Contains(err.Error(), "no compiler") {
|
||||
t.Fatalf("a failure is said in the builder's words: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABuildAtACommitKeepsTheBranchTheModuleFollows(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
manifest, _ := json.Marshal(map[string]any{"module": "unifi", "version": "1"})
|
||||
result := func(id, ref, commit string) link.BuildResult {
|
||||
return link.BuildResult{ID: id, Repository: "http://forge.internal:20000/novox/mesh-catalog.git",
|
||||
Path: "modules/unifi", Ref: ref, On: "anchor", Commit: commit, Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, result("b-1", "main", "1111111aaaa"))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, result("b-2", "9c97a8a", "9c97a8a1d2c3")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
src, err := open.inventory.SourceOf(ctx, "unifi")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if src.Ref != "main" || src.BuiltFrom != "9c97a8a1d2c3" {
|
||||
t.Errorf("after a build at a commit the module follows %q, built from %q; want main, 9c97a8a1d2c3", src.Ref, src.BuiltFrom)
|
||||
}
|
||||
|
||||
// One new to the catalogue, first built at a commit, follows the default branch.
|
||||
other, _ := json.Marshal(map[string]any{"module": "letta", "version": "1"})
|
||||
r := result("b-3", "deadbeef", "deadbeefcafe")
|
||||
r.Manifest, r.Path = other, "modules/letta"
|
||||
if _, _, err := takeIn(ctx, open.inventory, r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if src, _ := open.inventory.SourceOf(ctx, "letta"); src.Ref != "" {
|
||||
t.Errorf("a module first built at a commit follows %q, want the default branch", src.Ref)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOlderBuildHeardLaterDoesNotReplaceTheNewer(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
older := time.Date(2026, 10, 3, 21, 33, 45, 0, time.UTC)
|
||||
newer := time.Date(2026, 10, 3, 21, 51, 57, 0, time.UTC)
|
||||
result := func(asked time.Time, image string) link.BuildResult {
|
||||
manifest, _ := json.Marshal(map[string]any{"module": "postgres", "version": image})
|
||||
return link.BuildResult{ID: link.NewBuildID(asked), Repository: "http://forge.internal:20000/novox/mesh-catalog.git",
|
||||
Path: "modules/postgres", Ref: "main", On: "anchor", Commit: "efff5415", Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, result(newer, "4bcd5f73"))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, err := takeIn(ctx, open.inventory, result(older, "0ab07fa9"))
|
||||
if !errors.Is(err, inventory.ErrSuperseded) {
|
||||
t.Fatalf("the older request's outcome was taken in as current: %v", err)
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := shelf["postgres"].Version; got != "4bcd5f73" {
|
||||
t.Errorf("postgres is %q; want the newer request's 4bcd5f73", got)
|
||||
}
|
||||
if builds, _ := open.inventory.Builds(ctx, "postgres", 5); len(builds) != 2 {
|
||||
t.Errorf("the late build was not recorded: %v", builds)
|
||||
}
|
||||
}
|
||||
@@ -435,6 +435,20 @@ func mayRegisterFrom(ctx context.Context, inv *inventory.Inventory, module strin
|
||||
module, sourceWords(built.Repository, built.Seat))
|
||||
}
|
||||
}
|
||||
// **Or the move the operator approved** (novox/hq ADR 0420 §7, issue 1025): a known module built from another
|
||||
// repository by a merge, whose pull request mesh-delivery merged on the operator's Approve of that head.
|
||||
// A verb alone, or a merge with no such warrant, is refused as before.
|
||||
movedWhy := ""
|
||||
if why != "" && !terminal && !isNew {
|
||||
accepted, err := approvedMove(ctx, inv, forms, buildID, built)
|
||||
switch {
|
||||
case err == nil:
|
||||
movedWhy, why = accepted, ""
|
||||
case !errors.Is(err, errNotAMerge):
|
||||
fmt.Printf("%s: %s is built from another repository, and no approved move covers this build: %v\n",
|
||||
buildID, module, err)
|
||||
}
|
||||
}
|
||||
if why != "" && !terminal {
|
||||
return refuse(why)
|
||||
}
|
||||
@@ -471,7 +485,8 @@ func mayRegisterFrom(ctx context.Context, inv *inventory.Inventory, module strin
|
||||
}
|
||||
// The same repository by the forge's id, not only its name: one deleted and made again is another.
|
||||
recorded := map[string]int64{}
|
||||
if !isNew {
|
||||
// A moved module is registered from the repository the operator approved, not the one it was registered from.
|
||||
if !isNew && movedWhy == "" {
|
||||
id, err := inv.SourceIdentity(ctx, module)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
@@ -492,6 +507,10 @@ func mayRegisterFrom(ctx context.Context, inv *inventory.Inventory, module strin
|
||||
owner, name, m, facts.ID, m, id))
|
||||
}
|
||||
}
|
||||
if movedWhy != "" {
|
||||
fmt.Printf("%s: %s moved from %s to %s/%s — registered at the operator's Approve: %s\n", buildID, module,
|
||||
sourceWords(was.Repository, was.Seat), owner, name, movedWhy)
|
||||
}
|
||||
return facts.ID, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,353 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func theCatalogue(t *testing.T) *stores {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
for _, b := range []link.BuildResult{
|
||||
onTrunk("build-sudo", "novox/mesh-catalog", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "1"}),
|
||||
onTrunk("build-host", "novox/mesh-host", "git", "", map[string]any{"module": "mesh-host", "version": "1"}),
|
||||
} {
|
||||
keptAsked(t, open.inventory, b.ID, b.Source.Repository, b.Path, true)
|
||||
if _, _, err := takeIn(ctx, open.inventory, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return open
|
||||
}
|
||||
|
||||
func TestABuildFromAnAgentsRepositoryIsNotRegisteredAsAModuleTheMeshHolds(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
for _, c := range []struct {
|
||||
name, repository, path, module string
|
||||
}{
|
||||
{"its own repository naming sudo", "agent/sudo", "modules/sudo", "sudo"},
|
||||
{"a fork of the catalogue", "agent/mesh-catalog", "modules/sudo", "sudo"},
|
||||
{"a fork of the node-engine", "agent/mesh-host", "", "mesh-host"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
id := "build-" + strings.ReplaceAll(c.repository, "/", "-")
|
||||
keptAsked(t, open.inventory, id, c.repository, c.path, false) // through the build verb
|
||||
evil := onTrunk(id, c.repository, "git", c.path, map[string]any{"module": c.module, "version": "evil"})
|
||||
_, _, err := takeIn(ctx, open.inventory, evil)
|
||||
if !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("a build of %s was taken in as %s: %v", c.repository, c.module, err)
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := shelf[c.module].Version; got != "1" {
|
||||
t.Fatalf("%s is now %q, from %s", c.module, got, c.repository)
|
||||
}
|
||||
if _, found, _ := open.inventory.BuildByID(ctx, id); !found {
|
||||
t.Errorf("the refused build %s is not recorded", id)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestANewModuleFromARepositoryTheCatalogueDoesNotBuildFromIsNotRegistered(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
keptAsked(t, open.inventory, "build-new", "agent/tools", "", false)
|
||||
_, _, err := takeIn(ctx, open.inventory, onTrunk("build-new", "agent/tools", "git", "",
|
||||
map[string]any{"module": "agent-tools", "version": "1"}))
|
||||
if !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("a new module from an agent's repository was taken in: %v", err)
|
||||
}
|
||||
// And one asked of nobody here — an outcome on the bus no request was kept for — the same.
|
||||
_, _, err = takeIn(ctx, open.inventory, onTrunk("build-unasked", "agent/tools", "git", "",
|
||||
map[string]any{"module": "agent-tools", "version": "1"}))
|
||||
if !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("an outcome nobody asked for was taken in: %v", err)
|
||||
}
|
||||
if shelf, _ := open.inventory.Catalogue(ctx); shelf["agent-tools"].Module != "" {
|
||||
t.Fatal("the refused module is in the catalogue")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAtTheTerminalAnotherRepositoryIsRegistered(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
keptAsked(t, open.inventory, "build-moved", "novox/sudo", "", true)
|
||||
if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-moved", "novox/sudo", "git", "",
|
||||
map[string]any{"module": "sudo", "version": "2"})); err != nil {
|
||||
t.Fatalf("a move the operator asked for at the terminal was refused: %v", err)
|
||||
}
|
||||
if src, _ := open.inventory.SourceOf(ctx, "sudo"); src.Repository != "novox/sudo" {
|
||||
t.Fatalf("sudo is built from %q", src.Repository)
|
||||
}
|
||||
if err := open.inventory.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-external",
|
||||
Repository: "http://forge.internal:20000/someone/app.git", For: "build", AtTerminal: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-external", "someone/app", "", "",
|
||||
map[string]any{"module": "app", "version": "1"})); err != nil {
|
||||
t.Fatalf("a new module the operator asked for at the terminal was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestADeliveryFromTheRegisteredRepositoryIsRegistered(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
rebuilt := onTrunk("build-plan", "novox/mesh-catalog", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "2"})
|
||||
if err := open.inventory.RecordBuildRequest(ctx, inventory.BuildRequest{ID: rebuilt.ID,
|
||||
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/sudo", For: "plan"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, rebuilt); err != nil {
|
||||
t.Fatalf("a plan's build of the module's own repository was refused: %v", err)
|
||||
}
|
||||
added := onTrunk("build-merge", "novox/mesh-catalog", "git", "modules/zram", map[string]any{"module": "zram", "version": "1"})
|
||||
if _, _, err := takeIn(ctx, open.inventory, added); err != nil {
|
||||
t.Fatalf("a module a merge added to the catalogue repository was refused: %v", err)
|
||||
}
|
||||
shelf, _ := open.inventory.Catalogue(ctx)
|
||||
if shelf["sudo"].Version != "2" || shelf["zram"].Module == "" {
|
||||
t.Fatalf("not registered: sudo %q, zram %q", shelf["sudo"].Version, shelf["zram"].Module)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAVerbAsksNoBuildOfARepositoryTheCatalogueDoesNotBuildFrom(t *testing.T) {
|
||||
theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
t.Setenv(verbVar, "build")
|
||||
t.Setenv(link.CallerVar, "node-tools.anchor, through the mesh-controller seat")
|
||||
err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, "http://forge.internal:20000/agent/sudo.git")
|
||||
var policy *heldAtTheTerminal
|
||||
if !errors.As(err, &policy) {
|
||||
t.Fatalf("a verb's build of an agent's repository was asked: %v", err)
|
||||
}
|
||||
if err := verbMayAsk(ctx, buildSource{Repository: "novox/mesh-catalog", Seat: "git"},
|
||||
"http://forge.internal:20000/novox/mesh-catalog.git"); err != nil {
|
||||
t.Fatalf("a verb's build of the catalogue repository was refused: %v", err)
|
||||
}
|
||||
t.Setenv(verbVar, "")
|
||||
t.Setenv(link.CallerVar, "")
|
||||
if err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, ""); err != nil {
|
||||
t.Fatalf("the terminal was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARollbackNeverPutsBackABuildFromAnotherRepository(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
fork := onTrunk("build-1791500000000000000", "agent/mesh-catalog", "git", "modules/sudo",
|
||||
map[string]any{"module": "sudo", "version": "evil"})
|
||||
fork.Commit = "c0ffee0123456789" // the commit sudo was registered at
|
||||
keptAsked(t, inv, fork.ID, "agent/mesh-catalog", "modules/sudo", false)
|
||||
if _, _, err := takeIn(ctx, inv, fork); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("the fork's build was taken in: %v", err)
|
||||
}
|
||||
// Asked after the registered build, whenever the test runs: an id naming a fixed moment read as older
|
||||
// than the registered build once the clock passed it (2026-10-10 02:40 UTC), and the test failed on main.
|
||||
failed := onTrunk(fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano()), "novox/mesh-catalog", "git", "modules/sudo",
|
||||
map[string]any{"module": "sudo", "version": "2"})
|
||||
failed.Commit = "badbadbad0123456"
|
||||
if _, _, err := takeIn(ctx, inv, failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, _, err := inv.BuildByID(ctx, failed.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
previous, found, err := inv.PreviousBuild(ctx, "sudo", "c0ffee0123456789", record)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if found && previous.ID == fork.ID {
|
||||
t.Fatalf("a rollback would put back the fork's build %s", previous.ID)
|
||||
}
|
||||
if !found || previous.ID != "build-sudo" {
|
||||
t.Fatalf("a rollback puts back %q (found %v), want the registered build-sudo", previous.ID, found)
|
||||
}
|
||||
}
|
||||
|
||||
func TestATrunkTheForgeDoesNotProtectRegistersNothing(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
theForge.Store("novox/unguarded", forgeFacts{ID: 7, Why: "the rule main lets a person push to main directly"})
|
||||
t.Cleanup(func() { theForge.Delete("novox/unguarded") })
|
||||
first := onTrunk("build-unguarded-1", "novox/unguarded", "git", "", map[string]any{"module": "unguarded", "version": "1"})
|
||||
keptAsked(t, open.inventory, first.ID, "novox/unguarded", "", true)
|
||||
if _, _, err := takeIn(ctx, open.inventory, first); err != nil {
|
||||
t.Fatalf("at the terminal: %v", err)
|
||||
}
|
||||
again := onTrunk("build-unguarded-2", "novox/unguarded", "git", "", map[string]any{"module": "unguarded", "version": "2"})
|
||||
if _, _, err := takeIn(ctx, open.inventory, again); !errors.Is(err, errNotItsSource) ||
|
||||
!strings.Contains(err.Error(), "push to main directly") {
|
||||
t.Fatalf("a rebuild from an unprotected trunk was taken in: %v", err)
|
||||
}
|
||||
theForge.Store("novox/unguarded", errors.New("nothing serves gitea.gitea_api"))
|
||||
if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-unguarded-3", "novox/unguarded", "git", "",
|
||||
map[string]any{"module": "unguarded", "version": "3"})); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("a forge that could not say was read as a protected trunk: %v", err)
|
||||
}
|
||||
if shelf, _ := open.inventory.Catalogue(ctx); shelf["unguarded"].Version != "1" {
|
||||
t.Fatalf("unguarded is %q", shelf["unguarded"].Version)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARepositoryMadeAgainUnderItsNameIsNotTheModulesSource(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
theForge.Store("novox/remade", forgeFacts{ID: 100, Guarded: true})
|
||||
t.Cleanup(func() { theForge.Delete("novox/remade") })
|
||||
first := onTrunk("build-remade-1", "novox/remade", "git", "", map[string]any{"module": "remade", "version": "1"})
|
||||
keptAsked(t, open.inventory, first.ID, "novox/remade", "", true)
|
||||
if _, _, err := takeIn(ctx, open.inventory, first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if id, _ := open.inventory.SourceIdentity(ctx, "remade"); id != 100 {
|
||||
t.Fatalf("the forge's id was not recorded: %d", id)
|
||||
}
|
||||
theForge.Store("novox/remade", forgeFacts{ID: 101, Guarded: true}) // deleted, and made again by an agent
|
||||
if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-remade-2", "novox/remade", "git", "",
|
||||
map[string]any{"module": "remade", "version": "evil"})); !errors.Is(err, errNotItsSource) ||
|
||||
!strings.Contains(err.Error(), "made again") {
|
||||
t.Fatalf("a repository made again under the name was taken in: %v", err)
|
||||
}
|
||||
// And a new module from it, beside the one registered from the first, the same.
|
||||
if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-remade-3", "novox/remade", "git", "modules/other",
|
||||
map[string]any{"module": "other", "version": "1"})); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("a new module from a repository made again was taken in: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestATerminalRequestCoversOnlyWhatItAsked(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-asked", Repository: "novox/app",
|
||||
Seat: "git", Path: "modules/app", For: "build", AtTerminal: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
other := onTrunk("build-asked", "agent/sudo", "git", "modules/app", map[string]any{"module": "sudo", "version": "evil"})
|
||||
if _, _, err := takeIn(ctx, open.inventory, other); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("an outcome of another repository under a terminal request's id was taken in: %v", err)
|
||||
}
|
||||
elsewhere := onTrunk("build-asked", "novox/app", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "evil"})
|
||||
if _, _, err := takeIn(ctx, open.inventory, elsewhere); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("an outcome of another path under a terminal request's id was taken in: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheServingControllerIsNeverTheTerminal(t *testing.T) {
|
||||
t.Setenv(verbVar, "")
|
||||
t.Setenv(link.CallerVar, "")
|
||||
t.Setenv(servedVar, "")
|
||||
if !startedAtTheTerminal() {
|
||||
t.Fatal("a process started by hand is not the terminal")
|
||||
}
|
||||
markServed()
|
||||
if startedAtTheTerminal() {
|
||||
t.Fatal("the serving controller reads as the terminal")
|
||||
}
|
||||
child := exec.Command(os.Args[0], "-test.run=^$")
|
||||
child.Env = os.Environ()
|
||||
if !slices.Contains(child.Env, servedVar+"=1") {
|
||||
t.Fatal("what the serving controller starts does not carry its mark")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnknownIdentityKeepsTheRecordedOne(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.SetSourceIdentity(ctx, "sudo", 100); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetSourceIdentity(ctx, "sudo", 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if id, _ := open.inventory.SourceIdentity(ctx, "sudo"); id != 100 {
|
||||
t.Fatalf("the recorded id became %d", id)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModuleMovesOnTheOperatorsApproveOfTheMergeThatMovedIt(t *testing.T) {
|
||||
const commit = "c0ffee0123456789"
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
change func(*anApprovedMerge)
|
||||
unkept bool
|
||||
request string // who asked: "merge", or "build" through a verb
|
||||
moved bool
|
||||
}{
|
||||
{what: "the operator's Approve of the merged head", request: "merge", moved: true},
|
||||
{what: "no merge mesh-delivery announced", unkept: true, request: "merge"},
|
||||
{what: "a warrant for other heads", request: "merge", change: func(a *anApprovedMerge) {
|
||||
a.merge.Heads = map[string]string{"novox/mesh-host@aaaaaaaaaaaa": "e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5"}
|
||||
}},
|
||||
{what: "a Decline", request: "merge", change: func(a *anApprovedMerge) {
|
||||
a.warrant.Option, a.warrant.Label, a.merge.Option = "decline", "Decline", "decline"
|
||||
}},
|
||||
{what: "a person who is not the operator", request: "merge", change: func(a *anApprovedMerge) { a.warrant.By.Who = "agent" }},
|
||||
{what: "a build asked through a verb", request: "build"},
|
||||
} {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
a := approvedMerge(t)
|
||||
a.merge.Owner, a.merge.Repo, a.merge.Number, a.merge.HeadSHA = "novox", "mesh-host", 7, "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1"
|
||||
if c.change != nil {
|
||||
c.change(&a)
|
||||
}
|
||||
wasKept, wasAsk := keptDeliveryMerge, routerAskOf
|
||||
keptDeliveryMerge = func(_ context.Context, owner, repo string, number int, head string) (deliveryMerge, bool, error) {
|
||||
return a.merge, !c.unkept && owner == "novox" && repo == "mesh-host" && number == 7 && head == a.merge.HeadSHA, nil
|
||||
}
|
||||
routerAskOf = func(_ context.Context, _ *inventory.Inventory, id string) (*asks.Ask, *asks.Warrant, error) {
|
||||
if id != a.merge.AskID {
|
||||
t.Errorf("%s: the router was asked for %q", c.what, id)
|
||||
}
|
||||
return &a.ask, &a.warrant, nil
|
||||
}
|
||||
t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk })
|
||||
event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-host", "base": "main", "number": 7,
|
||||
"head_sha": a.merge.HeadSHA, "merge_commit_sha": commit})
|
||||
if _, err := inv.AddMerge(ctx, inventory.BatchedMerge{Repository: "novox/mesh-host", Branch: "main", Commit: commit,
|
||||
Merged: time.Now(), Heard: time.Now(), Event: event}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id := fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano())
|
||||
if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: id, Repository: "novox/mesh-host", Seat: "git",
|
||||
Path: "modules/sudo", Ref: "main", Commit: commit, For: c.request}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
moved := onTrunk(id, "novox/mesh-host", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "2"})
|
||||
moved.Commit = commit
|
||||
_, _, err := takeIn(ctx, inv, moved)
|
||||
src, _ := inv.SourceOf(ctx, "sudo")
|
||||
switch {
|
||||
case c.moved && err != nil:
|
||||
t.Errorf("%s: the move was refused: %v", c.what, err)
|
||||
case c.moved && src.Repository != "novox/mesh-host":
|
||||
t.Errorf("%s: sudo is built from %q", c.what, src.Repository)
|
||||
case !c.moved && !errors.Is(err, errNotItsSource):
|
||||
t.Errorf("%s: the move was not refused as ADR 0266 has it: %v", c.what, err)
|
||||
case !c.moved && src.Repository != "novox/mesh-catalog":
|
||||
t.Errorf("%s: a refused move changed sudo's source to %q", c.what, src.Repository)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,16 +3,10 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"os"
|
||||
"os/exec"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
@@ -45,141 +39,14 @@ func keptAsked(t *testing.T, inv *inventory.Inventory, id, repository, path stri
|
||||
}
|
||||
|
||||
// theCatalogue is a mesh whose sudo is built from the catalogue repository and whose node-engine from its own.
|
||||
func theCatalogue(t *testing.T) *stores {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
for _, b := range []link.BuildResult{
|
||||
onTrunk("build-sudo", "novox/mesh-catalog", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "1"}),
|
||||
onTrunk("build-host", "novox/mesh-host", "git", "", map[string]any{"module": "mesh-host", "version": "1"}),
|
||||
} {
|
||||
keptAsked(t, open.inventory, b.ID, b.Source.Repository, b.Path, true)
|
||||
if _, _, err := takeIn(ctx, open.inventory, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return open
|
||||
}
|
||||
|
||||
func TestABuildFromAnAgentsRepositoryIsNotRegisteredAsAModuleTheMeshHolds(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
for _, c := range []struct {
|
||||
name, repository, path, module string
|
||||
}{
|
||||
{"its own repository naming sudo", "agent/sudo", "modules/sudo", "sudo"},
|
||||
{"a fork of the catalogue", "agent/mesh-catalog", "modules/sudo", "sudo"},
|
||||
{"a fork of the node-engine", "agent/mesh-host", "", "mesh-host"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
id := "build-" + strings.ReplaceAll(c.repository, "/", "-")
|
||||
keptAsked(t, open.inventory, id, c.repository, c.path, false) // through the build verb
|
||||
evil := onTrunk(id, c.repository, "git", c.path, map[string]any{"module": c.module, "version": "evil"})
|
||||
_, _, err := takeIn(ctx, open.inventory, evil)
|
||||
if !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("a build of %s was taken in as %s: %v", c.repository, c.module, err)
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := shelf[c.module].Version; got != "1" {
|
||||
t.Fatalf("%s is now %q, from %s", c.module, got, c.repository)
|
||||
}
|
||||
if _, found, _ := open.inventory.BuildByID(ctx, id); !found {
|
||||
t.Errorf("the refused build %s is not recorded", id)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestANewModuleFromARepositoryTheCatalogueDoesNotBuildFromIsNotRegistered(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
keptAsked(t, open.inventory, "build-new", "agent/tools", "", false)
|
||||
_, _, err := takeIn(ctx, open.inventory, onTrunk("build-new", "agent/tools", "git", "",
|
||||
map[string]any{"module": "agent-tools", "version": "1"}))
|
||||
if !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("a new module from an agent's repository was taken in: %v", err)
|
||||
}
|
||||
// And one asked of nobody here — an outcome on the bus no request was kept for — the same.
|
||||
_, _, err = takeIn(ctx, open.inventory, onTrunk("build-unasked", "agent/tools", "git", "",
|
||||
map[string]any{"module": "agent-tools", "version": "1"}))
|
||||
if !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("an outcome nobody asked for was taken in: %v", err)
|
||||
}
|
||||
if shelf, _ := open.inventory.Catalogue(ctx); shelf["agent-tools"].Module != "" {
|
||||
t.Fatal("the refused module is in the catalogue")
|
||||
}
|
||||
}
|
||||
|
||||
// The operator at the terminal may still move a module, or add one from a new repository.
|
||||
func TestAtTheTerminalAnotherRepositoryIsRegistered(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
keptAsked(t, open.inventory, "build-moved", "novox/sudo", "", true)
|
||||
if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-moved", "novox/sudo", "git", "",
|
||||
map[string]any{"module": "sudo", "version": "2"})); err != nil {
|
||||
t.Fatalf("a move the operator asked for at the terminal was refused: %v", err)
|
||||
}
|
||||
if src, _ := open.inventory.SourceOf(ctx, "sudo"); src.Repository != "novox/sudo" {
|
||||
t.Fatalf("sudo is built from %q", src.Repository)
|
||||
}
|
||||
if err := open.inventory.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-external",
|
||||
Repository: "http://forge.internal:20000/someone/app.git", For: "build", AtTerminal: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-external", "someone/app", "", "",
|
||||
map[string]any{"module": "app", "version": "1"})); err != nil {
|
||||
t.Fatalf("a new module the operator asked for at the terminal was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The delivery's flow is untouched: a merge's rebuild of a module from its own repository, and a merge adding
|
||||
// a module to a repository the catalogue builds from (novox/hq issue 300), are registered with no terminal.
|
||||
func TestADeliveryFromTheRegisteredRepositoryIsRegistered(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
rebuilt := onTrunk("build-plan", "novox/mesh-catalog", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "2"})
|
||||
if err := open.inventory.RecordBuildRequest(ctx, inventory.BuildRequest{ID: rebuilt.ID,
|
||||
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/sudo", For: "plan"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, rebuilt); err != nil {
|
||||
t.Fatalf("a plan's build of the module's own repository was refused: %v", err)
|
||||
}
|
||||
added := onTrunk("build-merge", "novox/mesh-catalog", "git", "modules/zram", map[string]any{"module": "zram", "version": "1"})
|
||||
if _, _, err := takeIn(ctx, open.inventory, added); err != nil {
|
||||
t.Fatalf("a module a merge added to the catalogue repository was refused: %v", err)
|
||||
}
|
||||
shelf, _ := open.inventory.Catalogue(ctx)
|
||||
if shelf["sudo"].Version != "2" || shelf["zram"].Module == "" {
|
||||
t.Fatalf("not registered: sudo %q, zram %q", shelf["sudo"].Version, shelf["zram"].Module)
|
||||
}
|
||||
}
|
||||
|
||||
// Through a verb, a build of a repository the catalogue builds nothing from is not even asked: the build node
|
||||
// would run what the agent wrote.
|
||||
func TestAVerbAsksNoBuildOfARepositoryTheCatalogueDoesNotBuildFrom(t *testing.T) {
|
||||
theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
t.Setenv(verbVar, "build")
|
||||
t.Setenv(link.CallerVar, "node-tools.anchor, through the mesh-controller seat")
|
||||
err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, "http://forge.internal:20000/agent/sudo.git")
|
||||
var policy *heldAtTheTerminal
|
||||
if !errors.As(err, &policy) {
|
||||
t.Fatalf("a verb's build of an agent's repository was asked: %v", err)
|
||||
}
|
||||
if err := verbMayAsk(ctx, buildSource{Repository: "novox/mesh-catalog", Seat: "git"},
|
||||
"http://forge.internal:20000/novox/mesh-catalog.git"); err != nil {
|
||||
t.Fatalf("a verb's build of the catalogue repository was refused: %v", err)
|
||||
}
|
||||
t.Setenv(verbVar, "")
|
||||
t.Setenv(link.CallerVar, "")
|
||||
if err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, ""); err != nil {
|
||||
t.Fatalf("the terminal was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// asTheOperator keeps a build as asked at the controller's terminal, as the operator's first build of a module
|
||||
// from a repository the catalogue does not yet build from is (novox/hq ADR 0266), and hands it back.
|
||||
@@ -199,40 +66,6 @@ func asTheOperator(t *testing.T, inv *inventory.Inventory, b link.BuildResult) l
|
||||
// A rollback puts back only a build of the module's own repository: an agent's build of the module's name,
|
||||
// recorded and refused, at the very commit the machine ran before (a fork carries it), is never registered by
|
||||
// the back door of a failed gate.
|
||||
func TestARollbackNeverPutsBackABuildFromAnotherRepository(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
fork := onTrunk("build-1791500000000000000", "agent/mesh-catalog", "git", "modules/sudo",
|
||||
map[string]any{"module": "sudo", "version": "evil"})
|
||||
fork.Commit = "c0ffee0123456789" // the commit sudo was registered at
|
||||
keptAsked(t, inv, fork.ID, "agent/mesh-catalog", "modules/sudo", false)
|
||||
if _, _, err := takeIn(ctx, inv, fork); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("the fork's build was taken in: %v", err)
|
||||
}
|
||||
// Asked after the registered build, whenever the test runs: an id naming a fixed moment read as older
|
||||
// than the registered build once the clock passed it (2026-10-10 02:40 UTC), and the test failed on main.
|
||||
failed := onTrunk(fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano()), "novox/mesh-catalog", "git", "modules/sudo",
|
||||
map[string]any{"module": "sudo", "version": "2"})
|
||||
failed.Commit = "badbadbad0123456"
|
||||
if _, _, err := takeIn(ctx, inv, failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, _, err := inv.BuildByID(ctx, failed.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
previous, found, err := inv.PreviousBuild(ctx, "sudo", "c0ffee0123456789", record)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if found && previous.ID == fork.ID {
|
||||
t.Fatalf("a rollback would put back the fork's build %s", previous.ID)
|
||||
}
|
||||
if !found || previous.ID != "build-sudo" {
|
||||
t.Fatalf("a rollback puts back %q (found %v), want the registered build-sudo", previous.ID, found)
|
||||
}
|
||||
}
|
||||
|
||||
// theForge is what the forge says in a test, by owner/name: a repository not named here is protected as a
|
||||
// trunk must be, with an id of its own.
|
||||
@@ -256,97 +89,15 @@ func init() {
|
||||
|
||||
// A module's trunk the forge does not protect — direct pushes, no required status — or a forge that cannot say,
|
||||
// registers nothing through a verb: the trunk rule means nothing on a branch anyone pushes to.
|
||||
func TestATrunkTheForgeDoesNotProtectRegistersNothing(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
theForge.Store("novox/unguarded", forgeFacts{ID: 7, Why: "the rule main lets a person push to main directly"})
|
||||
t.Cleanup(func() { theForge.Delete("novox/unguarded") })
|
||||
first := onTrunk("build-unguarded-1", "novox/unguarded", "git", "", map[string]any{"module": "unguarded", "version": "1"})
|
||||
keptAsked(t, open.inventory, first.ID, "novox/unguarded", "", true)
|
||||
if _, _, err := takeIn(ctx, open.inventory, first); err != nil {
|
||||
t.Fatalf("at the terminal: %v", err)
|
||||
}
|
||||
again := onTrunk("build-unguarded-2", "novox/unguarded", "git", "", map[string]any{"module": "unguarded", "version": "2"})
|
||||
if _, _, err := takeIn(ctx, open.inventory, again); !errors.Is(err, errNotItsSource) ||
|
||||
!strings.Contains(err.Error(), "push to main directly") {
|
||||
t.Fatalf("a rebuild from an unprotected trunk was taken in: %v", err)
|
||||
}
|
||||
theForge.Store("novox/unguarded", errors.New("nothing serves gitea.gitea_api"))
|
||||
if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-unguarded-3", "novox/unguarded", "git", "",
|
||||
map[string]any{"module": "unguarded", "version": "3"})); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("a forge that could not say was read as a protected trunk: %v", err)
|
||||
}
|
||||
if shelf, _ := open.inventory.Catalogue(ctx); shelf["unguarded"].Version != "1" {
|
||||
t.Fatalf("unguarded is %q", shelf["unguarded"].Version)
|
||||
}
|
||||
}
|
||||
|
||||
// A repository deleted and made again under the module's repository's name is another repository: the forge's
|
||||
// id, recorded at registration, tells them apart.
|
||||
func TestARepositoryMadeAgainUnderItsNameIsNotTheModulesSource(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
theForge.Store("novox/remade", forgeFacts{ID: 100, Guarded: true})
|
||||
t.Cleanup(func() { theForge.Delete("novox/remade") })
|
||||
first := onTrunk("build-remade-1", "novox/remade", "git", "", map[string]any{"module": "remade", "version": "1"})
|
||||
keptAsked(t, open.inventory, first.ID, "novox/remade", "", true)
|
||||
if _, _, err := takeIn(ctx, open.inventory, first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if id, _ := open.inventory.SourceIdentity(ctx, "remade"); id != 100 {
|
||||
t.Fatalf("the forge's id was not recorded: %d", id)
|
||||
}
|
||||
theForge.Store("novox/remade", forgeFacts{ID: 101, Guarded: true}) // deleted, and made again by an agent
|
||||
if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-remade-2", "novox/remade", "git", "",
|
||||
map[string]any{"module": "remade", "version": "evil"})); !errors.Is(err, errNotItsSource) ||
|
||||
!strings.Contains(err.Error(), "made again") {
|
||||
t.Fatalf("a repository made again under the name was taken in: %v", err)
|
||||
}
|
||||
// And a new module from it, beside the one registered from the first, the same.
|
||||
if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-remade-3", "novox/remade", "git", "modules/other",
|
||||
map[string]any{"module": "other", "version": "1"})); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("a new module from a repository made again was taken in: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The terminal's mark is the operator's for the repository and path they asked: an outcome of another, under that
|
||||
// build's id, is not theirs.
|
||||
func TestATerminalRequestCoversOnlyWhatItAsked(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-asked", Repository: "novox/app",
|
||||
Seat: "git", Path: "modules/app", For: "build", AtTerminal: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
other := onTrunk("build-asked", "agent/sudo", "git", "modules/app", map[string]any{"module": "sudo", "version": "evil"})
|
||||
if _, _, err := takeIn(ctx, open.inventory, other); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("an outcome of another repository under a terminal request's id was taken in: %v", err)
|
||||
}
|
||||
elsewhere := onTrunk("build-asked", "novox/app", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "evil"})
|
||||
if _, _, err := takeIn(ctx, open.inventory, elsewhere); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("an outcome of another path under a terminal request's id was taken in: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The serving controller, and everything it starts, is never the terminal: a build asked in it reads as asked
|
||||
// through the mesh even when no verb and no caller is named.
|
||||
func TestTheServingControllerIsNeverTheTerminal(t *testing.T) {
|
||||
t.Setenv(verbVar, "")
|
||||
t.Setenv(link.CallerVar, "")
|
||||
t.Setenv(servedVar, "")
|
||||
if !startedAtTheTerminal() {
|
||||
t.Fatal("a process started by hand is not the terminal")
|
||||
}
|
||||
markServed()
|
||||
if startedAtTheTerminal() {
|
||||
t.Fatal("the serving controller reads as the terminal")
|
||||
}
|
||||
child := exec.Command(os.Args[0], "-test.run=^$")
|
||||
child.Env = os.Environ()
|
||||
if !slices.Contains(child.Env, servedVar+"=1") {
|
||||
t.Fatal("what the serving controller starts does not carry its mark")
|
||||
}
|
||||
}
|
||||
|
||||
// The forge applies one rule to a branch: the rule named for it, else the first glob that covers it. A stronger
|
||||
// rule later in the list is not what guards the branch, and is not read as if it were (the confirmation review).
|
||||
@@ -369,16 +120,9 @@ func TestTheRuleJudgedIsTheOneTheForgeApplies(t *testing.T) {
|
||||
}
|
||||
|
||||
// An id the forge could not give keeps the id already recorded.
|
||||
func TestAnUnknownIdentityKeepsTheRecordedOne(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.SetSourceIdentity(ctx, "sudo", 100); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetSourceIdentity(ctx, "sudo", 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if id, _ := open.inventory.SourceIdentity(ctx, "sudo"); id != 100 {
|
||||
t.Fatalf("the recorded id became %d", id)
|
||||
}
|
||||
}
|
||||
|
||||
// **The operator's Approve of a reviewed group registers a moved module from its new repository** (novox/hq ADR 0420 §7,
|
||||
// issue 1025). sudo is built from the catalogue; a merge into the repository of node-engine's module, announced by
|
||||
// mesh-delivery with the ask it merged on, brings its build from there. Accepted with the operator's Approve of the
|
||||
// merged head; refused with no kept merge, a warrant for other heads, a Decline, a person who is not the operator, and
|
||||
// through a verb.
|
||||
|
||||
@@ -1,13 +1,9 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
@@ -15,126 +11,12 @@ import (
|
||||
// the module registered with its source as the seat and path when the request said so — never the
|
||||
// URL. A definition naming an installation is recorded and not registered; a failure is recorded
|
||||
// and said.
|
||||
func TestABuildHeardIsRecordedAndRegistered(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
manifest, _ := json.Marshal(map[string]any{"module": "shop", "version": "3"})
|
||||
m, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, link.BuildResult{
|
||||
ID: "b-1", Repository: "http://forge.internal:20000/novox/shop.git", Path: "modules/shop",
|
||||
Ref: "main", On: "anchor", Commit: "abcdef0123", Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/shop"},
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m.Module != "shop" {
|
||||
t.Fatalf("registered %q", m.Module)
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, held := shelf["shop"]; !held {
|
||||
t.Fatal("the module a heard build produced is not in the catalogue")
|
||||
}
|
||||
src, err := open.inventory.SourceOf(ctx, "shop")
|
||||
if err != nil || src.Seat != "git" || src.Repository != "novox/shop" || src.BuiltFrom != "abcdef0123" {
|
||||
t.Fatalf("the source is the seat and the path, never the URL: %+v %v", src, err)
|
||||
}
|
||||
builds, err := open.inventory.Builds(ctx, "shop", 5)
|
||||
if err != nil || len(builds) != 1 || builds[0].ID != "b-1" {
|
||||
t.Fatalf("the build is not recorded once: %v %v", builds, err)
|
||||
}
|
||||
|
||||
named, _ := json.Marshal(map[string]any{"module": "idp", "version": "1", "resources": []any{
|
||||
map[string]any{"id": "server", "type": "container", "image": "x@sha256:aa",
|
||||
"env": map[string]any{"KC_HOSTNAME": "https://login.mesh-one.be"}}}})
|
||||
_, _, err = takeIn(ctx, open.inventory, link.BuildResult{
|
||||
ID: "b-2", Repository: "/r", On: "anchor", Commit: "0123456789", Manifest: named})
|
||||
if err == nil || !strings.Contains(err.Error(), "does not register it") {
|
||||
t.Fatalf("a definition naming an installation was taken in: %v", err)
|
||||
}
|
||||
if shelf, _ := open.inventory.Catalogue(ctx); shelf["idp"].Module != "" {
|
||||
t.Fatal("the refused module was registered anyway")
|
||||
}
|
||||
if builds, _ := open.inventory.Builds(ctx, "idp", 5); len(builds) != 1 {
|
||||
t.Fatalf("the refused build was not recorded: %v", builds)
|
||||
}
|
||||
|
||||
_, _, err = takeIn(ctx, open.inventory, link.BuildResult{ID: "b-3", Repository: "/r", On: "anchor", Failed: "no compiler"})
|
||||
if err == nil || !strings.Contains(err.Error(), "no compiler") {
|
||||
t.Fatalf("a failure is said in the builder's words: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq 04-ISSUES/215: a build asked at a commit is recorded as built from that commit, and the
|
||||
// module keeps following the branch it followed — a new one, the default branch.
|
||||
func TestABuildAtACommitKeepsTheBranchTheModuleFollows(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
manifest, _ := json.Marshal(map[string]any{"module": "unifi", "version": "1"})
|
||||
result := func(id, ref, commit string) link.BuildResult {
|
||||
return link.BuildResult{ID: id, Repository: "http://forge.internal:20000/novox/mesh-catalog.git",
|
||||
Path: "modules/unifi", Ref: ref, On: "anchor", Commit: commit, Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, result("b-1", "main", "1111111aaaa"))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, result("b-2", "9c97a8a", "9c97a8a1d2c3")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
src, err := open.inventory.SourceOf(ctx, "unifi")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if src.Ref != "main" || src.BuiltFrom != "9c97a8a1d2c3" {
|
||||
t.Errorf("after a build at a commit the module follows %q, built from %q; want main, 9c97a8a1d2c3", src.Ref, src.BuiltFrom)
|
||||
}
|
||||
|
||||
// One new to the catalogue, first built at a commit, follows the default branch.
|
||||
other, _ := json.Marshal(map[string]any{"module": "letta", "version": "1"})
|
||||
r := result("b-3", "deadbeef", "deadbeefcafe")
|
||||
r.Manifest, r.Path = other, "modules/letta"
|
||||
if _, _, err := takeIn(ctx, open.inventory, r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if src, _ := open.inventory.SourceOf(ctx, "letta"); src.Ref != "" {
|
||||
t.Errorf("a module first built at a commit follows %q, want the default branch", src.Ref)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq 04-ISSUES/219: an older request heard after a newer one is recorded and not registered,
|
||||
// so a push sends what the newer request built.
|
||||
func TestAnOlderBuildHeardLaterDoesNotReplaceTheNewer(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
older := time.Date(2026, 10, 3, 21, 33, 45, 0, time.UTC)
|
||||
newer := time.Date(2026, 10, 3, 21, 51, 57, 0, time.UTC)
|
||||
result := func(asked time.Time, image string) link.BuildResult {
|
||||
manifest, _ := json.Marshal(map[string]any{"module": "postgres", "version": image})
|
||||
return link.BuildResult{ID: link.NewBuildID(asked), Repository: "http://forge.internal:20000/novox/mesh-catalog.git",
|
||||
Path: "modules/postgres", Ref: "main", On: "anchor", Commit: "efff5415", Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, result(newer, "4bcd5f73"))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, err := takeIn(ctx, open.inventory, result(older, "0ab07fa9"))
|
||||
if !errors.Is(err, inventory.ErrSuperseded) {
|
||||
t.Fatalf("the older request's outcome was taken in as current: %v", err)
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := shelf["postgres"].Version; got != "4bcd5f73" {
|
||||
t.Errorf("postgres is %q; want the newer request's 4bcd5f73", got)
|
||||
}
|
||||
if builds, _ := open.inventory.Builds(ctx, "postgres", 5); len(builds) != 2 {
|
||||
t.Errorf("the late build was not recorded: %v", builds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABuildIDSaysWhenItWasAsked(t *testing.T) {
|
||||
at := time.Date(2026, 10, 3, 21, 51, 57, 392539762, time.UTC)
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
func TestASendHeldForTheBusStepIsFoundFromTheWalksItHolds(t *testing.T) {
|
||||
now := time.Date(2026, 10, 8, 18, 30, 0, 0, time.UTC)
|
||||
b := aBusOnAnchor()
|
||||
walk := func(id, repo, note string, updated time.Time) inventory.Plan {
|
||||
return inventory.Plan{ID: id, Repository: repo, Commit: "c0ffee001122", State: inventory.PlanRolling, Tier: 0,
|
||||
Tiers: [][]string{{"mesh-host"}}, Modules: map[string]*inventory.PlanModule{"mesh-host": {}}, Note: note,
|
||||
Updated: updated}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
bus busPending
|
||||
plans []inventory.Plan
|
||||
first map[string]time.Time
|
||||
want []string
|
||||
since time.Time
|
||||
}{
|
||||
{"a walk refused for the bus", b,
|
||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now.Add(-27*time.Minute))}, nil,
|
||||
[]string{"plan-1"}, now.Add(-27 * time.Minute)},
|
||||
{"refused earlier than its last save, as this controller saw it", b,
|
||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now.Add(-5*time.Minute))},
|
||||
map[string]time.Time{"plan-1": now.Add(-28 * time.Minute)}, []string{"plan-1"}, now.Add(-28 * time.Minute)},
|
||||
{"a walk refused for another reason", b,
|
||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", "tier 0: the build seat is paused — tried again", now)}, nil,
|
||||
nil, time.Time{}},
|
||||
{"refused for an older bus build than the one held now", func() busPending { o := b; o.to = "99999999cccc"; return o }(),
|
||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)}, nil, nil, time.Time{}},
|
||||
{"the bus already runs the build held: its step started", func() busPending {
|
||||
o := aBusOnAnchor()
|
||||
o.from = map[string]string{"anchor": o.to}
|
||||
return o
|
||||
}(), []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)}, nil, nil, time.Time{}},
|
||||
{"a walk waiting for its delivery's word asks no send", b, func() []inventory.Plan {
|
||||
p := walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)
|
||||
p.Delivery = &inventory.PlanDelivery{Awaits: "mesh-delivery"}
|
||||
return []inventory.Plan{p}
|
||||
}(), nil, nil, time.Time{}},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
got := busWaitsOf(c.plans, c.bus, func(id string) time.Time { return c.first[id] })
|
||||
var ids []string
|
||||
for _, w := range got.waits {
|
||||
ids = append(ids, w.plan)
|
||||
}
|
||||
if strings.Join(ids, ",") != strings.Join(c.want, ",") {
|
||||
t.Fatalf("held for the bus: %v, want %v", ids, c.want)
|
||||
}
|
||||
if len(c.want) > 0 {
|
||||
if !got.waits[0].since.Equal(c.since) {
|
||||
t.Errorf("waiting since %s, want %s", got.waits[0].since, c.since)
|
||||
}
|
||||
if strings.Join(got.machines, ",") != "anchor" || got.to != b.to || got.module != "nats" {
|
||||
t.Errorf("behind %+v", got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq issue 336: a send held because it would replace the bus outside its planned step waits for a
|
||||
@@ -27,65 +26,6 @@ func refusedNote(b busPending, tier int) string {
|
||||
return "tier " + string(rune('0'+tier)) + ": " + errBusWaits.Error() + ": " + held + " — tried again"
|
||||
}
|
||||
|
||||
func TestASendHeldForTheBusStepIsFoundFromTheWalksItHolds(t *testing.T) {
|
||||
now := time.Date(2026, 10, 8, 18, 30, 0, 0, time.UTC)
|
||||
b := aBusOnAnchor()
|
||||
walk := func(id, repo, note string, updated time.Time) inventory.Plan {
|
||||
return inventory.Plan{ID: id, Repository: repo, Commit: "c0ffee001122", State: inventory.PlanRolling, Tier: 0,
|
||||
Tiers: [][]string{{"mesh-host"}}, Modules: map[string]*inventory.PlanModule{"mesh-host": {}}, Note: note,
|
||||
Updated: updated}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
bus busPending
|
||||
plans []inventory.Plan
|
||||
first map[string]time.Time
|
||||
want []string
|
||||
since time.Time
|
||||
}{
|
||||
{"a walk refused for the bus", b,
|
||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now.Add(-27*time.Minute))}, nil,
|
||||
[]string{"plan-1"}, now.Add(-27 * time.Minute)},
|
||||
{"refused earlier than its last save, as this controller saw it", b,
|
||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now.Add(-5*time.Minute))},
|
||||
map[string]time.Time{"plan-1": now.Add(-28 * time.Minute)}, []string{"plan-1"}, now.Add(-28 * time.Minute)},
|
||||
{"a walk refused for another reason", b,
|
||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", "tier 0: the build seat is paused — tried again", now)}, nil,
|
||||
nil, time.Time{}},
|
||||
{"refused for an older bus build than the one held now", func() busPending { o := b; o.to = "99999999cccc"; return o }(),
|
||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)}, nil, nil, time.Time{}},
|
||||
{"the bus already runs the build held: its step started", func() busPending {
|
||||
o := aBusOnAnchor()
|
||||
o.from = map[string]string{"anchor": o.to}
|
||||
return o
|
||||
}(), []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)}, nil, nil, time.Time{}},
|
||||
{"a walk waiting for its delivery's word asks no send", b, func() []inventory.Plan {
|
||||
p := walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)
|
||||
p.Delivery = &inventory.PlanDelivery{Awaits: "mesh-delivery"}
|
||||
return []inventory.Plan{p}
|
||||
}(), nil, nil, time.Time{}},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
got := busWaitsOf(c.plans, c.bus, func(id string) time.Time { return c.first[id] })
|
||||
var ids []string
|
||||
for _, w := range got.waits {
|
||||
ids = append(ids, w.plan)
|
||||
}
|
||||
if strings.Join(ids, ",") != strings.Join(c.want, ",") {
|
||||
t.Fatalf("held for the bus: %v, want %v", ids, c.want)
|
||||
}
|
||||
if len(c.want) > 0 {
|
||||
if !got.waits[0].since.Equal(c.since) {
|
||||
t.Errorf("waiting since %s, want %s", got.waits[0].since, c.since)
|
||||
}
|
||||
if strings.Join(got.machines, ",") != "anchor" || got.to != b.to || got.module != "nats" {
|
||||
t.Errorf("behind %+v", got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// **Said at once, not as lateness, and cleared when the step starts**: the walk held only for the bus raises
|
||||
// the bus's condition on the first tick, before any tier bound, naming what waits, the bus build from and to,
|
||||
// since when and `bus upgrade`, for the operator; S3 says nothing of it even past its bound; and the
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCallsIsServedAndSaysWhatItKeeps(t *testing.T) {
|
||||
handlers, behind, err := seatToolHandlers()
|
||||
if err != nil || len(behind) != 0 {
|
||||
t.Fatalf("%v %v", behind, err)
|
||||
}
|
||||
calls, ok := handlers["calls"]
|
||||
if !ok {
|
||||
t.Fatal("calls is not served")
|
||||
}
|
||||
if _, err := calls(context.Background(), json.RawMessage(`{"node":"anchor"}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), `"node"`) {
|
||||
t.Errorf("calls took an argument it does not declare: %v", err)
|
||||
}
|
||||
if _, err := calls(context.Background(), json.RawMessage(`{"call":"call-0-0"}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), "not across a restart") {
|
||||
t.Errorf("an unknown call was not said plainly: %v", err)
|
||||
}
|
||||
got, err := calls(context.Background(), json.RawMessage(`{}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, listed := got.(map[string]any)["calls"]; !listed {
|
||||
t.Errorf("calls answered %v", got)
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,6 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -52,28 +49,3 @@ func TestAPushAnswersBeforeItSends(t *testing.T) {
|
||||
}
|
||||
|
||||
// `calls` is served, takes a call's id and nothing else, and says plainly when it holds no such call.
|
||||
func TestCallsIsServedAndSaysWhatItKeeps(t *testing.T) {
|
||||
handlers, behind, err := seatToolHandlers()
|
||||
if err != nil || len(behind) != 0 {
|
||||
t.Fatalf("%v %v", behind, err)
|
||||
}
|
||||
calls, ok := handlers["calls"]
|
||||
if !ok {
|
||||
t.Fatal("calls is not served")
|
||||
}
|
||||
if _, err := calls(context.Background(), json.RawMessage(`{"node":"anchor"}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), `"node"`) {
|
||||
t.Errorf("calls took an argument it does not declare: %v", err)
|
||||
}
|
||||
if _, err := calls(context.Background(), json.RawMessage(`{"call":"call-0-0"}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), "not across a restart") {
|
||||
t.Errorf("an unknown call was not said plainly: %v", err)
|
||||
}
|
||||
got, err := calls(context.Background(), json.RawMessage(`{}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, listed := got.(map[string]any)["calls"]; !listed {
|
||||
t.Errorf("calls answered %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// moduleCheck judges manifests where they are written, with no mesh (novox/hq ADR 0037, issue 148).
|
||||
@@ -114,6 +115,20 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
failed += len(wrong)
|
||||
faulted[m.Module] = true
|
||||
}
|
||||
// A module lives whole in one repository (novox/hq ADR 0420 §1, issue 1019): when the check is told the
|
||||
// repository it runs for (MESH_CHECK_REPOSITORY, set by the build's check), an artifact building from
|
||||
// another is refused, as the merge gate refuses it. A run by hand names no repository and judges none.
|
||||
if repo := os.Getenv("MESH_CHECK_REPOSITORY"); repo != "" && m.Build != nil {
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Context != nil && a.Context.Repository != "" && !sameRepository(a.Context.Repository,
|
||||
link.SourceMoved{Owner: ownerOf(repo), Repo: repoOf(repo)}) {
|
||||
fmt.Fprintf(out, "%s: %s builds artifact %s from %s: a module lives whole in one repository "+
|
||||
"(novox/hq ADR 0420 §1)\n", path, m.Module, a.Name, a.Context.Repository)
|
||||
failed++
|
||||
faulted[m.Module] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if named := catalogue.InstallationProblems(m); len(named) > 0 {
|
||||
for _, p := range named {
|
||||
fmt.Fprintf(out, "%s: %s\n", path, p)
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func TestOnlyAWalksAskTakesAChecksBuildAndADryRunNever(t *testing.T) {
|
||||
p := &inventory.Plan{Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{
|
||||
{Repository: "novox/shop", Number: 7}, {Repository: "http://forge.invalid/novox/web.git", Number: 3},
|
||||
{Repository: "novox/shop", Number: 7}, {Repository: "novox/shop"}}}}
|
||||
asked := reuseAskOf(p)
|
||||
if want := []link.PullRef{{Owner: "novox", Repo: "shop", Number: 7}, {Owner: "novox", Repo: "web", Number: 3}}; !reflect.DeepEqual(asked.Pulls, want) {
|
||||
t.Fatalf("the walk names %+v", asked.Pulls)
|
||||
}
|
||||
walk := withReuse(t.Context(), asked)
|
||||
if reuseAsked(walk, false) != asked {
|
||||
t.Fatal("a walk's ask does not carry its leave")
|
||||
}
|
||||
if reuseAsked(walk, true) != nil {
|
||||
t.Fatal("a dry run carries a leave to take a check's build")
|
||||
}
|
||||
if reuseAsked(t.Context(), false) != nil {
|
||||
t.Fatal("an ask that is not a walk's carries a leave")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := g.open.inventory
|
||||
gateEvery = 0
|
||||
|
||||
digest := "sha256:" + strings.Repeat("c", 64)
|
||||
reused := func(id, reference string) link.BuildResult {
|
||||
manifest := mustJSON(t, catalogue.Manifest{Module: "app", Version: "c3", Resources: []map[string]any{
|
||||
{"id": "svc", "type": "container", "name": "app", "image": reference}}})
|
||||
return link.BuildResult{ID: id, Repository: "novox/mesh-catalog", Path: "modules/app", Ref: "main", On: "anchor",
|
||||
Commit: "c3", Manifest: manifest, Trunk: "main", OnTrunk: true, Branches: []string{"main"},
|
||||
Made: []link.MadeArtifact{{Name: "server", Kind: "image", Reference: reference}},
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"},
|
||||
Reused: "reused from the check of novox/mesh-catalog#9 at head abcdefabcdef (merged into trunk 123456123456): server " + digest,
|
||||
ReusedFrom: &link.ReusedFrom{Pull: link.PullRef{Owner: "novox", Repo: "mesh-catalog", Number: 9},
|
||||
Head: "abcdefabcdef", Trunk: "123456123456", Key: "r1:" + strings.Repeat("1", 64)}}
|
||||
}
|
||||
if _, _, err := takeIn(ctx, inv, reused(link.NewBuildID(time.Now()), "store.internal:5000/scratch/app/server@"+digest)); err == nil ||
|
||||
!strings.Contains(err.Error(), "scratch") {
|
||||
t.Fatalf("a build in the scratch namespace was registered: %v", err)
|
||||
}
|
||||
id := link.NewBuildID(time.Now().Add(time.Second))
|
||||
result := reused(id, "store.internal:5000/app/server@"+digest)
|
||||
if _, _, err := takeIn(ctx, inv, result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil || current["app"].Commit != "c3" {
|
||||
t.Fatalf("the reused build is not what app is registered at: %+v %v", current["app"], err)
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(shelf["app"].Resources)
|
||||
if !strings.Contains(string(raw), "app/server@"+digest) {
|
||||
t.Fatalf("app is registered as %s, not by the check's digest in its own namespace", raw)
|
||||
}
|
||||
|
||||
// The walk asked it: the walk keeps what its build did, and its gate judges the registered build.
|
||||
asked := time.Now().UTC()
|
||||
p := g.plan(t)
|
||||
p.Commit = "c3"
|
||||
p.Modules["app"] = &inventory.PlanModule{State: "asked", AskedAt: &asked, Build: id}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
planBuiltSaying(ctx, g.open, "app", "c3", "", asked, id, result.MadeWords())
|
||||
p = g.plan(t)
|
||||
if m := p.Modules["app"]; m.State != "built" || !strings.HasPrefix(m.Made, "reused from the check of novox/mesh-catalog#9") {
|
||||
t.Fatalf("the walk keeps %+v", m)
|
||||
}
|
||||
for i := 0; i < 6; i++ {
|
||||
advancePlans(ctx, g.open)
|
||||
}
|
||||
p = g.plan(t)
|
||||
gate := p.Modules["app"].Gate
|
||||
if gate == nil || gate.To != "c3" || gate.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("the gate judged %+v, not the reused build", gate)
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c3" {
|
||||
t.Fatalf("the first machine was sent %v, not the reused build", sent)
|
||||
}
|
||||
}
|
||||
@@ -2,12 +2,10 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
@@ -46,25 +44,6 @@ func TestACheckIsAskedToBuildWhatTheMergeWouldMove(t *testing.T) {
|
||||
|
||||
// A walk's ask carries its leave to take a check's build, naming the pull requests of the merges it answers; a dry
|
||||
// run never does, and neither does an ask that is not a walk's.
|
||||
func TestOnlyAWalksAskTakesAChecksBuildAndADryRunNever(t *testing.T) {
|
||||
p := &inventory.Plan{Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{
|
||||
{Repository: "novox/shop", Number: 7}, {Repository: "http://forge.invalid/novox/web.git", Number: 3},
|
||||
{Repository: "novox/shop", Number: 7}, {Repository: "novox/shop"}}}}
|
||||
asked := reuseAskOf(p)
|
||||
if want := []link.PullRef{{Owner: "novox", Repo: "shop", Number: 7}, {Owner: "novox", Repo: "web", Number: 3}}; !reflect.DeepEqual(asked.Pulls, want) {
|
||||
t.Fatalf("the walk names %+v", asked.Pulls)
|
||||
}
|
||||
walk := withReuse(t.Context(), asked)
|
||||
if reuseAsked(walk, false) != asked {
|
||||
t.Fatal("a walk's ask does not carry its leave")
|
||||
}
|
||||
if reuseAsked(walk, true) != nil {
|
||||
t.Fatal("a dry run carries a leave to take a check's build")
|
||||
}
|
||||
if reuseAsked(t.Context(), false) != nil {
|
||||
t.Fatal("an ask that is not a walk's carries a leave")
|
||||
}
|
||||
}
|
||||
|
||||
// A walk ended lets go of its merges' check builds — only once it ended, after it took what it reuses.
|
||||
func TestAWalkEndedLetsGoOfItsMergesCheckBuilds(t *testing.T) {
|
||||
@@ -124,72 +103,6 @@ func TestAWalksBuildSaysWhetherItReusedOrBuilt(t *testing.T) {
|
||||
// module's namespace — and the walk keeps "reused from …" for it; its first-node gate judges that registered
|
||||
// build as it judges one the walk built. A scratch reference is refused. Needs the store (MESH_TEST_POSTGRES), and
|
||||
// is named in testdata/store-tests.
|
||||
func TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := g.open.inventory
|
||||
gateEvery = 0
|
||||
|
||||
digest := "sha256:" + strings.Repeat("c", 64)
|
||||
reused := func(id, reference string) link.BuildResult {
|
||||
manifest := mustJSON(t, catalogue.Manifest{Module: "app", Version: "c3", Resources: []map[string]any{
|
||||
{"id": "svc", "type": "container", "name": "app", "image": reference}}})
|
||||
return link.BuildResult{ID: id, Repository: "novox/mesh-catalog", Path: "modules/app", Ref: "main", On: "anchor",
|
||||
Commit: "c3", Manifest: manifest, Trunk: "main", OnTrunk: true, Branches: []string{"main"},
|
||||
Made: []link.MadeArtifact{{Name: "server", Kind: "image", Reference: reference}},
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"},
|
||||
Reused: "reused from the check of novox/mesh-catalog#9 at head abcdefabcdef (merged into trunk 123456123456): server " + digest,
|
||||
ReusedFrom: &link.ReusedFrom{Pull: link.PullRef{Owner: "novox", Repo: "mesh-catalog", Number: 9},
|
||||
Head: "abcdefabcdef", Trunk: "123456123456", Key: "r1:" + strings.Repeat("1", 64)}}
|
||||
}
|
||||
if _, _, err := takeIn(ctx, inv, reused(link.NewBuildID(time.Now()), "store.internal:5000/scratch/app/server@"+digest)); err == nil ||
|
||||
!strings.Contains(err.Error(), "scratch") {
|
||||
t.Fatalf("a build in the scratch namespace was registered: %v", err)
|
||||
}
|
||||
id := link.NewBuildID(time.Now().Add(time.Second))
|
||||
result := reused(id, "store.internal:5000/app/server@"+digest)
|
||||
if _, _, err := takeIn(ctx, inv, result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil || current["app"].Commit != "c3" {
|
||||
t.Fatalf("the reused build is not what app is registered at: %+v %v", current["app"], err)
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(shelf["app"].Resources)
|
||||
if !strings.Contains(string(raw), "app/server@"+digest) {
|
||||
t.Fatalf("app is registered as %s, not by the check's digest in its own namespace", raw)
|
||||
}
|
||||
|
||||
// The walk asked it: the walk keeps what its build did, and its gate judges the registered build.
|
||||
asked := time.Now().UTC()
|
||||
p := g.plan(t)
|
||||
p.Commit = "c3"
|
||||
p.Modules["app"] = &inventory.PlanModule{State: "asked", AskedAt: &asked, Build: id}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
planBuiltSaying(ctx, g.open, "app", "c3", "", asked, id, result.MadeWords())
|
||||
p = g.plan(t)
|
||||
if m := p.Modules["app"]; m.State != "built" || !strings.HasPrefix(m.Made, "reused from the check of novox/mesh-catalog#9") {
|
||||
t.Fatalf("the walk keeps %+v", m)
|
||||
}
|
||||
for i := 0; i < 6; i++ {
|
||||
advancePlans(ctx, g.open)
|
||||
}
|
||||
p = g.plan(t)
|
||||
gate := p.Modules["app"].Gate
|
||||
if gate == nil || gate.To != "c3" || gate.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("the gate judged %+v, not the reused build", gate)
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c3" {
|
||||
t.Fatalf("the first machine was sent %v, not the reused build", sent)
|
||||
}
|
||||
}
|
||||
|
||||
// A walk's leave names the builds its own pull requests' verdicts made at the head each merge merged — never an
|
||||
// earlier head's — read once per walk, and none when they cannot be read: the builder takes nothing a verdict does
|
||||
|
||||
@@ -127,10 +127,15 @@ func siblingsFor(p link.PullUpdated, entries []inventory.Entry, beside map[strin
|
||||
// could not be learned (ADR 0382 §1): a check never waits on mesh-delivery past groupAskWithin and never fails for it,
|
||||
// so a fix to mesh-delivery is still checked without it.
|
||||
func withSiblings(ctx context.Context, request *link.BuildRequest, p link.PullUpdated, entries []inventory.Entry) {
|
||||
lines, err := groupsAsked(ctx, deliverySeatHeld(entries))
|
||||
siblingsFrom(request, p, entries, lines, err)
|
||||
}
|
||||
|
||||
// siblingsFrom is withSiblings from mesh-delivery's answer already had.
|
||||
func siblingsFrom(request *link.BuildRequest, p link.PullUpdated, entries []inventory.Entry, lines []groupsLine, err error) {
|
||||
if request.Check == nil {
|
||||
return
|
||||
}
|
||||
lines, err := groupsAsked(ctx, deliverySeatHeld(entries))
|
||||
if err != nil {
|
||||
request.Check.SiblingsUnknown = fmt.Sprintf("mesh-delivery did not say the group's pull requests (%v): "+
|
||||
"the check reads no siblings", err)
|
||||
@@ -138,3 +143,23 @@ func withSiblings(ctx context.Context, request *link.BuildRequest, p link.PullUp
|
||||
}
|
||||
request.Check.Siblings, request.Check.SiblingsUnknown = siblingsFor(p, entries, request.Check.Beside, lines)
|
||||
}
|
||||
|
||||
// composedMember says the pull request is a member of a delivery group whose gate is the composed check's (novox/hq
|
||||
// ADR 0403 clauses 1 and 7): the group has two members or more and mesh-delivery says so on its line. The group is
|
||||
// found as siblingsFor finds it, by its head branch name, or, for a recheck that carries none, by the pull request's
|
||||
// repository and number — not by siblings, which name only what the check reads beside it.
|
||||
func composedMember(p link.PullUpdated, lines []groupsLine) bool {
|
||||
own := strings.ToLower(p.Owner + "/" + p.Repo)
|
||||
for _, l := range lines {
|
||||
found := p.Head != "" && l.ID == p.Head
|
||||
for _, m := range l.Details {
|
||||
if p.Head == "" && p.Number != 0 && m.Number == p.Number && strings.ToLower(m.Repository) == own {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if found {
|
||||
return l.ComposedGate && max(len(l.Details), len(l.Members)) >= 2
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -122,3 +122,40 @@ func TestASeatAnotherModuleDeclaresIsANoteOverSomeManifests(t *testing.T) {
|
||||
t.Fatalf("the note was not said:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// **A module lives whole in one repository** (novox/hq ADR 0420 §1, issue 1019): `module check`, told the
|
||||
// repository it runs for, refuses an artifact whose build context names another; run by hand, it does not know.
|
||||
func TestModuleCheckRefusesAnArtifactContextNamingAnotherRepository(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
write := func(name, context string) string {
|
||||
path := filepath.Join(dir, name+".json")
|
||||
raw := `{"module":"` + name + `","version":"1","build":{"artifacts":[{"name":"server","kind":"image","from":"Dockerfile"` + context + `}]}}`
|
||||
if err := os.WriteFile(path, []byte(raw), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
other := write("other", `,"context":{"seat":"git","repository":"novox/mesh-controller","ref":"main"}`)
|
||||
own := write("own", `,"context":{"seat":"git","repository":"novox/mesh-catalog","ref":"main"}`)
|
||||
none := write("none", ``)
|
||||
|
||||
t.Setenv("MESH_CHECK_REPOSITORY", "novox/mesh-catalog")
|
||||
var out bytes.Buffer
|
||||
if err := moduleCheck([]string{other}, &out); err == nil {
|
||||
t.Fatalf("a context naming another repository passed:\n%s", out.String())
|
||||
}
|
||||
if !strings.Contains(out.String(), "other builds artifact server from novox/mesh-controller: a module lives whole in one repository (novox/hq ADR 0420 §1)") {
|
||||
t.Errorf("the refusal is not in the words of the gate:\n%s", out.String())
|
||||
}
|
||||
for _, path := range []string{own, none} {
|
||||
out.Reset()
|
||||
if err := moduleCheck([]string{path}, &out); err != nil {
|
||||
t.Errorf("%s was refused: %v\n%s", path, err, out.String())
|
||||
}
|
||||
}
|
||||
t.Setenv("MESH_CHECK_REPOSITORY", "")
|
||||
out.Reset()
|
||||
if err := moduleCheck([]string{other}, &out); err != nil {
|
||||
t.Errorf("a local run, with no repository named, refused it: %v\n%s", err, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
+64
-120
@@ -3,7 +3,6 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path"
|
||||
"regexp"
|
||||
@@ -155,6 +154,24 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||
return err
|
||||
}
|
||||
scope := pullScope(p, entries, read, edges)
|
||||
// **A cycle of edges fails the check, naming every module and edge** (novox/hq ADR 0419 clause 6), a lone pull
|
||||
// request's as a group's composed check is: no order exists for what it changes.
|
||||
alone := orderMember{ID: p.Owner + "/" + p.Repo + "@" + twelve(p.Commit), Repository: p.Owner + "/" + p.Repo, Base: p.Base,
|
||||
Head: p.Commit, Number: p.Number, Paths: p.Paths, PathsTruncated: p.PathsTruncated, Removed: p.Removed, BaseSHA: p.BaseSHA}
|
||||
if failing, cyclic, err := cycleVerdict(ctx, "", []orderMember{alone},
|
||||
map[string]orderReach{alone.ID: {Moved: scope.Modules}}, edges, entries); err != nil {
|
||||
return err
|
||||
} else if cyclic {
|
||||
failing.Commit, failing.Trunk = p.Commit, twelve(p.BaseSHA)
|
||||
fmt.Printf("%s/%s#%d (%.8s): %s\n", p.Owner, p.Repo, p.Number, p.Commit, failing.Summary)
|
||||
sayChecked(ctx, failing)
|
||||
return nil
|
||||
}
|
||||
// **A member of a group whose gate is the composed check's runs its repository's check alone** (novox/hq ADR 0403
|
||||
// clause 1): mesh-delivery is asked once, and says so on the group's line. Told nothing, the pull request is
|
||||
// checked alone, gate included (clause 6).
|
||||
lines, groupsErr := groupsAsked(ctx, deliverySeatHeld(entries))
|
||||
composed := groupsErr == nil && composedMember(p, lines)
|
||||
// The change plan of the commit at hand (ADR 0238): what a merge of it would build and send, posted
|
||||
// with the verdict whatever the verdict is.
|
||||
plan := changePlanOf(p.Owner+"/"+p.Repo, p.Base, p.Commit, scope.Reach, entries, func(module string) (inventory.Upgrade, bool) {
|
||||
@@ -163,6 +180,10 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||
})
|
||||
fmt.Print(planText(plan))
|
||||
direct := directVerdict(p, plan)
|
||||
if composed {
|
||||
scope.Modules, scope.Dependents, scope.New, scope.Manifests = nil, nil, nil, nil
|
||||
composedGate(&direct)
|
||||
}
|
||||
switch {
|
||||
case !scope.gated() && !scope.Mesh:
|
||||
fmt.Printf("%s/%s#%d (%.8s): %s, and the repository is not the mesh's: said, nothing run\n",
|
||||
@@ -180,8 +201,9 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||
return err
|
||||
}
|
||||
request.Check.Plan = &plan
|
||||
request.Check.Composed = composed
|
||||
// **The group's other open pull requests are read beside it** (novox/hq ADR 0382 §1).
|
||||
withSiblings(ctx, &request, p, entries)
|
||||
siblingsFrom(&request, p, entries, lines, groupsErr)
|
||||
seat := buildSeatHeld(ctx)
|
||||
ask, err := askOverOn(seat)
|
||||
if err != nil {
|
||||
@@ -191,9 +213,13 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||
if err := ask.Ask(ctx, request); err != nil {
|
||||
return err
|
||||
}
|
||||
// A new head ends the check of the old one (novox/hq issue 726).
|
||||
endingSupersededChecks(ctx, seat, p, request.ID, f.oneShot)
|
||||
// **A newer request replaces the one still waiting, and a newer key stops the one running** (novox/hq ADR 0406 §3, §4,
|
||||
// issue 974): the check queue keeps one request a pull request, and a running check watches its key. Nothing is asked
|
||||
// of the forge here (issue 726).
|
||||
what := "its own merge-check.sh alone: " + noModuleTouched
|
||||
if composed {
|
||||
what = "its own merge-check.sh alone: the gate is its group's composed check's"
|
||||
}
|
||||
if scope.gated() {
|
||||
what = fmt.Sprintf("the gate over %s (a merge would build %d module(s) in %d tier(s))",
|
||||
strings.Join(append(append([]string{}, scope.Modules...), prefixedAll("new:", scope.New)...), ", "),
|
||||
@@ -204,6 +230,20 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// composedWords is what a group member's own verdict says of its gate layer (novox/hq ADR 0403 clause 1).
|
||||
const composedWords = "judged by its group's composed check"
|
||||
|
||||
// gateComposed is a gate layer's verdict when no gate was run for the member alone: never a pass. Mesh-delivery reads it
|
||||
// as the composed check's to judge, and the forge's holder sets no mesh/merge-gate from it.
|
||||
const gateComposed = "composed"
|
||||
|
||||
// composedGate says a group member's own verdict has no gate of its own: its gate layer, and the verdict that is
|
||||
// the gate's, read "composed" in place of a pass.
|
||||
func composedGate(c *link.Checked) {
|
||||
c.Verdict, c.Summary = gateComposed, composedWords
|
||||
c.Gate = &link.CheckLayer{Verdict: gateComposed, Summary: composedWords}
|
||||
}
|
||||
|
||||
// directVerdict is the verdict of a change no build judges — it touches no module, and either the repository is not
|
||||
// the mesh's or the head holds no merge-check.sh. **It names the trunk commit it holds for** (novox/hq ADR 0323 §3):
|
||||
// the base branch's head the forge announced with the head, so the merge tool can weigh it against the trunk's head
|
||||
@@ -253,12 +293,7 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
|
||||
if err != nil {
|
||||
return link.BuildRequest{}, err
|
||||
}
|
||||
// **A pull request into a release branch is judged by that branch** (novox/hq ADR 0358): asked of the forge once.
|
||||
release, err := releaseBranchOf(ctx, p, scope, entries)
|
||||
if err != nil {
|
||||
return link.BuildRequest{}, err
|
||||
}
|
||||
beside, err := besideOf(entries, func(module string) string { return current[module].Commit }, clone, release)
|
||||
beside, err := besideOf(entries, func(module string) string { return current[module].Commit }, clone)
|
||||
if err != nil {
|
||||
return link.BuildRequest{}, err
|
||||
}
|
||||
@@ -270,20 +305,17 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
|
||||
Held: heldBy(ctx),
|
||||
Seats: seatBases(ctx),
|
||||
Source: sourceOnSeat(source),
|
||||
Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base,
|
||||
Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base, Trunk: p.BaseSHA, Ready: p.Ready,
|
||||
Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents, New: scope.New,
|
||||
Manifests: scope.Manifests, Judge: scope.Judge, Builds: builds, Named: namedForTheCheck(ctx, p, builds),
|
||||
Release: release, Running: runningBy(ctx)},
|
||||
Running: runningBy(ctx)},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// besideOf is what a check reads beside its own repository, by the directory it is found under: each core repository
|
||||
// by the module the mesh builds from it, at besideRefs — or, for a pull request into a release branch (novox/hq ADR
|
||||
// 0358 §3), at its branch of that name where it has one and at its trunk otherwise, the controller's main kept for
|
||||
// its judge, and mesh-lab at its trunk whatever the base: its replays run with the container runtime's socket. running
|
||||
// is the commit the mesh runs of a module.
|
||||
// by the module the mesh builds from it, at besideRefs. running is the commit the mesh runs of a module.
|
||||
func besideOf(entries []inventory.Entry, running func(module string) string,
|
||||
clone func(inventory.Source) (string, error), release string) (map[string]link.CheckedOut, error) {
|
||||
clone func(inventory.Source) (string, error)) (map[string]link.CheckedOut, error) {
|
||||
beside := map[string]link.CheckedOut{}
|
||||
for _, e := range entries {
|
||||
dir, core := coreModules[e.Manifest.Module]
|
||||
@@ -295,34 +327,15 @@ func besideOf(entries []inventory.Entry, running func(module string) string,
|
||||
return nil, err
|
||||
}
|
||||
refs := besideRefs(dir, running(e.Manifest.Module))
|
||||
at := func(dir, trunk string) link.CheckedOut {
|
||||
if release == "" {
|
||||
return link.CheckedOut{Repository: url, Ref: refs[dir]}
|
||||
}
|
||||
if trunk == "" {
|
||||
trunk = "main"
|
||||
}
|
||||
return link.CheckedOut{Repository: url, Ref: release, Else: trunk}
|
||||
}
|
||||
beside[dir] = at(dir, followedBranch(e.Source.Ref))
|
||||
beside[dir] = link.CheckedOut{Repository: url, Ref: refs[dir]}
|
||||
if dir == "mesh-controller" {
|
||||
// The controller's main stays its main: it judges only a check the running controller predates.
|
||||
beside["mesh-controller-main"] = link.CheckedOut{Repository: url, Ref: refs["mesh-controller-main"]}
|
||||
if release != "" {
|
||||
// mesh-controller has the release branch: that is what makes it one (ADR 0358 §1).
|
||||
beside[dir] = link.CheckedOut{Repository: url, Ref: release}
|
||||
}
|
||||
if e.Source.Seat != "" {
|
||||
for _, sibling := range []string{"mesh-lab", "mesh-sdk"} {
|
||||
if url, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository,
|
||||
sibling)}); err == nil {
|
||||
b := link.CheckedOut{Repository: url, Ref: refs[sibling]}
|
||||
// **mesh-lab is read at its trunk, always** (ADR 0358): its replays run with the container
|
||||
// runtime's socket, so they are only ever reviewed code — never a branch anyone may create.
|
||||
if release != "" && sibling != "mesh-lab" {
|
||||
b = link.CheckedOut{Repository: url, Ref: release, Else: "main"}
|
||||
}
|
||||
beside[sibling] = b
|
||||
beside[sibling] = link.CheckedOut{Repository: url, Ref: refs[sibling]}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -331,47 +344,6 @@ func besideOf(entries []inventory.Entry, running func(module string) string,
|
||||
return beside, nil
|
||||
}
|
||||
|
||||
// releaseBranchOf is the release branch a pull request merges into (novox/hq ADR 0358 §1): its base, when that is not
|
||||
// its repository's trunk and mesh-controller has a branch of that name; empty otherwise, and the pull request is
|
||||
// checked as before it. **A base that is a trunk asks nothing**: the branch a module built from the repository follows
|
||||
// (main when it records none), or mesh-controller's own trunk. Any other base is asked of the forge once, and an
|
||||
// answer it cannot give is an error, never read as "no release branch".
|
||||
func releaseBranchOf(ctx context.Context, p link.PullUpdated, scope checkScope, entries []inventory.Entry) (string, error) {
|
||||
base := p.Base
|
||||
if base == "" {
|
||||
return "", nil
|
||||
}
|
||||
if scope.From != nil && base == trunkOf(scope.From.Source) {
|
||||
return "", nil
|
||||
}
|
||||
var controller *inventory.Entry
|
||||
for i, e := range entries {
|
||||
if e.Manifest.Module == "mesh-controller" && !e.Provided && e.Source.Repository != "" {
|
||||
controller = &entries[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if controller == nil || base == trunkOf(controller.Source) {
|
||||
return "", nil
|
||||
}
|
||||
if !tryRef.MatchString(base) || strings.Contains(base, "..") {
|
||||
return "", nil // no branch the forge could hold: checked as before
|
||||
}
|
||||
owner, repo := sourceOwner(controller.Source.Repository), sourceName(controller.Source.Repository)
|
||||
if owner == "" || repo == "" {
|
||||
return "", nil
|
||||
}
|
||||
has, err := forgeHasBranch(ctx, owner, repo, base)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("whether %s/%s#%d merges into a release branch cannot be told — the forge did not say "+
|
||||
"whether %s/%s has a branch %s (novox/hq ADR 0358): %w", p.Owner, p.Repo, p.Number, owner, repo, base, err)
|
||||
}
|
||||
if !has {
|
||||
return "", nil
|
||||
}
|
||||
return base, nil
|
||||
}
|
||||
|
||||
// trunkOf is the branch a module's source follows: main when it records none, or records a commit.
|
||||
func trunkOf(s inventory.Source) string {
|
||||
if t := strings.TrimSpace(followedBranch(s.Ref)); t != "" {
|
||||
@@ -389,46 +361,6 @@ func sourceName(repository string) string {
|
||||
return parts[len(parts)-1]
|
||||
}
|
||||
|
||||
// forgeHasBranch asks the forge, through its module's tools on the bus, whether a repository has a branch. A
|
||||
// variable so a test needs no forge.
|
||||
var forgeHasBranch = func(ctx context.Context, owner, repo, branch string) (bool, error) {
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer js.Close()
|
||||
raw, _ := json.Marshal(map[string]any{"path": "/repos/" + owner + "/" + repo + "/branches/" + branch})
|
||||
answer, err := link.Ask(ctx, link.OverNATS{Conn: js.Conn()}, "gitea", "gitea_api", raw, 30*time.Second)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return branchAnswered(answer, branch)
|
||||
}
|
||||
|
||||
// branchAnswered reads the forge's answer about a branch: a 404 is "no such branch"; an answer naming the branch is
|
||||
// "it has it"; anything else — another error, an unreadable answer, one naming no branch or another — is an error,
|
||||
// never read as "no release branch".
|
||||
func branchAnswered(answer link.Answer, branch string) (bool, error) {
|
||||
if answer.Error != "" {
|
||||
if strings.Contains(answer.Error, " 404 ") {
|
||||
return false, nil
|
||||
}
|
||||
return false, errors.New(answer.Error)
|
||||
}
|
||||
var found struct {
|
||||
Result struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"result"`
|
||||
}
|
||||
if err := json.Unmarshal(answer.Result, &found); err != nil {
|
||||
return false, fmt.Errorf("the forge's answer about branch %s cannot be read: %w", branch, err)
|
||||
}
|
||||
if found.Result.Name != branch {
|
||||
return false, fmt.Errorf("the forge answered about branch %s naming %q", branch, found.Result.Name)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// namedForTheCheck are the builds earlier checks of the same pull request at the same head named, read once for the
|
||||
// check (novox/hq issue 703): asked again because only the trunk moved, the check keeps what they built of a key
|
||||
// still the same. Another pull request's build, or one of an earlier head, is never kept. None when they cannot be
|
||||
@@ -582,6 +514,13 @@ const maxCheckReport = 60 << 10
|
||||
// checked says a merge check's verdict as the controller's `checked`. Nothing is recorded or
|
||||
// registered: a check builds nothing (issue 240's rule for a dry run, kept for a check).
|
||||
func checked(ctx context.Context, result link.BuildResult) {
|
||||
// **A check stopped by its key says nothing** (novox/hq ADR 0406 §4, issue 974, review 2): it stopped because the head or
|
||||
// the trunk commit moved on, and the newer request will judge. Said as an error it would name the live head — red
|
||||
// statuses on the forge, a delivery refused — and written to the key it would stand in the newer request's way.
|
||||
if result.Failed == link.SupersededByKey {
|
||||
fmt.Printf("%s: stopped, its pull request's key moved on; the newer check will judge\n", result.ID)
|
||||
return
|
||||
}
|
||||
sayChecked(ctx, checkedOf(result))
|
||||
}
|
||||
|
||||
@@ -600,7 +539,6 @@ func checkedOf(result link.BuildResult) link.Checked {
|
||||
// repositories read beside it.
|
||||
c.Trunk, c.Beside, c.Took = result.Check.Trunk, result.Check.Beside, result.Check.Took
|
||||
c.Built = result.Check.Built
|
||||
c.Release, c.JudgedBy, c.BesideAt = result.Check.Release, result.Check.JudgedBy, result.Check.BesideAt
|
||||
c.Validator = result.Check.Validator
|
||||
c.Siblings, c.SiblingsUnknown, c.JudgeFrom = result.Check.Siblings, result.Check.SiblingsUnknown, result.Check.JudgeFrom
|
||||
if c.Gate == nil {
|
||||
@@ -620,6 +558,12 @@ func checkedOf(result link.BuildResult) link.Checked {
|
||||
c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
|
||||
c.RepoCheck = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
|
||||
}
|
||||
// **A group member's own check has no gate of its own** (novox/hq ADR 0403 clause 1): what ran was the repository's
|
||||
// check alone, and its gate layer is "composed", never a pass. A failure or an error stays what it was.
|
||||
if result.Checked != nil && result.Checked.Composed && result.Check != nil && c.Gate != nil &&
|
||||
(c.Gate.Verdict == "pass" || c.Gate.Verdict == "warning") {
|
||||
composedGate(&c)
|
||||
}
|
||||
if result.Checked != nil && c.Gate != nil && len(c.Gate.Modules) == 0 {
|
||||
c.Gate.Modules = append(append([]string{}, result.Checked.Modules...), prefixedAll("new:", result.Checked.New)...)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
func TestACheckStoppedByItsKeyIsAnnouncedAsNoVerdict(t *testing.T) {
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
legacy, err := js.Conn().JetStream()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
was := checkEvents
|
||||
checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy}
|
||||
t.Cleanup(func() { checkEvents = was })
|
||||
heard := make(chan *nats.Msg, 4)
|
||||
sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyChecked), heard)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog", Number: 3}
|
||||
checked(t.Context(), link.BuildResult{ID: "stopped", Ref: "abc", Checked: asked, Failed: link.SupersededByKey})
|
||||
checked(t.Context(), link.BuildResult{ID: "broken", Ref: "abc", Checked: asked, Failed: "the facts snapshot cannot be read"})
|
||||
select {
|
||||
case m := <-heard:
|
||||
var c link.Checked
|
||||
_ = json.Unmarshal(m.Data, &c)
|
||||
if c.ID != "broken" || c.Verdict != "error" {
|
||||
t.Fatalf("the first verdict said is %+v; want only the broken check's error", c)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("nothing was said")
|
||||
}
|
||||
select {
|
||||
case m := <-heard:
|
||||
t.Fatalf("a second verdict was said: %s", m.Data)
|
||||
case <-time.After(300 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// Acceptance 5 of milestone 2 (novox/hq ADR 0406 §3, issue 974), live, with the store and a real bus: two merges into one
|
||||
// repository within a minute, with two open pull requests, leave each pull request with exactly one queued check, by the
|
||||
// stream's rule. Each merge moves the trunk, and mesh-delivery asks the controller to check both open pull requests again
|
||||
// against it (`delivery check`, which is the controller's own path for a head announced); nothing moves a request, nothing
|
||||
// asks the forge, and the queue is read back whole.
|
||||
func TestTwoMergesWithinAMinuteLeaveEachOpenPullRequestOneQueuedCheck(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
js, _ := aBuildQueue(t)
|
||||
servingBus.Store(js)
|
||||
t.Cleanup(func() { servingBus.Store(nil) })
|
||||
holdTheDeliverySeat(t, open)
|
||||
was := askDeliveryOwner
|
||||
t.Cleanup(func() { askDeliveryOwner = was })
|
||||
askDeliveryOwner = func(_ context.Context, _ *nats.Conn, verb string, _ map[string]any) (json.RawMessage, error) {
|
||||
if verb != "groups" {
|
||||
t.Fatalf("asked the owner %s", verb)
|
||||
}
|
||||
return json.RawMessage(`[]`), nil
|
||||
}
|
||||
// The repository the two pull requests are into, as a module the mesh builds from it.
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-controller", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
heads := map[int]string{7: "7777777777777777777777777777777777777777", 8: "8888888888888888888888888888888888888888"}
|
||||
check := func(number int, trunk string) {
|
||||
t.Helper()
|
||||
p := orderMember{ID: fmt.Sprintf("novox/mesh-controller@%.12s", heads[number]), Repository: "novox/mesh-controller",
|
||||
Base: "main", Head: heads[number], Number: number, Paths: []string{"main.go"}, ModuleDirsSaid: true,
|
||||
BaseSHA: trunk, Ready: number == 8}.pull()
|
||||
p.MergeCheck, p.MergeCheckSaid = true, true
|
||||
if err := (following{open: open, oneShot: true}).PullUpdated(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// Announced against the trunk as it stands, then asked again after each of two merges.
|
||||
for _, trunk := range []string{"1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222",
|
||||
"3333333333333333333333333333333333333333"} {
|
||||
check(7, trunk)
|
||||
check(8, trunk)
|
||||
}
|
||||
|
||||
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seen := map[int]string{}
|
||||
for _, a := range q.Asks {
|
||||
c := a.Request.Check
|
||||
if c == nil {
|
||||
t.Fatalf("%s is no check", a.ID)
|
||||
}
|
||||
if _, twice := seen[c.Number]; twice {
|
||||
t.Errorf("pull request %d has two queued checks", c.Number)
|
||||
}
|
||||
seen[c.Number] = c.Trunk
|
||||
if a.State != link.AskWaiting {
|
||||
t.Errorf("%s is %s", a.ID, a.State)
|
||||
}
|
||||
if want := "3333333333333333333333333333333333333333"; c.Trunk != want {
|
||||
t.Errorf("pull request %d is queued against %.8s; want the newest trunk %.8s", c.Number, c.Trunk, want)
|
||||
}
|
||||
}
|
||||
if len(seen) != 2 {
|
||||
t.Fatalf("queued checks for %v; want one for each of the two open pull requests", seen)
|
||||
}
|
||||
// By the streams' own count: the ready one on its stream, the other on its.
|
||||
for stream, want := range map[string]uint64{broker.ChecksReadyStream: 1, broker.ChecksOtherStream: 1} {
|
||||
info, err := js.Context().StreamInfo(stream)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.State.Msgs != want {
|
||||
t.Errorf("%s holds %d request(s); want %d", stream, info.State.Msgs, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -288,3 +288,6 @@ func TestARecheckedHeadsDirectVerdictNamesItsTrunk(t *testing.T) {
|
||||
t.Fatalf("a rechecked head's direct verdict names trunk %q", c.Trunk)
|
||||
}
|
||||
}
|
||||
|
||||
// A check stopped by its pull request's key moving on says no verdict (issue 974, review 2): an error said for it would name the
|
||||
// live head, and the newer request is the one that judges. A check that failed for any other reason still says its error.
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A member of a group whose gate is the composed check's is asked of the build seat with no module: its repository's
|
||||
// check alone (novox/hq ADR 0403 clause 1). With no word from mesh-delivery, or one that does not say the gate is
|
||||
// composed, it is checked alone with its gate (clause 6).
|
||||
|
||||
func askedForMember(t *testing.T, answer []groupsLine, answerErr error) link.CheckRequest {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
js, _ := aBuildQueue(t)
|
||||
servingBus.Store(js)
|
||||
t.Cleanup(func() { servingBus.Store(nil) })
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-controller", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
was := groupsAsked
|
||||
groupsAsked = func(context.Context, bool) ([]groupsLine, error) { return answer, answerErr }
|
||||
t.Cleanup(func() { groupsAsked = was })
|
||||
p := link.PullUpdated{Owner: "novox", Repo: "mesh-controller", Number: 346, Base: "main", Head: "feat/x", Commit: headController,
|
||||
Paths: []string{"main.go"}, ModuleDirsSaid: true, MergeCheck: true, MergeCheckSaid: true}
|
||||
if err := (following{open: open, oneShot: true}).PullUpdated(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||
if err != nil || len(q.Asks) != 1 {
|
||||
t.Fatalf("asks %d: %v", len(q.Asks), err)
|
||||
}
|
||||
return *q.Asks[0].Request.Check
|
||||
}
|
||||
|
||||
func TestAComposedMembersCheckIsItsRepositoryChecksAlone(t *testing.T) {
|
||||
line := group("feat/x", member("novox/mesh-controller", 346, headController, "main", true), member("novox/mesh-catalog", 350, headHost, "main", true))
|
||||
line.ComposedGate = true
|
||||
c := askedForMember(t, []groupsLine{line}, nil)
|
||||
if len(c.Modules) != 0 || len(c.New) != 0 || len(c.Dependents) != 0 || !c.Composed {
|
||||
t.Fatalf("the member's check is asked as %+v; want no module and composed", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMemberOfAGroupOfAnOlderMeshDeliveryKeepsItsOwnGate(t *testing.T) {
|
||||
line := group("feat/x", member("novox/mesh-controller", 346, headController, "main", true), member("novox/mesh-catalog", 350, headHost, "main", true))
|
||||
c := askedForMember(t, []groupsLine{line}, nil)
|
||||
if !strings.Contains(strings.Join(c.Modules, ","), "mesh-controller") || c.Composed {
|
||||
t.Fatalf("an older mesh-delivery's member is asked as %+v; want its own gate", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMemberWhoseGroupCannotBeLearnedIsCheckedAloneWithItsGate(t *testing.T) {
|
||||
c := askedForMember(t, nil, context.DeadlineExceeded)
|
||||
if !strings.Contains(strings.Join(c.Modules, ","), "mesh-controller") || c.Composed ||
|
||||
!strings.Contains(c.SiblingsUnknown, "reads no siblings") {
|
||||
t.Fatalf("the member is asked as %+v; want its gate and why its siblings are unknown", c)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A delivery group's gate is the composed check's, and its members' own checks run their repository's check alone
|
||||
// (novox/hq ADR 0403 clauses 1 and 7). Pure: no store, no bus.
|
||||
|
||||
// A member of a group whose line says composed_gate is a composed member; one whose group says nothing of it (an older
|
||||
// mesh-delivery), one of a group of a single pull request and a lone pull request are not.
|
||||
func TestAMemberOfAComposedGroupIsKnownByTheGroupsLine(t *testing.T) {
|
||||
two := func(composed bool) groupsLine {
|
||||
l := group("feat/x", member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-controller", 346, headController, "main", true))
|
||||
l.ComposedGate = composed
|
||||
return l
|
||||
}
|
||||
p := link.PullUpdated{Owner: "novox", Repo: "mesh-host", Number: 114, Base: "main", Head: "feat/x", Commit: headHost}
|
||||
recheck := link.PullUpdated{Owner: "novox", Repo: "mesh-host", Number: 114, Base: "main", Commit: headHost}
|
||||
for name, c := range map[string]struct {
|
||||
p link.PullUpdated
|
||||
lines []groupsLine
|
||||
want bool
|
||||
}{
|
||||
"composed": {p, []groupsLine{two(true)}, true},
|
||||
"a recheck, by its number": {recheck, []groupsLine{two(true)}, true},
|
||||
"an older mesh-delivery": {p, []groupsLine{two(false)}, false},
|
||||
"another group": {p, []groupsLine{group("feat/y", member("novox/mesh-host", 1, headHost, "main", true))}, false},
|
||||
"no group": {p, nil, false},
|
||||
} {
|
||||
if got := composedMember(c.p, c.lines); got != c.want {
|
||||
t.Errorf("%s: composed %v, want %v", name, got, c.want)
|
||||
}
|
||||
}
|
||||
alone := group("feat/x", member("novox/mesh-host", 114, headHost, "main", true))
|
||||
alone.ComposedGate = true
|
||||
if composedMember(p, []groupsLine{alone}) {
|
||||
t.Error("a group of a single pull request is composed")
|
||||
}
|
||||
}
|
||||
|
||||
// A composed member's verdict never says its gate passed: the gate layer, and the verdict that is the gate's, read
|
||||
// "composed"; a failure or an error stays what it was.
|
||||
func TestAComposedMembersVerdictHasNoPassingGate(t *testing.T) {
|
||||
asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-host", Number: 114, Composed: true}
|
||||
pass := link.BuildResult{ID: "b", Ref: headHost, Checked: asked, Check: &link.CheckOutcome{Verdict: "pass", Summary: "no module touched",
|
||||
Gate: &link.CheckLayer{Verdict: "pass", Summary: "no module touched"}, RepoCheck: &link.CheckLayer{Verdict: "pass", Summary: "ok"}}}
|
||||
c := checkedOf(pass)
|
||||
if c.Gate == nil || c.Gate.Verdict != "composed" || c.Verdict != "composed" || c.RepoCheck == nil || c.RepoCheck.Verdict != "pass" {
|
||||
t.Fatalf("the member's verdict says %+v gate %+v", c, c.Gate)
|
||||
}
|
||||
asked.Composed = false
|
||||
if c := checkedOf(pass); c.Gate.Verdict != "pass" || c.Verdict != "pass" {
|
||||
t.Fatalf("a member of no composed group has its own gate: %+v", c.Gate)
|
||||
}
|
||||
asked.Composed = true
|
||||
failing := pass
|
||||
failing.Check = &link.CheckOutcome{Verdict: "fail", Summary: "conflict", Gate: &link.CheckLayer{Verdict: "fail", Summary: "conflict"},
|
||||
RepoCheck: &link.CheckLayer{Verdict: "fail", Summary: "conflict"}}
|
||||
if c := checkedOf(failing); c.Gate.Verdict != "fail" || c.Verdict != "fail" {
|
||||
t.Fatalf("a failure was read as composed: %+v", c.Gate)
|
||||
}
|
||||
broken := link.BuildResult{ID: "b", Ref: headHost, Checked: asked, Failed: "the facts snapshot cannot be read"}
|
||||
if c := checkedOf(broken); c.Verdict != "error" || c.Gate.Verdict != "error" {
|
||||
t.Fatalf("an error was read as composed: %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// The composed verdict on the bus carries every member, each with its trunk commit, for the forge's holder to set each
|
||||
// member's mesh/merge-gate from it.
|
||||
func TestTheComposedVerdictCarriesEveryMember(t *testing.T) {
|
||||
asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-host", Number: 114, Group: "feat/x",
|
||||
Members: []link.GroupMember{{Owner: "novox", Repo: "mesh-controller", Number: 346, Ref: headController}}}
|
||||
c := checkedOf(link.BuildResult{ID: "g", Ref: headHost, Checked: asked, Check: &link.CheckOutcome{Verdict: "pass", Summary: "ok",
|
||||
Trunk: "aaaaaaaaaaaa", Members: []link.CheckedMember{{Owner: "novox", Repo: "mesh-controller", Commit: headController[:12], Trunk: "bbbbbbbbbbbb"}}}})
|
||||
body, _ := json.Marshal(c)
|
||||
for _, want := range []string{`"group":"feat/x"`, `"members":[{"owner":"novox","repo":"mesh-host","number":114,"commit":"` + headHost + `","trunk":"aaaaaaaaaaaa"}`,
|
||||
`{"owner":"novox","repo":"mesh-controller","number":346,"commit":"` + headController + `","trunk":"bbbbbbbbbbbb"}`} {
|
||||
if !strings.Contains(string(body), want) {
|
||||
t.Fatalf("the composed verdict lacks %s:\n%s", want, body)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Members whose manifests order their modules in a cycle fail the composed check, naming every module and edge, and no
|
||||
// ask is made of the build seat.
|
||||
func TestAGroupInACycleFailsItsComposedCheckWithoutABuild(t *testing.T) {
|
||||
members := []orderMember{
|
||||
{ID: "novox/x@" + headHost[:12], Repository: "novox/x", Base: "main", Head: headHost, Number: 7, Paths: []string{"module.json"}},
|
||||
{ID: "novox/y@" + headController[:12], Repository: "novox/y", Base: "main", Head: headController, Number: 9, Paths: []string{"module.json"}},
|
||||
}
|
||||
reach := map[string]orderReach{members[0].ID: {Moved: []string{"mx"}}, members[1].ID: {Moved: []string{"my"}}}
|
||||
was, wasParent := manifestAt, parentOf
|
||||
t.Cleanup(func() { manifestAt, parentOf = was, wasParent })
|
||||
parentOf = func(context.Context, string, string, string) (string, error) { return "", nil }
|
||||
manifestAt = func(_ context.Context, _, repo, _, ref string) (catalogue.Manifest, bool, error) {
|
||||
if ref == "main" {
|
||||
return catalogue.Manifest{}, false, nil // what the pull requests merge into holds no such manifest
|
||||
}
|
||||
if repo == "x" {
|
||||
return catalogue.Manifest{Module: "mx", DefinesSeats: []catalogue.SeatDeclaration{{Name: "s", Serves: []catalogue.Verb{{Name: "a"}}}},
|
||||
Invokes: []string{"seat:s.b"}}, true, nil
|
||||
}
|
||||
return catalogue.Manifest{Module: "my", DefinesSeats: []catalogue.SeatDeclaration{{Name: "s", Serves: []catalogue.Verb{{Name: "b"}}}},
|
||||
Invokes: []string{"seat:s.a"}}, true, nil
|
||||
}
|
||||
c, cyclic, err := cycleVerdict(context.Background(), "feat/cycle", members, reach, []inventory.Edge(nil), nil)
|
||||
if err != nil || !cyclic {
|
||||
t.Fatalf("no cycle found: %v", err)
|
||||
}
|
||||
if c.Verdict != "fail" || c.Group != "feat/cycle" || len(c.Members) != 2 || c.Gate == nil || c.Gate.Verdict != "fail" {
|
||||
t.Fatalf("the verdict is %+v", c)
|
||||
}
|
||||
for _, w := range []string{"mx", "my", "s.a", "s.b"} {
|
||||
if !strings.Contains(c.Summary, w) {
|
||||
t.Errorf("the cycle does not name %s: %s", w, c.Summary)
|
||||
}
|
||||
}
|
||||
// Members with no edge between them are no cycle.
|
||||
manifestsReadMu.Lock()
|
||||
clear(manifestsRead)
|
||||
manifestsReadMu.Unlock()
|
||||
manifestAt = func(context.Context, string, string, string, string) (catalogue.Manifest, bool, error) {
|
||||
return catalogue.Manifest{}, false, nil
|
||||
}
|
||||
if _, cyclic, err := cycleVerdict(context.Background(), "feat/cycle", members, reach, nil, nil); err != nil || cyclic {
|
||||
t.Fatalf("a cycle where there is none: %v %v", cyclic, err)
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestAVerdictCarriesAProbesFindingsInBrief(t *testing.T) {
|
||||
var found []string
|
||||
for i := 0; i < 689; i++ {
|
||||
found = append(found, fmt.Sprintf("delivery d-%012x has been checking for 13h, past its bound of 45m", i))
|
||||
}
|
||||
run := doctorRun{Run: "run-1", Probes: []probeVerdict{{ID: "D14", Verdict: "failed", Found: found},
|
||||
{ID: "D1", Verdict: "passed"}}}
|
||||
answer, err := verdictAnswer(run, time.Now(), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
brief := answer["run"].(doctorRun).Probes[0]
|
||||
if len(brief.Found) != overviewFindings || brief.FoundCount != 689 || !strings.Contains(brief.More, "probe=D14") {
|
||||
t.Fatalf("the verdict carries %d findings of D14, count %d, more %q", len(brief.Found), brief.FoundCount, brief.More)
|
||||
}
|
||||
if len(run.Probes[0].Found) != 689 {
|
||||
t.Fatal("the overview cut the run it was made from")
|
||||
}
|
||||
whole, err := verdictAnswer(run, time.Now(), "d14")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p := whole["run"].(doctorRun).Probes; len(p) != 1 || len(p[0].Found) != 689 || p[0].More != "" {
|
||||
t.Fatalf("doctor probe=D14 answered %+v", p)
|
||||
}
|
||||
if _, err := verdictAnswer(run, time.Now(), "D99"); err == nil {
|
||||
t.Fatal("a probe the run does not have was answered")
|
||||
}
|
||||
if !strings.Contains(doctorText(answer), "probe=D14` gives them all") {
|
||||
t.Fatalf("the text does not say how to read them all:\n%s", doctorText(answer))
|
||||
}
|
||||
}
|
||||
@@ -74,35 +74,3 @@ func TestAListOfConditionsCarriesEachInBrief(t *testing.T) {
|
||||
|
||||
// The self-check's verdict carries at most overviewFindings of one probe's findings, with their count and
|
||||
// how to read them all; `doctor probe=` gives that probe whole (novox/hq issue 314).
|
||||
func TestAVerdictCarriesAProbesFindingsInBrief(t *testing.T) {
|
||||
var found []string
|
||||
for i := 0; i < 689; i++ {
|
||||
found = append(found, fmt.Sprintf("delivery d-%012x has been checking for 13h, past its bound of 45m", i))
|
||||
}
|
||||
run := doctorRun{Run: "run-1", Probes: []probeVerdict{{ID: "D14", Verdict: "failed", Found: found},
|
||||
{ID: "D1", Verdict: "passed"}}}
|
||||
answer, err := verdictAnswer(run, time.Now(), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
brief := answer["run"].(doctorRun).Probes[0]
|
||||
if len(brief.Found) != overviewFindings || brief.FoundCount != 689 || !strings.Contains(brief.More, "probe=D14") {
|
||||
t.Fatalf("the verdict carries %d findings of D14, count %d, more %q", len(brief.Found), brief.FoundCount, brief.More)
|
||||
}
|
||||
if len(run.Probes[0].Found) != 689 {
|
||||
t.Fatal("the overview cut the run it was made from")
|
||||
}
|
||||
whole, err := verdictAnswer(run, time.Now(), "d14")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p := whole["run"].(doctorRun).Probes; len(p) != 1 || len(p[0].Found) != 689 || p[0].More != "" {
|
||||
t.Fatalf("doctor probe=D14 answered %+v", p)
|
||||
}
|
||||
if _, err := verdictAnswer(run, time.Now(), "D99"); err == nil {
|
||||
t.Fatal("a probe the run does not have was answered")
|
||||
}
|
||||
if !strings.Contains(doctorText(answer), "probe=D14` gives them all") {
|
||||
t.Fatalf("the text does not say how to read them all:\n%s", doctorText(answer))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestUnreadableConditionsAreNotAWellMesh(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
_, store := withConditionsInMemory(t)
|
||||
store.SetFail(errors.New("the bus is away"))
|
||||
asked, err := theThreeQuestions(t.Context(), open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if asked.well() || asked.conditionsUnread == "" {
|
||||
t.Fatalf("well with its conditions unread: %+v", asked.conditionsUnread)
|
||||
}
|
||||
said := printed(t, func() error { return printStatus(asked) })
|
||||
if !strings.HasPrefix(said, "the open conditions could NOT be read") || strings.Contains(said, "no open conditions") {
|
||||
t.Fatalf("%s", said)
|
||||
}
|
||||
store.SetFail(nil)
|
||||
asked, _ = theThreeQuestions(t.Context(), open)
|
||||
said = printed(t, func() error { return printStatus(asked) })
|
||||
if asked.well() && !strings.Contains(said, "no open conditions;") {
|
||||
t.Fatalf("the all-well sentence does not say no conditions are open:\n%s", said)
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -83,25 +82,3 @@ func TestASilenceThroughTheVerbHoldsAndTheConditionStaysOpen(t *testing.T) {
|
||||
}
|
||||
|
||||
// **Conditions that cannot be read are not none open**: status says so, and is not well.
|
||||
func TestUnreadableConditionsAreNotAWellMesh(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
_, store := withConditionsInMemory(t)
|
||||
store.SetFail(errors.New("the bus is away"))
|
||||
asked, err := theThreeQuestions(t.Context(), open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if asked.well() || asked.conditionsUnread == "" {
|
||||
t.Fatalf("well with its conditions unread: %+v", asked.conditionsUnread)
|
||||
}
|
||||
said := printed(t, func() error { return printStatus(asked) })
|
||||
if !strings.HasPrefix(said, "the open conditions could NOT be read") || strings.Contains(said, "no open conditions") {
|
||||
t.Fatalf("%s", said)
|
||||
}
|
||||
store.SetFail(nil)
|
||||
asked, _ = theThreeQuestions(t.Context(), open)
|
||||
said = printed(t, func() error { return printStatus(asked) })
|
||||
if asked.well() && !strings.Contains(said, "no open conditions;") {
|
||||
t.Fatalf("the all-well sentence does not say no conditions are open:\n%s", said)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,467 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
func aConfiguredMesh(t *testing.T) (*stores, configuredMesh, map[string]map[string]any) {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
for i, name := range []string{"workstation", "desktop"} {
|
||||
record, err := inv.AddNode(ctx, name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.SetPlace(ctx, name, name+".example:51820", "here", false, fmt.Sprintf("10.77.0.%d", i+3)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordProfile(ctx, record.ID, map[string]any{"capabilities": []any{
|
||||
map[string]any{"name": "container-runtime", "present": true},
|
||||
map[string]any{"name": "systemd", "present": true}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, name, overlay.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "config-moved", "mesh.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var w configuredMesh
|
||||
if err := json.Unmarshal(raw, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
list, err := catalogue.ParsePredefined(w.Predefined)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
catalogue.UsePredefined(list)
|
||||
served := map[string]map[string]any{}
|
||||
for _, text := range w.Modules {
|
||||
var shape map[string]json.RawMessage
|
||||
if err := json.Unmarshal(text, &shape); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var name string
|
||||
_ = json.Unmarshal(shape["module"], &name)
|
||||
if s, ok := shape["serves-checked"]; ok {
|
||||
var serves map[string]any
|
||||
if err := json.Unmarshal(s, &serves); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
served[name] = serves
|
||||
delete(shape, "serves-checked")
|
||||
}
|
||||
text, err := json.Marshal(shape)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(text)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
register(t, open, m)
|
||||
}
|
||||
for _, l := range w.Layers {
|
||||
values, err := json.Marshal(l.Values)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if l.Node == "" {
|
||||
inv.ExecForTest(t, `insert into settings (node, module, values) values (null, $1, $2)`, l.Module, string(values))
|
||||
continue
|
||||
}
|
||||
inv.ExecForTest(t, `insert into settings (node, module, values) select id, $2, $3 from node where name = $1`,
|
||||
l.Node, l.Module, string(values))
|
||||
}
|
||||
for _, module := range movedKeys(w.On) {
|
||||
for _, node := range w.On[module] {
|
||||
if _, err := assign(ctx, open, node, module); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", node, module, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return open, w, served
|
||||
}
|
||||
|
||||
func TestEveryMachineIsToldTheSameAfterTheMoveOfConfig(t *testing.T) {
|
||||
open, w, served := aConfiguredMesh(t)
|
||||
ctx := t.Context()
|
||||
readBackTries = 0
|
||||
t.Cleanup(func() { readBackTries = 15 })
|
||||
inv := open.inventory
|
||||
config := &standInMeshConfig{open: open, values: map[string]any{}}
|
||||
inv.ReadsConfigFrom(config.laid)
|
||||
before := toldEveryMachine(t, open, served, w.On)
|
||||
listedBefore := effectiveEverywhere(t, open, w)
|
||||
factsBefore, err := gatherFacts(ctx, open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
asked, err := runConfigMove(ctx, inv, config, false, "the move of config, run by a test", "")
|
||||
if err != nil || !asked.DryRun || asked.Switched || len(asked.Problems) > 0 || len(config.sets) > 0 {
|
||||
t.Fatalf("the move, not confirmed, says %+v (%v) and set %v", asked, err, config.sets)
|
||||
}
|
||||
// nodered's live layer says the file's name for its time zone: set under the config key (issue 751).
|
||||
if s := setAt(asked, "module.anchor.nodered.time-zone"); len(s) != 1 || s[0].Value != "Europe/Brussels" ||
|
||||
!strings.HasSuffix(s[0].From[0], ": timeZone") {
|
||||
t.Errorf("nodered's time zone moves as %+v", s)
|
||||
}
|
||||
for _, s := range asked.Sets {
|
||||
if s.Key == "timeZone" {
|
||||
t.Errorf("a key is set under the file's name: %+v", s)
|
||||
}
|
||||
}
|
||||
// claude-code's whole-mesh managed settings become mesh config for agent-managed-settings (ADR 0348's
|
||||
// consequences), and its whole-mesh layer sets no module value.
|
||||
if s := setAt(asked, "mesh.agent-managed-settings"); len(s) != 1 || s[0].From[0] != "claude-code on the whole mesh: managed_settings" {
|
||||
t.Errorf("claude-code's managed settings move as %+v", s)
|
||||
}
|
||||
for _, node := range fourMachines {
|
||||
if s := setAt(asked, "module."+node+".claude-code.managed_settings"); len(s) > 0 {
|
||||
t.Errorf("claude-code's managed settings are also set on %s: %+v", node, s)
|
||||
}
|
||||
}
|
||||
// A whole-mesh layer of a value no predefined key holds goes to each machine it is assigned to, under that
|
||||
// machine's layer: power's on the workstation is the workstation's, its own elsewhere.
|
||||
if s := setAt(asked, "module.workstation.power.handle-power-key"); len(s) != 1 || s[0].Value != "suspend" {
|
||||
t.Errorf("power's key on the workstation moves as %+v", s)
|
||||
}
|
||||
if s := setAt(asked, "module.anchor.power.handle-power-key"); len(s) != 1 || s[0].Value != "poweroff" {
|
||||
t.Errorf("power's key on the anchor moves as %+v", s)
|
||||
}
|
||||
for _, s := range asked.Sets {
|
||||
if s.Level == "node" {
|
||||
t.Errorf("a machine's layer became node config: %+v", s)
|
||||
}
|
||||
}
|
||||
var stays []string
|
||||
for _, s := range asked.Stays {
|
||||
stays = append(stays, strings.Fields(s)[0])
|
||||
}
|
||||
sort.Strings(stays)
|
||||
if want := []string{"bazarr", "bazarr", "bookshelf", "jackett", "kernel", "kometa", "lidarr", "nzbget", "ombi",
|
||||
"plex", "qbittorrent", "radarr", "sonarr", "tautulli"}; len(stays) < 10 || !containsString(stays, "kernel") {
|
||||
t.Errorf("the layers that stay are %v, want those of %v", stays, want)
|
||||
}
|
||||
if after := toldEveryMachine(t, open, served, w.On); !reflect.DeepEqual(after, before) {
|
||||
t.Fatal("a dry run changed what a machine is told")
|
||||
}
|
||||
|
||||
moved, err := runConfigMove(ctx, inv, config, true, "the move of config, run by a test", "release 57 is delivered")
|
||||
if err != nil || moved.DryRun || !moved.Switched {
|
||||
t.Fatalf("the move says %+v (%v)", moved, err)
|
||||
}
|
||||
if len(config.sets) != len(asked.Sets) {
|
||||
t.Fatalf("the move set %d values, and said it would set %d", len(config.sets), len(asked.Sets))
|
||||
}
|
||||
if held := inv.QueryForTest(t, `select module from settings s where values <> '{}'
|
||||
and module in (select name from module where manifest ? 'config')`); len(held) != 0 {
|
||||
t.Fatalf("a layer of a module in the config form still holds values: %v", held)
|
||||
}
|
||||
after := toldEveryMachine(t, open, served, w.On)
|
||||
for _, node := range fourMachines {
|
||||
if after[node] != before[node] {
|
||||
t.Errorf("%s is told something else after the move:\n%s\nwas:\n%s", node, after[node], before[node])
|
||||
}
|
||||
}
|
||||
if listedAfter := effectiveEverywhere(t, open, w); !reflect.DeepEqual(listedAfter, listedBefore) {
|
||||
t.Errorf("settings lists other values after the move:\n%v\nwas:\n%v", listedAfter, listedBefore)
|
||||
}
|
||||
// And config lists what settings listed: each module's resolved value on each machine is the value settings
|
||||
// said before the move.
|
||||
resolved, err := config.Resolved(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range resolved {
|
||||
said, listed := listedBefore[r.Node+"."+r.Module+"."+r.Key]
|
||||
if r.Source == "unset" {
|
||||
if listed && said != nil {
|
||||
t.Errorf("%s of %s on %s is unset in config, and settings said %v", r.Key, r.Module, r.Node, said)
|
||||
}
|
||||
continue
|
||||
}
|
||||
var v any
|
||||
_ = json.Unmarshal(r.Value, &v)
|
||||
if !listed || !reflect.DeepEqual(v, said) {
|
||||
t.Errorf("config resolves %s of %s on %s as %v, and settings said %v", r.Key, r.Module, r.Node, v, said)
|
||||
}
|
||||
}
|
||||
|
||||
// The facts snapshot carries mesh-config's values for the merge gate, as each machine's layer of each module, and
|
||||
// each machine's declaration composes to the digest it did before the move.
|
||||
factsAfter, err := gatherFacts(ctx, open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Machines are named in a snapshot by stand-ins of its own; each is found by what it composes.
|
||||
digests := func(f snapshotFacts) ([]string, string) {
|
||||
var out []string
|
||||
zone := ""
|
||||
for _, m := range f.Machines {
|
||||
out = append(out, m.Declaration.Digest)
|
||||
for _, l := range m.Settings {
|
||||
if l.Module == "nodered" {
|
||||
zone = compactJSON(l.Values["time-zone"])
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, zone
|
||||
}
|
||||
was, _ := digests(factsBefore)
|
||||
now, zone := digests(factsAfter)
|
||||
if zone != `"Europe/Brussels"` {
|
||||
t.Errorf("the facts snapshot does not carry nodered's time zone from mesh-config: %s", zone)
|
||||
}
|
||||
if len(now) != len(fourMachines) || now[0] == "" || !reflect.DeepEqual(now, was) {
|
||||
t.Errorf("the declarations in the facts snapshot are %v after the move, %v before", now, was)
|
||||
}
|
||||
for _, s := range factsAfter.Settings {
|
||||
if _, isConfig := map[string]bool{"betterbird": true, "claude-code": true, "power": true, "xdg": true}[s.Module]; isConfig &&
|
||||
len(s.Values) > 0 {
|
||||
t.Errorf("the facts snapshot still carries the whole mesh's layer of %s: %v", s.Module, s.Values)
|
||||
}
|
||||
}
|
||||
|
||||
config.sets = nil
|
||||
again, err := runConfigMove(ctx, inv, config, true, "the move of config, run by a test", "again")
|
||||
if err != nil || len(config.sets) != 0 || len(again.Moved) != 0 {
|
||||
t.Fatalf("asked again, the move set %v and says %+v (%v)", config.sets, again, err)
|
||||
}
|
||||
if after := toldEveryMachine(t, open, served, w.On); !reflect.DeepEqual(after, before) {
|
||||
t.Fatal("the move asked again changed what a machine is told")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheMoveOfConfigRefusesWhatWouldChangeAMachine(t *testing.T) {
|
||||
open, _, _ := aConfiguredMesh(t)
|
||||
ctx := t.Context()
|
||||
readBackTries = 0
|
||||
t.Cleanup(func() { readBackTries = 15 })
|
||||
inv := open.inventory
|
||||
config := &standInMeshConfig{open: open, values: map[string]any{}}
|
||||
inv.ReadsConfigFrom(config.laid)
|
||||
|
||||
inv.ExecForTest(t, `update settings set values = values || '{"endpoints": {"web": {"reach": "public"}}}'
|
||||
where module = 'nodered'`)
|
||||
inv.ExecForTest(t, `update settings set values = values || '{"colour": "blue"}' where module = 'unifi'`)
|
||||
inv.ExecForTest(t, `update settings set values = '{"font-size": 900}' where module = 'dunst'`)
|
||||
answer, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered")
|
||||
if err == nil || answer.Switched || len(config.sets) > 0 {
|
||||
t.Fatalf("the move went ahead: %+v (%v), set %v", answer, err, config.sets)
|
||||
}
|
||||
for _, want := range []string{"nodered on anchor still holds placement (endpoints): run move-placement first",
|
||||
`unifi on anchor says "colour", which unifi does not declare`, "font-size at dunst's own value on workstation"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not say %q: %v", want, err)
|
||||
}
|
||||
}
|
||||
inv.ExecForTest(t, `update settings set values = values - 'endpoints' where module = 'nodered'`)
|
||||
inv.ExecForTest(t, `update settings set values = values - 'colour' where module = 'unifi'`)
|
||||
inv.ExecForTest(t, `update settings set values = '{"font-size": 13, "width": 500}' where module = 'dunst'`)
|
||||
|
||||
// Set in mesh-config before the move, on a module whose layers say nothing of it: the switch would lay it.
|
||||
config.values["module.anchor.grafana."+movedKeys(func() map[string]catalogue.SettingDeclaration {
|
||||
shelf, _ := inv.Catalogue(ctx)
|
||||
return shelf["grafana"].Settings
|
||||
}())[0]] = "set before the move"
|
||||
answer, err = runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered")
|
||||
if err == nil || !strings.Contains(err.Error(), "grafana on anchor resolves") || answer.Switched {
|
||||
t.Fatalf("a value no layer says was laid by the switch: %+v (%v)", answer, err)
|
||||
}
|
||||
if moved, _ := inv.ConfigMoved(ctx); moved.Moved {
|
||||
t.Fatal("the switch is on after a refused move")
|
||||
}
|
||||
if held := inv.QueryForTest(t, `select count(*) from settings where values <> '{}'`); held[0][0] == "0" {
|
||||
t.Fatal("a refused move emptied the layers")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSettingsVerbForwardsToConfigOnceTheValuesMoved(t *testing.T) {
|
||||
open, w, served := aConfiguredMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
readBackTries = 0
|
||||
t.Cleanup(func() { readBackTries = 15 })
|
||||
config := &standInMeshConfig{open: open, values: map[string]any{}}
|
||||
inv.ReadsConfigFrom(config.laid)
|
||||
savedRead, savedAsk := readConfigValues, configAsk
|
||||
readConfigValues, configAsk = config.laid, config.ask
|
||||
t.Cleanup(func() { readConfigValues, configAsk = savedRead, savedAsk })
|
||||
if _, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
shown := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"show", "nodered", "--node", "anchor"}) })
|
||||
for _, want := range []string{`"time-zone": "Europe/Brussels"`, `time-zone = "Europe/Brussels" (module)`,
|
||||
"mesh-config.config", `"mqtt": {`} {
|
||||
if !strings.Contains(shown, want) {
|
||||
t.Errorf("settings show does not say %q:\n%s", want, shown)
|
||||
}
|
||||
}
|
||||
if strings.Contains(shown, "timeZone") {
|
||||
t.Errorf("settings show still says the file's name:\n%s", shown)
|
||||
}
|
||||
|
||||
config.asked = nil
|
||||
said := stdoutOf(t, func() error {
|
||||
return settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12, "width": 500}`, "--node", "workstation"})
|
||||
})
|
||||
if config.values["module.workstation.dunst.font-size"] != float64(12) || !strings.Contains(said, "~ font-size") ||
|
||||
containsString(config.asked, "set module.workstation.dunst.width") {
|
||||
t.Fatalf("settings set reached mesh-config as %v (%v):\n%s", config.values, config.asked, said)
|
||||
}
|
||||
told := toldEveryMachine(t, open, served, w.On)["workstation"]
|
||||
if !strings.Contains(told, "12") {
|
||||
t.Error("the workstation is not told the font size set through settings")
|
||||
}
|
||||
if err := settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12}`, "--node", "workstation"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "would no longer set width") {
|
||||
t.Fatalf("a set dropping a value without --replace: %v", err)
|
||||
}
|
||||
if err := settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12}`}); err == nil ||
|
||||
!strings.Contains(err.Error(), "no own value for the whole mesh") {
|
||||
t.Fatalf("a module value for the whole mesh: %v", err)
|
||||
}
|
||||
|
||||
t.Setenv(verbVar, "settings")
|
||||
for _, c := range []struct{ module, node, values, key, held, why string }{
|
||||
// A file root reads: refused as before the move (novox/hq issue 339).
|
||||
{"hostname", "anchor", `{"hostname": "elsewhere"}`, "module.anchor.hostname.hostname", "Ace", "issue 339"},
|
||||
{"messenger", "laptop", `{"desktop-machines": ["g14", "shanks"], "time-zone": "UTC"}`,
|
||||
"module.laptop.messenger.time-zone", "Europe/Brussels", "issue 339"},
|
||||
} {
|
||||
err := settingsCommand(ctx, []string{"set", c.module, c.values, "--node", c.node})
|
||||
if err == nil || !strings.Contains(err.Error(), c.why) || config.values[c.key] != c.held {
|
||||
t.Fatalf("a trusted value of %s through a verb: %v, holds %v", c.module, err, config.values[c.key])
|
||||
}
|
||||
}
|
||||
// And a trusted value no file makes the terminal's is refused as trusted, since mesh-config takes the controller's
|
||||
// word for it (ADR 0348 §8).
|
||||
if err := refuseTrustedThroughAVerb(ctx, inv, "messenger", "laptop", []string{"time-zone"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "ADR 0348 §8") {
|
||||
t.Fatalf("a trusted value through a verb: %v", err)
|
||||
}
|
||||
if err := refuseTrustedThroughAVerb(ctx, inv, "dunst", "workstation", []string{"width"}); err != nil {
|
||||
t.Fatalf("a value nobody trusts through a verb: %v", err)
|
||||
}
|
||||
stdoutOf(t, func() error {
|
||||
return settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12, "width": 600}`, "--node", "workstation"})
|
||||
})
|
||||
if config.values["module.workstation.dunst.width"] != float64(600) {
|
||||
t.Fatalf("a value nobody trusts was not set through a verb: %v", config.values["module.workstation.dunst.width"])
|
||||
}
|
||||
t.Setenv(verbVar, "")
|
||||
|
||||
stdoutOf(t, func() error { return settingsCommand(ctx, []string{"clear", "dunst", "--node", "workstation"}) })
|
||||
for _, k := range []string{"font-size", "width"} {
|
||||
if _, held := config.values["module.workstation.dunst."+k]; held {
|
||||
t.Errorf("settings clear left dunst's %s on the workstation", k)
|
||||
}
|
||||
}
|
||||
|
||||
config.asked = nil
|
||||
proposed := stdoutOf(t, func() error {
|
||||
return settingsCommand(ctx, []string{"propose", "hostname", `{"hostname": "elsewhere"}`, "--node", "anchor"})
|
||||
})
|
||||
if !containsString(config.asked, "propose module.anchor.hostname.hostname") || !strings.Contains(proposed, "p-1") {
|
||||
t.Fatalf("settings propose reached mesh-config as %v:\n%s", config.asked, proposed)
|
||||
}
|
||||
|
||||
config.asked = nil
|
||||
kept := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"show", "kernel", "--node", "laptop"}) })
|
||||
if !strings.Contains(kept, "transparent-huge-pages") || len(config.asked) > 0 {
|
||||
t.Fatalf("a module in the settings form is shown as %q, asking mesh-config %v", kept, config.asked)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnApprovalRaisedBeforeTheMoveOfConfigIsRefusedAfterIt(t *testing.T) {
|
||||
open, _, _ := aConfiguredMesh(t)
|
||||
ctx := t.Context()
|
||||
readBackTries = 0
|
||||
t.Cleanup(func() { readBackTries = 15 })
|
||||
inv := open.inventory
|
||||
config := &standInMeshConfig{open: open, values: map[string]any{}}
|
||||
inv.ReadsConfigFrom(config.laid)
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := movedKeys(func() map[string]catalogue.SettingDeclaration { return shelf["grafana"].Settings }())[0]
|
||||
values := map[string]any{key: "raised before the move"}
|
||||
p := settingsProposal{Module: "grafana", Node: "anchor", Values: values, From: "g14/claude-code", At: time.Now(),
|
||||
Digest: layerDigest(values), BeforeDigest: layerDigest(nil)}
|
||||
if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has {
|
||||
t.Fatalf("grafana has a layer on anchor before the test: %v %v", has, err)
|
||||
}
|
||||
if moved, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil || !moved.Switched {
|
||||
t.Fatalf("the move says %+v (%v)", moved, err)
|
||||
}
|
||||
if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads {
|
||||
t.Fatalf("grafana does not read config after the move: %v %v", reads, err)
|
||||
}
|
||||
_, err = setLayerIn(open)(ctx, p, "s30", "approved by the operator")
|
||||
if err == nil || !strings.Contains(err.Error(), "moved to mesh-config since the operator was shown the change") ||
|
||||
!strings.Contains(err.Error(), "propose it again") {
|
||||
t.Fatalf("the stale approval: %v", err)
|
||||
}
|
||||
if layer, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has && len(layer) > 0 {
|
||||
t.Errorf("the refused approval wrote the layer: %v %v %v", layer, has, err)
|
||||
}
|
||||
if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads {
|
||||
t.Errorf("the refused approval turned ReadsConfig off: %v %v", reads, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPlacementOnlyApprovalRaisedBeforeTheMoveIsSetAfterIt(t *testing.T) {
|
||||
open, _, _ := aConfiguredMesh(t)
|
||||
ctx := t.Context()
|
||||
readBackTries = 0
|
||||
t.Cleanup(func() { readBackTries = 15 })
|
||||
inv := open.inventory
|
||||
config := &standInMeshConfig{open: open, values: map[string]any{}}
|
||||
inv.ReadsConfigFrom(config.laid)
|
||||
values := map[string]any{catalogue.PlacesSetting: map[string]any{"data": "/srv/grafana-data"}}
|
||||
p := settingsProposal{Module: "grafana", Node: "anchor", Values: values, From: "g14/claude-code", At: time.Now(),
|
||||
Digest: layerDigest(values), BeforeDigest: layerDigest(nil)}
|
||||
if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has {
|
||||
t.Fatalf("grafana has a layer on anchor before the test: %v %v", has, err)
|
||||
}
|
||||
if moved, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil || !moved.Switched {
|
||||
t.Fatalf("the move says %+v (%v)", moved, err)
|
||||
}
|
||||
if _, err := setLayerIn(open)(ctx, p, "s31", "approved by the operator"); err != nil {
|
||||
t.Fatalf("the placement-only approval after the move: %v", err)
|
||||
}
|
||||
if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || !has {
|
||||
t.Errorf("the approval did not set the layer: %v %v", has, err)
|
||||
}
|
||||
if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads {
|
||||
t.Errorf("the approval turned ReadsConfig off: %v %v", reads, err)
|
||||
}
|
||||
}
|
||||
@@ -4,18 +4,12 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
snapshot "github.com/novox/mesh-controller/internal/facts"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// configuredMesh is every live settings layer (issue 664) — read with the settings verb on 2026-10-15, on the four
|
||||
@@ -42,94 +36,6 @@ var fourMachines = []string{"anchor", "laptop", "workstation", "desktop"}
|
||||
|
||||
// aConfiguredMesh is aMesh with two machines more, every module of the configured mesh registered and assigned, and
|
||||
// its layers held as the store holds them once move-placement has run.
|
||||
func aConfiguredMesh(t *testing.T) (*stores, configuredMesh, map[string]map[string]any) {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
for i, name := range []string{"workstation", "desktop"} {
|
||||
record, err := inv.AddNode(ctx, name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.SetPlace(ctx, name, name+".example:51820", "here", false, fmt.Sprintf("10.77.0.%d", i+3)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordProfile(ctx, record.ID, map[string]any{"capabilities": []any{
|
||||
map[string]any{"name": "container-runtime", "present": true},
|
||||
map[string]any{"name": "systemd", "present": true}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, name, overlay.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "config-moved", "mesh.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var w configuredMesh
|
||||
if err := json.Unmarshal(raw, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
list, err := catalogue.ParsePredefined(w.Predefined)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
catalogue.UsePredefined(list)
|
||||
served := map[string]map[string]any{}
|
||||
for _, text := range w.Modules {
|
||||
var shape map[string]json.RawMessage
|
||||
if err := json.Unmarshal(text, &shape); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var name string
|
||||
_ = json.Unmarshal(shape["module"], &name)
|
||||
if s, ok := shape["serves-checked"]; ok {
|
||||
var serves map[string]any
|
||||
if err := json.Unmarshal(s, &serves); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
served[name] = serves
|
||||
delete(shape, "serves-checked")
|
||||
}
|
||||
text, err := json.Marshal(shape)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(text)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
register(t, open, m)
|
||||
}
|
||||
for _, l := range w.Layers {
|
||||
values, err := json.Marshal(l.Values)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if l.Node == "" {
|
||||
inv.ExecForTest(t, `insert into settings (node, module, values) values (null, $1, $2)`, l.Module, string(values))
|
||||
continue
|
||||
}
|
||||
inv.ExecForTest(t, `insert into settings (node, module, values) select id, $2, $3 from node where name = $1`,
|
||||
l.Node, l.Module, string(values))
|
||||
}
|
||||
for _, module := range movedKeys(w.On) {
|
||||
for _, node := range w.On[module] {
|
||||
if _, err := assign(ctx, open, node, module); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", node, module, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return open, w, served
|
||||
}
|
||||
|
||||
// standInMeshConfig is mesh-config as this test holds it: each value at its level, and resolved from the catalogue
|
||||
// the store holds as mesh-config resolves (novox/hq ADR 0348 §2): the module's own value on the node, else node
|
||||
@@ -330,205 +236,11 @@ func setAt(a configMoveAnswer, key string) []configSet {
|
||||
// and where from, and sets nothing; confirmed, it sets them in mesh-config, reads them back, empties the layers it
|
||||
// moved and switches. Each machine is then told byte for byte what it was told before, every module serves the same,
|
||||
// and `settings` lists the same values `config` resolves. Asked again, it sets nothing.
|
||||
func TestEveryMachineIsToldTheSameAfterTheMoveOfConfig(t *testing.T) {
|
||||
open, w, served := aConfiguredMesh(t)
|
||||
ctx := t.Context()
|
||||
readBackTries = 0
|
||||
t.Cleanup(func() { readBackTries = 15 })
|
||||
inv := open.inventory
|
||||
config := &standInMeshConfig{open: open, values: map[string]any{}}
|
||||
inv.ReadsConfigFrom(config.laid)
|
||||
before := toldEveryMachine(t, open, served, w.On)
|
||||
listedBefore := effectiveEverywhere(t, open, w)
|
||||
factsBefore, err := gatherFacts(ctx, open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
asked, err := runConfigMove(ctx, inv, config, false, "the move of config, run by a test", "")
|
||||
if err != nil || !asked.DryRun || asked.Switched || len(asked.Problems) > 0 || len(config.sets) > 0 {
|
||||
t.Fatalf("the move, not confirmed, says %+v (%v) and set %v", asked, err, config.sets)
|
||||
}
|
||||
// nodered's live layer says the file's name for its time zone: set under the config key (issue 751).
|
||||
if s := setAt(asked, "module.anchor.nodered.time-zone"); len(s) != 1 || s[0].Value != "Europe/Brussels" ||
|
||||
!strings.HasSuffix(s[0].From[0], ": timeZone") {
|
||||
t.Errorf("nodered's time zone moves as %+v", s)
|
||||
}
|
||||
for _, s := range asked.Sets {
|
||||
if s.Key == "timeZone" {
|
||||
t.Errorf("a key is set under the file's name: %+v", s)
|
||||
}
|
||||
}
|
||||
// claude-code's whole-mesh managed settings become mesh config for agent-managed-settings (ADR 0348's
|
||||
// consequences), and its whole-mesh layer sets no module value.
|
||||
if s := setAt(asked, "mesh.agent-managed-settings"); len(s) != 1 || s[0].From[0] != "claude-code on the whole mesh: managed_settings" {
|
||||
t.Errorf("claude-code's managed settings move as %+v", s)
|
||||
}
|
||||
for _, node := range fourMachines {
|
||||
if s := setAt(asked, "module."+node+".claude-code.managed_settings"); len(s) > 0 {
|
||||
t.Errorf("claude-code's managed settings are also set on %s: %+v", node, s)
|
||||
}
|
||||
}
|
||||
// A whole-mesh layer of a value no predefined key holds goes to each machine it is assigned to, under that
|
||||
// machine's layer: power's on the workstation is the workstation's, its own elsewhere.
|
||||
if s := setAt(asked, "module.workstation.power.handle-power-key"); len(s) != 1 || s[0].Value != "suspend" {
|
||||
t.Errorf("power's key on the workstation moves as %+v", s)
|
||||
}
|
||||
if s := setAt(asked, "module.anchor.power.handle-power-key"); len(s) != 1 || s[0].Value != "poweroff" {
|
||||
t.Errorf("power's key on the anchor moves as %+v", s)
|
||||
}
|
||||
for _, s := range asked.Sets {
|
||||
if s.Level == "node" {
|
||||
t.Errorf("a machine's layer became node config: %+v", s)
|
||||
}
|
||||
}
|
||||
var stays []string
|
||||
for _, s := range asked.Stays {
|
||||
stays = append(stays, strings.Fields(s)[0])
|
||||
}
|
||||
sort.Strings(stays)
|
||||
if want := []string{"bazarr", "bazarr", "bookshelf", "jackett", "kernel", "kometa", "lidarr", "nzbget", "ombi",
|
||||
"plex", "qbittorrent", "radarr", "sonarr", "tautulli"}; len(stays) < 10 || !containsString(stays, "kernel") {
|
||||
t.Errorf("the layers that stay are %v, want those of %v", stays, want)
|
||||
}
|
||||
if after := toldEveryMachine(t, open, served, w.On); !reflect.DeepEqual(after, before) {
|
||||
t.Fatal("a dry run changed what a machine is told")
|
||||
}
|
||||
|
||||
moved, err := runConfigMove(ctx, inv, config, true, "the move of config, run by a test", "release 57 is delivered")
|
||||
if err != nil || moved.DryRun || !moved.Switched {
|
||||
t.Fatalf("the move says %+v (%v)", moved, err)
|
||||
}
|
||||
if len(config.sets) != len(asked.Sets) {
|
||||
t.Fatalf("the move set %d values, and said it would set %d", len(config.sets), len(asked.Sets))
|
||||
}
|
||||
if held := inv.QueryForTest(t, `select module from settings s where values <> '{}'
|
||||
and module in (select name from module where manifest ? 'config')`); len(held) != 0 {
|
||||
t.Fatalf("a layer of a module in the config form still holds values: %v", held)
|
||||
}
|
||||
after := toldEveryMachine(t, open, served, w.On)
|
||||
for _, node := range fourMachines {
|
||||
if after[node] != before[node] {
|
||||
t.Errorf("%s is told something else after the move:\n%s\nwas:\n%s", node, after[node], before[node])
|
||||
}
|
||||
}
|
||||
if listedAfter := effectiveEverywhere(t, open, w); !reflect.DeepEqual(listedAfter, listedBefore) {
|
||||
t.Errorf("settings lists other values after the move:\n%v\nwas:\n%v", listedAfter, listedBefore)
|
||||
}
|
||||
// And config lists what settings listed: each module's resolved value on each machine is the value settings
|
||||
// said before the move.
|
||||
resolved, err := config.Resolved(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range resolved {
|
||||
said, listed := listedBefore[r.Node+"."+r.Module+"."+r.Key]
|
||||
if r.Source == "unset" {
|
||||
if listed && said != nil {
|
||||
t.Errorf("%s of %s on %s is unset in config, and settings said %v", r.Key, r.Module, r.Node, said)
|
||||
}
|
||||
continue
|
||||
}
|
||||
var v any
|
||||
_ = json.Unmarshal(r.Value, &v)
|
||||
if !listed || !reflect.DeepEqual(v, said) {
|
||||
t.Errorf("config resolves %s of %s on %s as %v, and settings said %v", r.Key, r.Module, r.Node, v, said)
|
||||
}
|
||||
}
|
||||
|
||||
// The facts snapshot carries mesh-config's values for the merge gate, as each machine's layer of each module, and
|
||||
// each machine's declaration composes to the digest it did before the move.
|
||||
factsAfter, err := gatherFacts(ctx, open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Machines are named in a snapshot by stand-ins of its own; each is found by what it composes.
|
||||
digests := func(f snapshotFacts) ([]string, string) {
|
||||
var out []string
|
||||
zone := ""
|
||||
for _, m := range f.Machines {
|
||||
out = append(out, m.Declaration.Digest)
|
||||
for _, l := range m.Settings {
|
||||
if l.Module == "nodered" {
|
||||
zone = compactJSON(l.Values["time-zone"])
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, zone
|
||||
}
|
||||
was, _ := digests(factsBefore)
|
||||
now, zone := digests(factsAfter)
|
||||
if zone != `"Europe/Brussels"` {
|
||||
t.Errorf("the facts snapshot does not carry nodered's time zone from mesh-config: %s", zone)
|
||||
}
|
||||
if len(now) != len(fourMachines) || now[0] == "" || !reflect.DeepEqual(now, was) {
|
||||
t.Errorf("the declarations in the facts snapshot are %v after the move, %v before", now, was)
|
||||
}
|
||||
for _, s := range factsAfter.Settings {
|
||||
if _, isConfig := map[string]bool{"betterbird": true, "claude-code": true, "power": true, "xdg": true}[s.Module]; isConfig &&
|
||||
len(s.Values) > 0 {
|
||||
t.Errorf("the facts snapshot still carries the whole mesh's layer of %s: %v", s.Module, s.Values)
|
||||
}
|
||||
}
|
||||
|
||||
config.sets = nil
|
||||
again, err := runConfigMove(ctx, inv, config, true, "the move of config, run by a test", "again")
|
||||
if err != nil || len(config.sets) != 0 || len(again.Moved) != 0 {
|
||||
t.Fatalf("asked again, the move set %v and says %+v (%v)", config.sets, again, err)
|
||||
}
|
||||
if after := toldEveryMachine(t, open, served, w.On); !reflect.DeepEqual(after, before) {
|
||||
t.Fatal("the move asked again changed what a machine is told")
|
||||
}
|
||||
}
|
||||
|
||||
// **The move refuses what it cannot move as it composes, and sets nothing** (issue 664): a layer that still holds
|
||||
// placement (move-placement has not run), a key its module does not declare, and a value its declaration refuses —
|
||||
// each said by name; and a value mesh-config resolves from a level no layer says (it would be laid once the switch
|
||||
// is on) leaves every layer and the switch as they were.
|
||||
func TestTheMoveOfConfigRefusesWhatWouldChangeAMachine(t *testing.T) {
|
||||
open, _, _ := aConfiguredMesh(t)
|
||||
ctx := t.Context()
|
||||
readBackTries = 0
|
||||
t.Cleanup(func() { readBackTries = 15 })
|
||||
inv := open.inventory
|
||||
config := &standInMeshConfig{open: open, values: map[string]any{}}
|
||||
inv.ReadsConfigFrom(config.laid)
|
||||
|
||||
inv.ExecForTest(t, `update settings set values = values || '{"endpoints": {"web": {"reach": "public"}}}'
|
||||
where module = 'nodered'`)
|
||||
inv.ExecForTest(t, `update settings set values = values || '{"colour": "blue"}' where module = 'unifi'`)
|
||||
inv.ExecForTest(t, `update settings set values = '{"font-size": 900}' where module = 'dunst'`)
|
||||
answer, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered")
|
||||
if err == nil || answer.Switched || len(config.sets) > 0 {
|
||||
t.Fatalf("the move went ahead: %+v (%v), set %v", answer, err, config.sets)
|
||||
}
|
||||
for _, want := range []string{"nodered on anchor still holds placement (endpoints): run move-placement first",
|
||||
`unifi on anchor says "colour", which unifi does not declare`, "font-size at dunst's own value on workstation"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not say %q: %v", want, err)
|
||||
}
|
||||
}
|
||||
inv.ExecForTest(t, `update settings set values = values - 'endpoints' where module = 'nodered'`)
|
||||
inv.ExecForTest(t, `update settings set values = values - 'colour' where module = 'unifi'`)
|
||||
inv.ExecForTest(t, `update settings set values = '{"font-size": 13, "width": 500}' where module = 'dunst'`)
|
||||
|
||||
// Set in mesh-config before the move, on a module whose layers say nothing of it: the switch would lay it.
|
||||
config.values["module.anchor.grafana."+movedKeys(func() map[string]catalogue.SettingDeclaration {
|
||||
shelf, _ := inv.Catalogue(ctx)
|
||||
return shelf["grafana"].Settings
|
||||
}())[0]] = "set before the move"
|
||||
answer, err = runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered")
|
||||
if err == nil || !strings.Contains(err.Error(), "grafana on anchor resolves") || answer.Switched {
|
||||
t.Fatalf("a value no layer says was laid by the switch: %+v (%v)", answer, err)
|
||||
}
|
||||
if moved, _ := inv.ConfigMoved(ctx); moved.Moved {
|
||||
t.Fatal("the switch is on after a refused move")
|
||||
}
|
||||
if held := inv.QueryForTest(t, `select count(*) from settings where values <> '{}'`); held[0][0] == "0" {
|
||||
t.Fatal("a refused move emptied the layers")
|
||||
}
|
||||
}
|
||||
|
||||
// The verb's command line: a dry run unless confirmed, and a confirmed move says why, recorded as a person's step of
|
||||
// the release, never a repair (issue 664).
|
||||
@@ -614,172 +326,11 @@ func (c *standInMeshConfig) ask(ctx context.Context, args map[string]any) (json.
|
||||
// has run, a module in the config form is shown from mesh-config, its values set and cleared there as its own on the
|
||||
// machine (placement still beside the assignment), and proposed through it; a trusted value is refused through a
|
||||
// verb, a module value for the whole mesh is refused, and a module still in the settings form keeps today's verb.
|
||||
func TestTheSettingsVerbForwardsToConfigOnceTheValuesMoved(t *testing.T) {
|
||||
open, w, served := aConfiguredMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
readBackTries = 0
|
||||
t.Cleanup(func() { readBackTries = 15 })
|
||||
config := &standInMeshConfig{open: open, values: map[string]any{}}
|
||||
inv.ReadsConfigFrom(config.laid)
|
||||
savedRead, savedAsk := readConfigValues, configAsk
|
||||
readConfigValues, configAsk = config.laid, config.ask
|
||||
t.Cleanup(func() { readConfigValues, configAsk = savedRead, savedAsk })
|
||||
if _, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
shown := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"show", "nodered", "--node", "anchor"}) })
|
||||
for _, want := range []string{`"time-zone": "Europe/Brussels"`, `time-zone = "Europe/Brussels" (module)`,
|
||||
"mesh-config.config", `"mqtt": {`} {
|
||||
if !strings.Contains(shown, want) {
|
||||
t.Errorf("settings show does not say %q:\n%s", want, shown)
|
||||
}
|
||||
}
|
||||
if strings.Contains(shown, "timeZone") {
|
||||
t.Errorf("settings show still says the file's name:\n%s", shown)
|
||||
}
|
||||
|
||||
config.asked = nil
|
||||
said := stdoutOf(t, func() error {
|
||||
return settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12, "width": 500}`, "--node", "workstation"})
|
||||
})
|
||||
if config.values["module.workstation.dunst.font-size"] != float64(12) || !strings.Contains(said, "~ font-size") ||
|
||||
containsString(config.asked, "set module.workstation.dunst.width") {
|
||||
t.Fatalf("settings set reached mesh-config as %v (%v):\n%s", config.values, config.asked, said)
|
||||
}
|
||||
told := toldEveryMachine(t, open, served, w.On)["workstation"]
|
||||
if !strings.Contains(told, "12") {
|
||||
t.Error("the workstation is not told the font size set through settings")
|
||||
}
|
||||
if err := settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12}`, "--node", "workstation"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "would no longer set width") {
|
||||
t.Fatalf("a set dropping a value without --replace: %v", err)
|
||||
}
|
||||
if err := settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12}`}); err == nil ||
|
||||
!strings.Contains(err.Error(), "no own value for the whole mesh") {
|
||||
t.Fatalf("a module value for the whole mesh: %v", err)
|
||||
}
|
||||
|
||||
t.Setenv(verbVar, "settings")
|
||||
for _, c := range []struct{ module, node, values, key, held, why string }{
|
||||
// A file root reads: refused as before the move (novox/hq issue 339).
|
||||
{"hostname", "anchor", `{"hostname": "elsewhere"}`, "module.anchor.hostname.hostname", "Ace", "issue 339"},
|
||||
{"messenger", "laptop", `{"desktop-machines": ["g14", "shanks"], "time-zone": "UTC"}`,
|
||||
"module.laptop.messenger.time-zone", "Europe/Brussels", "issue 339"},
|
||||
} {
|
||||
err := settingsCommand(ctx, []string{"set", c.module, c.values, "--node", c.node})
|
||||
if err == nil || !strings.Contains(err.Error(), c.why) || config.values[c.key] != c.held {
|
||||
t.Fatalf("a trusted value of %s through a verb: %v, holds %v", c.module, err, config.values[c.key])
|
||||
}
|
||||
}
|
||||
// And a trusted value no file makes the terminal's is refused as trusted, since mesh-config takes the controller's
|
||||
// word for it (ADR 0348 §8).
|
||||
if err := refuseTrustedThroughAVerb(ctx, inv, "messenger", "laptop", []string{"time-zone"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "ADR 0348 §8") {
|
||||
t.Fatalf("a trusted value through a verb: %v", err)
|
||||
}
|
||||
if err := refuseTrustedThroughAVerb(ctx, inv, "dunst", "workstation", []string{"width"}); err != nil {
|
||||
t.Fatalf("a value nobody trusts through a verb: %v", err)
|
||||
}
|
||||
stdoutOf(t, func() error {
|
||||
return settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12, "width": 600}`, "--node", "workstation"})
|
||||
})
|
||||
if config.values["module.workstation.dunst.width"] != float64(600) {
|
||||
t.Fatalf("a value nobody trusts was not set through a verb: %v", config.values["module.workstation.dunst.width"])
|
||||
}
|
||||
t.Setenv(verbVar, "")
|
||||
|
||||
stdoutOf(t, func() error { return settingsCommand(ctx, []string{"clear", "dunst", "--node", "workstation"}) })
|
||||
for _, k := range []string{"font-size", "width"} {
|
||||
if _, held := config.values["module.workstation.dunst."+k]; held {
|
||||
t.Errorf("settings clear left dunst's %s on the workstation", k)
|
||||
}
|
||||
}
|
||||
|
||||
config.asked = nil
|
||||
proposed := stdoutOf(t, func() error {
|
||||
return settingsCommand(ctx, []string{"propose", "hostname", `{"hostname": "elsewhere"}`, "--node", "anchor"})
|
||||
})
|
||||
if !containsString(config.asked, "propose module.anchor.hostname.hostname") || !strings.Contains(proposed, "p-1") {
|
||||
t.Fatalf("settings propose reached mesh-config as %v:\n%s", config.asked, proposed)
|
||||
}
|
||||
|
||||
config.asked = nil
|
||||
kept := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"show", "kernel", "--node", "laptop"}) })
|
||||
if !strings.Contains(kept, "transparent-huge-pages") || len(config.asked) > 0 {
|
||||
t.Fatalf("a module in the settings form is shown as %q, asking mesh-config %v", kept, config.asked)
|
||||
}
|
||||
}
|
||||
|
||||
// **An approval for a module whose config has moved is refused, and writes nothing** (issue 830, novox/hq ADR 0348):
|
||||
// a proposal raised while grafana read its layer (it had none, so the layer is as the operator was shown it) and
|
||||
// approved after the move would put its values back into the layer, ReadsConfig would turn false, and the module
|
||||
// would compose from emptied layers everywhere.
|
||||
func TestAnApprovalRaisedBeforeTheMoveOfConfigIsRefusedAfterIt(t *testing.T) {
|
||||
open, _, _ := aConfiguredMesh(t)
|
||||
ctx := t.Context()
|
||||
readBackTries = 0
|
||||
t.Cleanup(func() { readBackTries = 15 })
|
||||
inv := open.inventory
|
||||
config := &standInMeshConfig{open: open, values: map[string]any{}}
|
||||
inv.ReadsConfigFrom(config.laid)
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := movedKeys(func() map[string]catalogue.SettingDeclaration { return shelf["grafana"].Settings }())[0]
|
||||
values := map[string]any{key: "raised before the move"}
|
||||
p := settingsProposal{Module: "grafana", Node: "anchor", Values: values, From: "g14/claude-code", At: time.Now(),
|
||||
Digest: layerDigest(values), BeforeDigest: layerDigest(nil)}
|
||||
if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has {
|
||||
t.Fatalf("grafana has a layer on anchor before the test: %v %v", has, err)
|
||||
}
|
||||
if moved, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil || !moved.Switched {
|
||||
t.Fatalf("the move says %+v (%v)", moved, err)
|
||||
}
|
||||
if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads {
|
||||
t.Fatalf("grafana does not read config after the move: %v %v", reads, err)
|
||||
}
|
||||
_, err = setLayerIn(open)(ctx, p, "s30", "approved by the operator")
|
||||
if err == nil || !strings.Contains(err.Error(), "moved to mesh-config since the operator was shown the change") ||
|
||||
!strings.Contains(err.Error(), "propose it again") {
|
||||
t.Fatalf("the stale approval: %v", err)
|
||||
}
|
||||
if layer, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has && len(layer) > 0 {
|
||||
t.Errorf("the refused approval wrote the layer: %v %v %v", layer, has, err)
|
||||
}
|
||||
if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads {
|
||||
t.Errorf("the refused approval turned ReadsConfig off: %v %v", reads, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **A proposal of placement only is set after the move, as before** (issue 830): forwardPropose sends it down the
|
||||
// old path on purpose, so refusing it would loop the operator (refused, proposed again, refused).
|
||||
func TestAPlacementOnlyApprovalRaisedBeforeTheMoveIsSetAfterIt(t *testing.T) {
|
||||
open, _, _ := aConfiguredMesh(t)
|
||||
ctx := t.Context()
|
||||
readBackTries = 0
|
||||
t.Cleanup(func() { readBackTries = 15 })
|
||||
inv := open.inventory
|
||||
config := &standInMeshConfig{open: open, values: map[string]any{}}
|
||||
inv.ReadsConfigFrom(config.laid)
|
||||
values := map[string]any{catalogue.PlacesSetting: map[string]any{"data": "/srv/grafana-data"}}
|
||||
p := settingsProposal{Module: "grafana", Node: "anchor", Values: values, From: "g14/claude-code", At: time.Now(),
|
||||
Digest: layerDigest(values), BeforeDigest: layerDigest(nil)}
|
||||
if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has {
|
||||
t.Fatalf("grafana has a layer on anchor before the test: %v %v", has, err)
|
||||
}
|
||||
if moved, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil || !moved.Switched {
|
||||
t.Fatalf("the move says %+v (%v)", moved, err)
|
||||
}
|
||||
if _, err := setLayerIn(open)(ctx, p, "s31", "approved by the operator"); err != nil {
|
||||
t.Fatalf("the placement-only approval after the move: %v", err)
|
||||
}
|
||||
if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || !has {
|
||||
t.Errorf("the approval did not set the layer: %v %v", has, err)
|
||||
}
|
||||
if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads {
|
||||
t.Errorf("the approval turned ReadsConfig off: %v %v", reads, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func aCoreMesh(t *testing.T, contract int) (*stores, *coreStandIn) {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
client := aClientModule()
|
||||
register(t, open, client)
|
||||
if _, err := assign(ctx, open, "laptop", client.Module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := issueProvisionerClient(ctx, open.inventory, client, "laptop", broker.Broker{Fingerprint: "ab"},
|
||||
"10.0.0.1:4222"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.RecordHealth(ctx, inventory.NodeHealth{Node: "laptop", Contract: contract,
|
||||
SaidAt: time.Now()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v := &coreStandIn{generation: 1}
|
||||
withVault(t, v)
|
||||
return open, v
|
||||
}
|
||||
|
||||
func TestTheSendCarriesTheVaultsBoxForACoreCredentialAndKeepsIt(t *testing.T) {
|
||||
open, vault := aCoreMesh(t, link.VaultBoxContract)
|
||||
key, err := open.inventory.SealingKeyOf(t.Context(), "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := clientsBroker(sentTo(t, open, "laptop"))
|
||||
if f == nil || f["sealed"] != nil || f[catalogue.VaultSealedField] == nil {
|
||||
t.Fatalf("the core credential is sent as %v: the vault's box, never the controller's seal", f)
|
||||
}
|
||||
if want := []string{"own/laptop/mesh-provisioner-client/" + broker.ProvisionerSecret}; !reflect.DeepEqual(vault.asked, want) {
|
||||
t.Fatalf("the vault was asked %v", vault.asked)
|
||||
}
|
||||
if want, _ := sealingKeyPEM(key); vault.keys[0] != want {
|
||||
t.Fatalf("the vault was asked to seal to %q, not the machine's key", vault.keys[0])
|
||||
}
|
||||
first, _ := json.Marshal(f[catalogue.VaultSealedField])
|
||||
|
||||
// The vault answers afresh with the same generation: the kept box is carried again, so the declaration is stable.
|
||||
again, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField])
|
||||
if string(again) != string(first) {
|
||||
t.Fatalf("a fresh box of the same generation changed the declaration: %s, then %s", first, again)
|
||||
}
|
||||
// A rotation in the vault (a newer generation) is carried.
|
||||
vault.generation = 2
|
||||
rotated, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField])
|
||||
if string(rotated) == string(first) {
|
||||
t.Fatal("a newer generation was not carried")
|
||||
}
|
||||
if _, found := vaultSilence(t, "laptop"); found {
|
||||
t.Fatal("a send the vault answered raised its silence")
|
||||
}
|
||||
}
|
||||
|
||||
func TestASilentVaultHoldsTheKeptBoxAndSaysSo(t *testing.T) {
|
||||
open, vault := aCoreMesh(t, link.VaultBoxContract)
|
||||
first, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField])
|
||||
vault.silent = errors.New("no responders")
|
||||
held := clientsBroker(sentTo(t, open, "laptop"))
|
||||
got, _ := json.Marshal(held[catalogue.VaultSealedField])
|
||||
if held == nil || held["sealed"] != nil || string(got) != string(first) {
|
||||
t.Fatalf("a silent vault's credential is sent as %v: the box kept, never the controller's seal", held)
|
||||
}
|
||||
c, found := vaultSilence(t, "laptop")
|
||||
if !found || !strings.Contains(c.Summary, "vault") || !strings.Contains(c.Summary, "laptop") {
|
||||
t.Fatalf("no condition names the vault and the machine: %+v", c)
|
||||
}
|
||||
// The vault answers again: the condition clears.
|
||||
vault.silent = nil
|
||||
sentTo(t, open, "laptop")
|
||||
if _, found := vaultSilence(t, "laptop"); found {
|
||||
t.Fatal("the condition stayed after the vault answered")
|
||||
}
|
||||
}
|
||||
|
||||
func TestACoreModuleFirstPlacedWhileTheVaultIsSilentIsSealedByTheController(t *testing.T) {
|
||||
open, vault := aCoreMesh(t, link.VaultBoxContract)
|
||||
vault.silent = errors.New("no responders")
|
||||
plan, settings, err := planFor(t.Context(), open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationFor(t.Context(), open, "laptop", plan, settings)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := clientsBroker(declared.Resources)
|
||||
if f == nil || f["sealed"] == nil || f[catalogue.VaultSealedField] != nil {
|
||||
t.Fatalf("the core credential is composed as %v: the controller's own seal, until the vault holds it", f)
|
||||
}
|
||||
if why, left := declared.leftOutWhy["mesh-provisioner-client"]; left {
|
||||
t.Fatalf("the core module was left out: %v", why)
|
||||
}
|
||||
c, found := vaultSilence(t, "laptop")
|
||||
if !found || !strings.Contains(c.Summary, "sealed by the controller until the vault holds it") {
|
||||
t.Fatalf("the condition does not say why: %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnEngineBelowTheContractIsSentTheControllersSealAsBefore(t *testing.T) {
|
||||
open, vault := aCoreMesh(t, link.HandedContract)
|
||||
f := clientsBroker(sentTo(t, open, "laptop"))
|
||||
if f == nil || f["sealed"] == nil || f[catalogue.VaultSealedField] != nil {
|
||||
t.Fatalf("an older engine is sent %v", f)
|
||||
}
|
||||
if len(vault.asked) != 0 {
|
||||
t.Fatalf("the vault was asked %v for an older engine", vault.asked)
|
||||
}
|
||||
}
|
||||
@@ -7,13 +7,11 @@ import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The core's own credentials travel sealed by the vault to their machine (novox/hq ADR 0349 §16, issue 755; the lead's
|
||||
@@ -46,28 +44,6 @@ func withVault(t *testing.T, v *coreStandIn) {
|
||||
t.Cleanup(func() { coreVault = before })
|
||||
}
|
||||
|
||||
func aCoreMesh(t *testing.T, contract int) (*stores, *coreStandIn) {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
client := aClientModule()
|
||||
register(t, open, client)
|
||||
if _, err := assign(ctx, open, "laptop", client.Module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := issueProvisionerClient(ctx, open.inventory, client, "laptop", broker.Broker{Fingerprint: "ab"},
|
||||
"10.0.0.1:4222"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.RecordHealth(ctx, inventory.NodeHealth{Node: "laptop", Contract: contract,
|
||||
SaidAt: time.Now()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v := &coreStandIn{generation: 1}
|
||||
withVault(t, v)
|
||||
return open, v
|
||||
}
|
||||
|
||||
func clientsBroker(resources []map[string]any) map[string]any {
|
||||
for _, r := range resources {
|
||||
if r["id"] == "mesh-provisioner-client.needs-"+broker.ProvisionerSecret {
|
||||
@@ -86,85 +62,6 @@ func vaultSilence(t *testing.T, node string) (conditions.Condition, bool) {
|
||||
return c, found
|
||||
}
|
||||
|
||||
func TestTheSendCarriesTheVaultsBoxForACoreCredentialAndKeepsIt(t *testing.T) {
|
||||
open, vault := aCoreMesh(t, link.VaultBoxContract)
|
||||
key, err := open.inventory.SealingKeyOf(t.Context(), "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := clientsBroker(sentTo(t, open, "laptop"))
|
||||
if f == nil || f["sealed"] != nil || f[catalogue.VaultSealedField] == nil {
|
||||
t.Fatalf("the core credential is sent as %v: the vault's box, never the controller's seal", f)
|
||||
}
|
||||
if want := []string{"own/laptop/mesh-provisioner-client/" + broker.ProvisionerSecret}; !reflect.DeepEqual(vault.asked, want) {
|
||||
t.Fatalf("the vault was asked %v", vault.asked)
|
||||
}
|
||||
if want, _ := sealingKeyPEM(key); vault.keys[0] != want {
|
||||
t.Fatalf("the vault was asked to seal to %q, not the machine's key", vault.keys[0])
|
||||
}
|
||||
first, _ := json.Marshal(f[catalogue.VaultSealedField])
|
||||
|
||||
// The vault answers afresh with the same generation: the kept box is carried again, so the declaration is stable.
|
||||
again, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField])
|
||||
if string(again) != string(first) {
|
||||
t.Fatalf("a fresh box of the same generation changed the declaration: %s, then %s", first, again)
|
||||
}
|
||||
// A rotation in the vault (a newer generation) is carried.
|
||||
vault.generation = 2
|
||||
rotated, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField])
|
||||
if string(rotated) == string(first) {
|
||||
t.Fatal("a newer generation was not carried")
|
||||
}
|
||||
if _, found := vaultSilence(t, "laptop"); found {
|
||||
t.Fatal("a send the vault answered raised its silence")
|
||||
}
|
||||
}
|
||||
|
||||
func TestASilentVaultHoldsTheKeptBoxAndSaysSo(t *testing.T) {
|
||||
open, vault := aCoreMesh(t, link.VaultBoxContract)
|
||||
first, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField])
|
||||
vault.silent = errors.New("no responders")
|
||||
held := clientsBroker(sentTo(t, open, "laptop"))
|
||||
got, _ := json.Marshal(held[catalogue.VaultSealedField])
|
||||
if held == nil || held["sealed"] != nil || string(got) != string(first) {
|
||||
t.Fatalf("a silent vault's credential is sent as %v: the box kept, never the controller's seal", held)
|
||||
}
|
||||
c, found := vaultSilence(t, "laptop")
|
||||
if !found || !strings.Contains(c.Summary, "vault") || !strings.Contains(c.Summary, "laptop") {
|
||||
t.Fatalf("no condition names the vault and the machine: %+v", c)
|
||||
}
|
||||
// The vault answers again: the condition clears.
|
||||
vault.silent = nil
|
||||
sentTo(t, open, "laptop")
|
||||
if _, found := vaultSilence(t, "laptop"); found {
|
||||
t.Fatal("the condition stayed after the vault answered")
|
||||
}
|
||||
}
|
||||
|
||||
func TestACoreModuleFirstPlacedWhileTheVaultIsSilentIsSealedByTheController(t *testing.T) {
|
||||
open, vault := aCoreMesh(t, link.VaultBoxContract)
|
||||
vault.silent = errors.New("no responders")
|
||||
plan, settings, err := planFor(t.Context(), open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationFor(t.Context(), open, "laptop", plan, settings)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := clientsBroker(declared.Resources)
|
||||
if f == nil || f["sealed"] == nil || f[catalogue.VaultSealedField] != nil {
|
||||
t.Fatalf("the core credential is composed as %v: the controller's own seal, until the vault holds it", f)
|
||||
}
|
||||
if why, left := declared.leftOutWhy["mesh-provisioner-client"]; left {
|
||||
t.Fatalf("the core module was left out: %v", why)
|
||||
}
|
||||
c, found := vaultSilence(t, "laptop")
|
||||
if !found || !strings.Contains(c.Summary, "sealed by the controller until the vault holds it") {
|
||||
t.Fatalf("the condition does not say why: %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 908: a machine at the vault-box contract, the vault silent, no kept box: nats is composed with the
|
||||
// controller's own seal, never left out, and the vault-silent condition names it.
|
||||
func TestReplay908(t *testing.T) {
|
||||
@@ -191,16 +88,6 @@ func TestReplay908(t *testing.T) {
|
||||
}
|
||||
|
||||
// An engine below VaultBoxContract is sent the controller's own seal, as before, and the vault is not asked.
|
||||
func TestAnEngineBelowTheContractIsSentTheControllersSealAsBefore(t *testing.T) {
|
||||
open, vault := aCoreMesh(t, link.HandedContract)
|
||||
f := clientsBroker(sentTo(t, open, "laptop"))
|
||||
if f == nil || f["sealed"] == nil || f[catalogue.VaultSealedField] != nil {
|
||||
t.Fatalf("an older engine is sent %v", f)
|
||||
}
|
||||
if len(vault.asked) != 0 {
|
||||
t.Fatalf("the vault was asked %v for an older engine", vault.asked)
|
||||
}
|
||||
}
|
||||
|
||||
// Every question but a send reads the kept box and asks nothing; with none kept, a stand-in the next send asks for.
|
||||
func TestOnlyASendAsksTheVault(t *testing.T) {
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
func TestAMovedPathIsNoShrinkAndAShrinkThereStillIs(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
ctx := t.Context()
|
||||
shelf := shelfFor(t, houseManifest)
|
||||
declared := []inventory.DeclaredData{{Module: "house", Item: "config", Class: "irreplaceable", Owned: true}}
|
||||
start := time.Now().Add(-6 * time.Hour)
|
||||
measure := func(at time.Time, path string, size int64) []conditions.Observation {
|
||||
t.Helper()
|
||||
if _, err := inv.RecordData(ctx, "home", declared, map[string]map[string]inventory.Measurement{"house": {
|
||||
"config": {Path: path, Size: bytesOf(size), MeasuredAt: when(at), LastWrite: when(at),
|
||||
LastBackup: when(at)}}}, "", at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
records, err := inv.Data(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
peaks, err := inv.DataPeaks(ctx, at.Add(-shrinkWindow))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return dataFindings(records, peaks, shelf, nil, nil, at)
|
||||
}
|
||||
measure(start, "/home/operator/.claude", 94_700_000)
|
||||
if got := findingsByKind(measure(start.Add(10*time.Minute), "/home/agent/.claude", 490)); got[kindDataShrank].Kind != "" {
|
||||
t.Fatalf("a moved path raised a shrink: %+v", got[kindDataShrank])
|
||||
}
|
||||
measure(start.Add(2*time.Hour), "/home/agent/.claude", 300<<20)
|
||||
got := findingsByKind(measure(start.Add(4*time.Hour), "/home/agent/.claude", 1<<20))[kindDataShrank]
|
||||
if got.Severity != conditions.Urgent || !strings.Contains(got.Summary, "shrank") {
|
||||
t.Fatalf("a genuine shrink at the new path was not raised: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "keeper", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}})
|
||||
house, err := catalogue.ParseManifest([]byte(houseManifest))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, house)
|
||||
if _, err := assign(ctx, open, "laptop", "house"); err == nil {
|
||||
t.Fatal("irreplaceable data was assigned to a machine with nothing to back it up")
|
||||
}
|
||||
for _, node := range []string{"laptop", "anchor"} {
|
||||
if _, err := assign(ctx, open, node, "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "house"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Sent, and applied: a holder is asked only once it has been (novox/hq issue 275).
|
||||
for _, node := range []string{"laptop", "anchor"} {
|
||||
pushedAndApplied(t, open, node)
|
||||
}
|
||||
|
||||
conn := onATestBus(t)
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
laptop, anchor := &fakeHolder{}, &fakeHolder{}
|
||||
laptop.serve(t, conn, "laptop")
|
||||
anchor.serve(t, conn, "anchor")
|
||||
now := time.Now()
|
||||
heard := &watchdogs{last: &signalFacts{now: now, machines: []machineFacts{
|
||||
{name: "laptop", lastHeard: now}, {name: "anchor", lastHeard: now}}}}
|
||||
d := &doctor{open: open, js: js, watchdogs: heard}
|
||||
var d13 probe
|
||||
for _, p := range probeRegistry {
|
||||
if p.ID == probeDataID {
|
||||
d13 = p
|
||||
}
|
||||
}
|
||||
run := func() []conditions.Observation {
|
||||
t.Helper()
|
||||
probing := context.WithValue(ctx, probeAsksKey{}, d13)
|
||||
obs, err := probeData(probing, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return obs
|
||||
}
|
||||
|
||||
laptop.set(measuredHouse("/var/lib/house/config", 2<<30, now))
|
||||
if obs := run(); len(obs) != 0 {
|
||||
t.Fatalf("a measured, backed-up item raised %+v", obs)
|
||||
}
|
||||
r, err := inv.DataOf(ctx, "laptop", "house", "config")
|
||||
if err != nil || r.Path != "/var/lib/house/config" || r.Size == nil || *r.Size != 2<<30 || r.LastBackup == nil {
|
||||
t.Fatalf("what the holder measured was not kept: %+v, %v", r, err)
|
||||
}
|
||||
|
||||
said, err := unassign(ctx, open, "laptop", "house")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(said, "house's config (/var/lib/house/config, 2.0 GB) stays where it is") {
|
||||
t.Fatalf("the unassignment does not say the data stays:\n%s", said)
|
||||
}
|
||||
laptop.set()
|
||||
run()
|
||||
r, err = inv.DataOf(ctx, "laptop", "house", "config")
|
||||
if err != nil || !r.Retired() || r.Path != "/var/lib/house/config" {
|
||||
t.Fatalf("unassigned, the irreplaceable item is not kept retired: %+v, %v", r, err)
|
||||
}
|
||||
records, _ := inv.Data(ctx)
|
||||
if rows := retiredData(records, time.Now()); len(rows) != 1 || rows[0].Path != "/var/lib/house/config" {
|
||||
t.Fatalf("cleanup list: %+v", rows)
|
||||
}
|
||||
|
||||
// Assigned on the anchor, onto an empty directory.
|
||||
if _, err := assign(ctx, open, "anchor", "house"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
anchor.set(measuredHouse("/var/lib/house/config", 300<<10, time.Now()))
|
||||
obs := findingsByKind(run())
|
||||
o, ok := obs[kindEmptyReplacement]
|
||||
if !ok || o.Severity != conditions.Urgent || o.Machine != "anchor" || o.Also[0] != "laptop" {
|
||||
t.Fatalf("an empty replacement of a module's data was not urgent: %+v", obs)
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -10,14 +9,11 @@ import (
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
func bytesOf(n int64) *int64 { return &n }
|
||||
@@ -173,39 +169,6 @@ func TestAShrinkOfMoreThanHalfIsUrgent(t *testing.T) {
|
||||
// operator's own home (94.7 MB) to the agent account's fresh one (490 B), and `data-shrank` was raised
|
||||
// for data that was never lost. A moved item is read against its new path only, so nothing is raised —
|
||||
// and a genuine shrink at the new path, a week of history later, still is.
|
||||
func TestAMovedPathIsNoShrinkAndAShrinkThereStillIs(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
ctx := t.Context()
|
||||
shelf := shelfFor(t, houseManifest)
|
||||
declared := []inventory.DeclaredData{{Module: "house", Item: "config", Class: "irreplaceable", Owned: true}}
|
||||
start := time.Now().Add(-6 * time.Hour)
|
||||
measure := func(at time.Time, path string, size int64) []conditions.Observation {
|
||||
t.Helper()
|
||||
if _, err := inv.RecordData(ctx, "home", declared, map[string]map[string]inventory.Measurement{"house": {
|
||||
"config": {Path: path, Size: bytesOf(size), MeasuredAt: when(at), LastWrite: when(at),
|
||||
LastBackup: when(at)}}}, "", at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
records, err := inv.Data(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
peaks, err := inv.DataPeaks(ctx, at.Add(-shrinkWindow))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return dataFindings(records, peaks, shelf, nil, nil, at)
|
||||
}
|
||||
measure(start, "/home/operator/.claude", 94_700_000)
|
||||
if got := findingsByKind(measure(start.Add(10*time.Minute), "/home/agent/.claude", 490)); got[kindDataShrank].Kind != "" {
|
||||
t.Fatalf("a moved path raised a shrink: %+v", got[kindDataShrank])
|
||||
}
|
||||
measure(start.Add(2*time.Hour), "/home/agent/.claude", 300<<20)
|
||||
got := findingsByKind(measure(start.Add(4*time.Hour), "/home/agent/.claude", 1<<20))[kindDataShrank]
|
||||
if got.Severity != conditions.Urgent || !strings.Contains(got.Summary, "shrank") {
|
||||
t.Fatalf("a genuine shrink at the new path was not raised: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Data said to be written all the time and not written; data with no backup or an old one — urgent when
|
||||
// irreplaceable, a warning when valuable; and a new item given its bound before it is said.
|
||||
@@ -313,103 +276,6 @@ func measuredHouse(path string, size int64, at time.Time) map[string]any {
|
||||
// directory is an empty replacement — through the real stores and a real bus. The unassignment says the
|
||||
// data stays; the self-check's next run retires it (kept, listed by cleanup), and when the module comes
|
||||
// up on another machine with an empty directory while the full one waits retired, that is urgent.
|
||||
func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "keeper", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}})
|
||||
house, err := catalogue.ParseManifest([]byte(houseManifest))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, house)
|
||||
if _, err := assign(ctx, open, "laptop", "house"); err == nil {
|
||||
t.Fatal("irreplaceable data was assigned to a machine with nothing to back it up")
|
||||
}
|
||||
for _, node := range []string{"laptop", "anchor"} {
|
||||
if _, err := assign(ctx, open, node, "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "house"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Sent, and applied: a holder is asked only once it has been (novox/hq issue 275).
|
||||
for _, node := range []string{"laptop", "anchor"} {
|
||||
pushedAndApplied(t, open, node)
|
||||
}
|
||||
|
||||
conn := onATestBus(t)
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
laptop, anchor := &fakeHolder{}, &fakeHolder{}
|
||||
laptop.serve(t, conn, "laptop")
|
||||
anchor.serve(t, conn, "anchor")
|
||||
now := time.Now()
|
||||
heard := &watchdogs{last: &signalFacts{now: now, machines: []machineFacts{
|
||||
{name: "laptop", lastHeard: now}, {name: "anchor", lastHeard: now}}}}
|
||||
d := &doctor{open: open, js: js, watchdogs: heard}
|
||||
var d13 probe
|
||||
for _, p := range probeRegistry {
|
||||
if p.ID == probeDataID {
|
||||
d13 = p
|
||||
}
|
||||
}
|
||||
run := func() []conditions.Observation {
|
||||
t.Helper()
|
||||
probing := context.WithValue(ctx, probeAsksKey{}, d13)
|
||||
obs, err := probeData(probing, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return obs
|
||||
}
|
||||
|
||||
laptop.set(measuredHouse("/var/lib/house/config", 2<<30, now))
|
||||
if obs := run(); len(obs) != 0 {
|
||||
t.Fatalf("a measured, backed-up item raised %+v", obs)
|
||||
}
|
||||
r, err := inv.DataOf(ctx, "laptop", "house", "config")
|
||||
if err != nil || r.Path != "/var/lib/house/config" || r.Size == nil || *r.Size != 2<<30 || r.LastBackup == nil {
|
||||
t.Fatalf("what the holder measured was not kept: %+v, %v", r, err)
|
||||
}
|
||||
|
||||
said, err := unassign(ctx, open, "laptop", "house")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(said, "house's config (/var/lib/house/config, 2.0 GB) stays where it is") {
|
||||
t.Fatalf("the unassignment does not say the data stays:\n%s", said)
|
||||
}
|
||||
laptop.set()
|
||||
run()
|
||||
r, err = inv.DataOf(ctx, "laptop", "house", "config")
|
||||
if err != nil || !r.Retired() || r.Path != "/var/lib/house/config" {
|
||||
t.Fatalf("unassigned, the irreplaceable item is not kept retired: %+v, %v", r, err)
|
||||
}
|
||||
records, _ := inv.Data(ctx)
|
||||
if rows := retiredData(records, time.Now()); len(rows) != 1 || rows[0].Path != "/var/lib/house/config" {
|
||||
t.Fatalf("cleanup list: %+v", rows)
|
||||
}
|
||||
|
||||
// Assigned on the anchor, onto an empty directory.
|
||||
if _, err := assign(ctx, open, "anchor", "house"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
anchor.set(measuredHouse("/var/lib/house/config", 300<<10, time.Now()))
|
||||
obs := findingsByKind(run())
|
||||
o, ok := obs[kindEmptyReplacement]
|
||||
if !ok || o.Severity != conditions.Urgent || o.Machine != "anchor" || o.Also[0] != "laptop" {
|
||||
t.Fatalf("an empty replacement of a module's data was not urgent: %+v", obs)
|
||||
}
|
||||
}
|
||||
|
||||
// Data on redundant storage: the array it is on is watched, one condition per array as loud as the most
|
||||
// precious item on it; an item said to be on redundancy and found on plain storage is said; an item
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
func servingDeadLetters(t *testing.T) *broker.JetStream {
|
||||
t.Helper()
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := servingBus.Load()
|
||||
servingBus.Store(js)
|
||||
t.Cleanup(func() { servingBus.Store(before) })
|
||||
return js
|
||||
}
|
||||
|
||||
func TestDeadLettersListsDropsAndRecordsWhy(t *testing.T) {
|
||||
js := servingDeadLetters(t)
|
||||
if _, err := js.Context().Publish("mesh.mod.gitea.event.pull.merged", []byte(`{"n":1}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, err := link.KeepDeadLetter(js.Context(), []byte(`{"stream":"EVENTS","consumer":"media_sonarr","stream_seq":1,"deliveries":5}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
answer, err := askDeadLetters(t, map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
listed := answer.(map[string]any)
|
||||
if listed["held"] != 1 || len(listed["dead_letters"].([]link.DeadLetter)) != 1 {
|
||||
t.Fatalf("listed %v", listed)
|
||||
}
|
||||
|
||||
// Refused before anything is done: an act without why, an argument the shape passes over, both acts.
|
||||
for _, args := range []map[string]any{
|
||||
{"drop": "1"},
|
||||
{"drop": "1", "why": "x", "limit": "3"},
|
||||
{"drop": "1", "deliver": "1", "why": "x"},
|
||||
{"why": "x"},
|
||||
{"id": "nought"},
|
||||
} {
|
||||
if _, err := askDeadLetters(t, args); err == nil {
|
||||
t.Errorf("%v was done", args)
|
||||
}
|
||||
}
|
||||
if _, total, _ := link.DeadLetters(js.Context(), "", 0); total != 1 {
|
||||
t.Fatalf("a refused call changed what is kept: %d left", total)
|
||||
}
|
||||
|
||||
done, err := askDeadLetters(t, map[string]any{"drop": "1", "why": "the media server took the download in by hand"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if said := done.(map[string]any); said["recorded"] == nil || !strings.Contains(said["done"].(string), "dropped") {
|
||||
t.Fatalf("answered %v", said)
|
||||
}
|
||||
acts, err := link.HandActs(context.Background(), js.Conn(), time.Now().Add(-time.Minute))
|
||||
if err != nil || len(acts) != 1 || acts[0].Verb != "dead-letters drop" || acts[0].Cause != causeDeadLetter ||
|
||||
!strings.Contains(acts[0].Condition, "media_sonarr") {
|
||||
t.Fatalf("recorded %+v (%v)", acts, err)
|
||||
}
|
||||
if !personsDecision(acts[0]) {
|
||||
t.Error("dropping a dead letter is counted as a repair, so S15 would want a healer for it")
|
||||
}
|
||||
if _, err := askDeadLetters(t, map[string]any{"id": "1"}); err == nil {
|
||||
t.Errorf("dead letter %d is still answered after it was dropped", d.ID)
|
||||
}
|
||||
}
|
||||
@@ -6,31 +6,10 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// The serving controller's bus, with the mesh's streams, for the verb to read and act on.
|
||||
func servingDeadLetters(t *testing.T) *broker.JetStream {
|
||||
t.Helper()
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := servingBus.Load()
|
||||
servingBus.Store(js)
|
||||
t.Cleanup(func() { servingBus.Store(before) })
|
||||
return js
|
||||
}
|
||||
|
||||
func askDeadLetters(t *testing.T, args map[string]any) (any, error) {
|
||||
t.Helper()
|
||||
@@ -41,61 +20,6 @@ func askDeadLetters(t *testing.T, args map[string]any) (any, error) {
|
||||
return deadLettersAnswer(context.Background(), a)
|
||||
}
|
||||
|
||||
func TestDeadLettersListsDropsAndRecordsWhy(t *testing.T) {
|
||||
js := servingDeadLetters(t)
|
||||
if _, err := js.Context().Publish("mesh.mod.gitea.event.pull.merged", []byte(`{"n":1}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, err := link.KeepDeadLetter(js.Context(), []byte(`{"stream":"EVENTS","consumer":"media_sonarr","stream_seq":1,"deliveries":5}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
answer, err := askDeadLetters(t, map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
listed := answer.(map[string]any)
|
||||
if listed["held"] != 1 || len(listed["dead_letters"].([]link.DeadLetter)) != 1 {
|
||||
t.Fatalf("listed %v", listed)
|
||||
}
|
||||
|
||||
// Refused before anything is done: an act without why, an argument the shape passes over, both acts.
|
||||
for _, args := range []map[string]any{
|
||||
{"drop": "1"},
|
||||
{"drop": "1", "why": "x", "limit": "3"},
|
||||
{"drop": "1", "deliver": "1", "why": "x"},
|
||||
{"why": "x"},
|
||||
{"id": "nought"},
|
||||
} {
|
||||
if _, err := askDeadLetters(t, args); err == nil {
|
||||
t.Errorf("%v was done", args)
|
||||
}
|
||||
}
|
||||
if _, total, _ := link.DeadLetters(js.Context(), "", 0); total != 1 {
|
||||
t.Fatalf("a refused call changed what is kept: %d left", total)
|
||||
}
|
||||
|
||||
done, err := askDeadLetters(t, map[string]any{"drop": "1", "why": "the media server took the download in by hand"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if said := done.(map[string]any); said["recorded"] == nil || !strings.Contains(said["done"].(string), "dropped") {
|
||||
t.Fatalf("answered %v", said)
|
||||
}
|
||||
acts, err := link.HandActs(context.Background(), js.Conn(), time.Now().Add(-time.Minute))
|
||||
if err != nil || len(acts) != 1 || acts[0].Verb != "dead-letters drop" || acts[0].Cause != causeDeadLetter ||
|
||||
!strings.Contains(acts[0].Condition, "media_sonarr") {
|
||||
t.Fatalf("recorded %+v (%v)", acts, err)
|
||||
}
|
||||
if !personsDecision(acts[0]) {
|
||||
t.Error("dropping a dead letter is counted as a repair, so S15 would want a healer for it")
|
||||
}
|
||||
if _, err := askDeadLetters(t, map[string]any{"id": "1"}); err == nil {
|
||||
t.Errorf("dead letter %d is still answered after it was dropped", d.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// Open while DEAD_LETTERS holds a message for the consumer, in words the operator reads in one pass:
|
||||
// what is held, and where to act.
|
||||
func TestAConsumersDeadLettersAreSaidUntilActedOn(t *testing.T) {
|
||||
|
||||
+131
-183
@@ -136,8 +136,9 @@ func letGo(ctx context.Context, inv *inventory.Inventory, id, by, why string) (i
|
||||
return p, fmt.Errorf("%s was let go by %s at %s already", p.ID, p.Delivery.By,
|
||||
p.Delivery.Go.Local().Format("15:04:05"))
|
||||
}
|
||||
// **One walk at a time** (novox/hq ADR 0276): the word is taken, and the walk starts once no other walk is
|
||||
// started (advanceOnce), so the delivery's owner says it once and is not refused.
|
||||
// **One walk at a time beside a related one** (novox/hq ADR 0276, ADR 0407 decision 4, issue 973): the word is
|
||||
// taken, and the walk starts once no started walk it is related to is open (advanceOnce), so the delivery's
|
||||
// owner says it once and is not refused.
|
||||
now := time.Now().UTC()
|
||||
p.Delivery.Go, p.Delivery.By, p.Delivery.Why = &now, by, why
|
||||
p.Note = "let go by " + by + "; its first tier is asked next"
|
||||
@@ -268,7 +269,7 @@ func stopWalkWithout(ctx context.Context, inv *inventory.Inventory, id, by, why
|
||||
append(moves, foldMoves...)); err != nil {
|
||||
return p, said, fmt.Errorf("%s is not stopped, and none of its merges moved: %w", p.ID, err)
|
||||
}
|
||||
// What the batch holds and names now: written again by the cutter's next look should this fail.
|
||||
// What the batch holds and names now: written again by the next look should this fail.
|
||||
if err := keepBatch(ctx, inv, &target, now, ""); err != nil {
|
||||
fmt.Printf(" %s holds its merges; its record is written again at the next look: %v\n", target.ID, err)
|
||||
}
|
||||
@@ -578,11 +579,12 @@ type orderMember struct {
|
||||
ModuleDirs []string `json:"module_dirs,omitempty"`
|
||||
ModuleDirsSaid bool `json:"module_dirs_said,omitempty"`
|
||||
CloneURL string `json:"clone_url,omitempty"`
|
||||
// After are the repositories its pull request says it goes after (`after: <repository>`).
|
||||
After []string `json:"after,omitempty"`
|
||||
// BaseSHA is the base branch's head the forge last announced the head with (novox/hq ADR 0323): the trunk commit
|
||||
// a verdict no build judges names when the head is checked again through mesh-delivery.
|
||||
BaseSHA string `json:"base_sha,omitempty"`
|
||||
// Ready says mesh-delivery wants the member's check queued before every other check (novox/hq ADR 0406 §3, issue
|
||||
// 974). There is no ready mark any more (issue 1022): mesh-delivery says it when it asks for the Approve.
|
||||
Ready bool `json:"ready,omitempty"`
|
||||
}
|
||||
|
||||
// pull is the member as the forge announced it.
|
||||
@@ -594,7 +596,7 @@ func (m orderMember) pull() link.PullUpdated {
|
||||
}
|
||||
return link.PullUpdated{Owner: owner, Repo: repo, Number: m.Number, Base: base, Commit: m.Head, BaseSHA: m.BaseSHA,
|
||||
CloneURL: m.CloneURL, Paths: m.Paths, PathsTruncated: m.PathsTruncated, Removed: m.Removed,
|
||||
ModuleDirs: m.ModuleDirs, ModuleDirsSaid: m.ModuleDirsSaid}
|
||||
ModuleDirs: m.ModuleDirs, ModuleDirsSaid: m.ModuleDirsSaid, Ready: m.Ready}
|
||||
}
|
||||
|
||||
// orderPair is one "before" among a group's members, why, and the rules it won over.
|
||||
@@ -607,14 +609,6 @@ type orderPair struct {
|
||||
Over []string `json:"over,omitempty"`
|
||||
}
|
||||
|
||||
// orderContradiction is two members that rules of one rank order both ways: no precedence resolves it, so
|
||||
// the group is refused, the rules named, and Said says how a person declares the order.
|
||||
type orderContradiction struct {
|
||||
Members []string `json:"members"`
|
||||
Rules []orderPair `json:"rules"`
|
||||
Said string `json:"said"`
|
||||
}
|
||||
|
||||
// orderReach is what a member moves, as the planner says.
|
||||
type orderReach struct {
|
||||
Moved []string `json:"moved,omitempty"`
|
||||
@@ -623,114 +617,76 @@ type orderReach struct {
|
||||
Manifests []string `json:"manifests,omitempty"`
|
||||
}
|
||||
|
||||
// groupOrder is a group's order: the members in it, every pair and why, and the cycle when there is one —
|
||||
// with each contradiction no precedence resolves.
|
||||
// groupOrder is a group's order: the members in it, every pair and why, and the cycle when there is one. The
|
||||
// order is read from the module graph and the manifests' change alone (novox/hq ADR 0419): a cycle is the only
|
||||
// refusal, and CycleSaid names every module and edge of it. Edges are the module edges the pairs come from.
|
||||
type groupOrder struct {
|
||||
Order []string `json:"order"`
|
||||
Pairs []orderPair `json:"pairs,omitempty"`
|
||||
Cycle []string `json:"cycle,omitempty"`
|
||||
Contradictions []orderContradiction `json:"contradictions,omitempty"`
|
||||
Reach map[string]orderReach `json:"reach,omitempty"`
|
||||
Order []string `json:"order"`
|
||||
Pairs []orderPair `json:"pairs,omitempty"`
|
||||
Cycle []string `json:"cycle,omitempty"`
|
||||
CycleSaid string `json:"cycle_said,omitempty"`
|
||||
Edges []inventory.Edge `json:"edges,omitempty"`
|
||||
Reach map[string]orderReach `json:"reach,omitempty"`
|
||||
}
|
||||
|
||||
// The reasons a pair is ordered, in the words the delivery plan shows.
|
||||
const (
|
||||
orderDeclared = "declared"
|
||||
orderBuiltBy = "built by"
|
||||
orderVersionSkew = "version skew"
|
||||
orderEngineFirst = "engine before controller"
|
||||
)
|
||||
// orderBuiltBy is the reason a member goes before another that moves a module built by, standing on, packaging or
|
||||
// declaring one of its modules: the module graph's own edge.
|
||||
const orderBuiltBy = "built by"
|
||||
|
||||
// orderRank is a rule's precedence when two rules order one pair both ways (novox/hq ADR 0249), the lower
|
||||
// winning: what a person declared; then what the graph and the change say (built by, version skew); then
|
||||
// the rollout default, which infers a direction from the modules' names alone.
|
||||
func orderRank(why string) int {
|
||||
switch why {
|
||||
case orderDeclared:
|
||||
return 0
|
||||
case orderBuiltBy, orderVersionSkew:
|
||||
return 1
|
||||
case orderEngineFirst:
|
||||
return 2
|
||||
}
|
||||
return 3
|
||||
}
|
||||
|
||||
// orderOf is a group's order (novox/hq ADR 0239 decision 4, ADR 0249), pure. A member goes before another
|
||||
// when:
|
||||
//
|
||||
// - its pull request is named in the other's `after:` lines (declared);
|
||||
// - it moves a module the other's moved modules are built by, stand on, package or declare (built by);
|
||||
// - it moves the controller and the other changes any module's manifest (version skew: the newer
|
||||
// controller parses what the newer manifest says) — the node-engine's excepted, which goes first;
|
||||
// - it moves the node-engine and the other moves the controller (the witness reads nothing the controller
|
||||
// does not yet grant, and a controller sends nothing an older engine would refuse — ADR 0236's rollout
|
||||
// order).
|
||||
//
|
||||
// When rules order one pair both ways, the one of higher precedence wins (orderRank) and the pair says
|
||||
// what it won over. Rules of one rank both ways are a contradiction: both are kept as pairs, the members
|
||||
// are the cycle, and the contradiction says how to declare the order. Otherwise, by repository then id, so
|
||||
// every reading gives one order. A cycle through three members or more is named by its members, and none
|
||||
// of them is ordered.
|
||||
func orderOf(members []orderMember, reach map[string]orderReach, edges []inventory.Edge) groupOrder {
|
||||
// orderOf is a group's order (novox/hq ADR 0419), pure: a member goes before another when a module it moves goes
|
||||
// before a module the other moves by an edge of the module graph (built by), or by a rule of delivery_edges.go
|
||||
// (verb before caller, version skew, engine before controller), which changes names the manifests' added verbs.
|
||||
// No `after:` line is read. Ties break by repository then id, so every reading gives one order. A cycle through
|
||||
// two members or more is named by its members, and none of them is ordered.
|
||||
func orderOf(members []orderMember, reach map[string]orderReach, edges []inventory.Edge, changes map[string]moduleChange) groupOrder {
|
||||
out := groupOrder{Reach: reach}
|
||||
byID := map[string]orderMember{}
|
||||
movedBy := map[string][]string{}
|
||||
var all []string
|
||||
for _, m := range members {
|
||||
byID[m.ID] = m
|
||||
for _, n := range reach[m.ID].Moved {
|
||||
movedBy[n] = append(movedBy[n], m.ID)
|
||||
all = appendNew(all, n)
|
||||
}
|
||||
}
|
||||
var claims []orderPair
|
||||
add := func(before, after, why string) {
|
||||
if before == after {
|
||||
return
|
||||
}
|
||||
for _, p := range claims {
|
||||
for _, p := range out.Pairs {
|
||||
if p.Before == before && p.After == after && p.Why == why {
|
||||
return
|
||||
}
|
||||
}
|
||||
claims = append(claims, orderPair{Before: before, After: after, Why: why})
|
||||
out.Pairs = append(out.Pairs, orderPair{Before: before, After: after, Why: why})
|
||||
}
|
||||
named := func(said, repository string) bool {
|
||||
said = strings.TrimSuffix(strings.TrimSpace(said), ".git")
|
||||
if strings.EqualFold(said, repository) {
|
||||
return true
|
||||
for _, e := range edges {
|
||||
if !orders(e) {
|
||||
continue
|
||||
}
|
||||
_, repo, _ := strings.Cut(repository, "/")
|
||||
return strings.EqualFold(said, repo)
|
||||
for _, before := range movedBy[e.To] {
|
||||
for _, after := range movedBy[e.From] {
|
||||
add(before, after, orderBuiltBy)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, a := range members {
|
||||
for _, b := range members {
|
||||
if a.ID == b.ID {
|
||||
continue
|
||||
}
|
||||
ra, rb := reach[a.ID], reach[b.ID]
|
||||
for _, said := range b.After {
|
||||
if named(said, a.Repository) {
|
||||
add(a.ID, b.ID, orderDeclared)
|
||||
}
|
||||
}
|
||||
for _, e := range edges {
|
||||
if slices.Contains(ra.Moved, e.To) && slices.Contains(rb.Moved, e.From) && e.From != e.To {
|
||||
add(a.ID, b.ID, orderBuiltBy)
|
||||
break
|
||||
}
|
||||
}
|
||||
if slices.Contains(ra.Moved, "mesh-controller") && len(rb.Manifests) > 0 &&
|
||||
!slices.Contains(rb.Moved, "mesh-controller") && !slices.Contains(rb.Moved, "mesh-host") {
|
||||
add(a.ID, b.ID, orderVersionSkew)
|
||||
}
|
||||
if slices.Contains(ra.Moved, "mesh-host") && slices.Contains(rb.Moved, "mesh-controller") &&
|
||||
!slices.Contains(ra.Moved, "mesh-controller") {
|
||||
add(a.ID, b.ID, orderEngineFirst)
|
||||
order := orderEdges(all, changes, edges)
|
||||
out.Edges = order.Edges
|
||||
out.CycleSaid = order.CycleSaid()
|
||||
for _, e := range order.Edges {
|
||||
for _, before := range movedBy[e.To] {
|
||||
for _, after := range movedBy[e.From] {
|
||||
add(before, after, e.Why)
|
||||
}
|
||||
}
|
||||
}
|
||||
out.Pairs, out.Contradictions = resolveClaims(claims, byID)
|
||||
sort.Slice(out.Pairs, func(i, j int) bool {
|
||||
if out.Pairs[i].Before != out.Pairs[j].Before {
|
||||
return out.Pairs[i].Before < out.Pairs[j].Before
|
||||
}
|
||||
return out.Pairs[i].After < out.Pairs[j].After
|
||||
if out.Pairs[i].After != out.Pairs[j].After {
|
||||
return out.Pairs[i].After < out.Pairs[j].After
|
||||
}
|
||||
return out.Pairs[i].Why < out.Pairs[j].Why
|
||||
})
|
||||
// Kahn's walk, the next always the first by repository and id among those with nothing before them.
|
||||
waiting := map[string]int{}
|
||||
@@ -775,89 +731,6 @@ func orderOf(members []orderMember, reach map[string]orderReach, edges []invento
|
||||
return out
|
||||
}
|
||||
|
||||
// orderRules is the rules in the order a pair names them when more than one says the same.
|
||||
var orderRules = []string{orderDeclared, orderBuiltBy, orderVersionSkew, orderEngineFirst}
|
||||
|
||||
// resolveClaims is the pairs the rules' claims come to (novox/hq ADR 0249): for each two members, the claims
|
||||
// of the highest precedence decide, and the claims they overrule are said on the pair. Claims of one rank
|
||||
// both ways are a contradiction: both pairs are kept, so the walk finds the cycle, and the contradiction
|
||||
// says how a person declares the order.
|
||||
func resolveClaims(claims []orderPair, byID map[string]orderMember) ([]orderPair, []orderContradiction) {
|
||||
byTwo := map[[2]string][]orderPair{}
|
||||
var keys [][2]string
|
||||
for _, c := range claims {
|
||||
k := [2]string{min(c.Before, c.After), max(c.Before, c.After)}
|
||||
if _, ok := byTwo[k]; !ok {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
byTwo[k] = append(byTwo[k], c)
|
||||
}
|
||||
slices.SortFunc(keys, func(a, b [2]string) int {
|
||||
if c := strings.Compare(a[0], b[0]); c != 0 {
|
||||
return c
|
||||
}
|
||||
return strings.Compare(a[1], b[1])
|
||||
})
|
||||
ruleIndex := func(why string) int { return slices.Index(orderRules, why) }
|
||||
var pairs []orderPair
|
||||
var contradictions []orderContradiction
|
||||
for _, k := range keys {
|
||||
cs := byTwo[k]
|
||||
slices.SortStableFunc(cs, func(a, b orderPair) int { return ruleIndex(a.Why) - ruleIndex(b.Why) })
|
||||
top := orderRank(cs[0].Why)
|
||||
var won []orderPair
|
||||
ways := map[string]bool{}
|
||||
for _, c := range cs {
|
||||
if orderRank(c.Why) == top {
|
||||
won = append(won, c)
|
||||
ways[c.Before] = true
|
||||
}
|
||||
}
|
||||
if len(ways) == 1 {
|
||||
p := orderPair{Before: won[0].Before, After: won[0].After, Why: won[0].Why}
|
||||
for _, c := range cs {
|
||||
if c.Before != p.Before && !slices.Contains(p.Over, c.Why) {
|
||||
p.Over = append(p.Over, c.Why)
|
||||
}
|
||||
}
|
||||
pairs = append(pairs, p)
|
||||
continue
|
||||
}
|
||||
// One rank both ways: no precedence resolves it.
|
||||
var first, second *orderPair
|
||||
var said []string
|
||||
for i := range won {
|
||||
c := won[i]
|
||||
if c.Before == k[0] && first == nil {
|
||||
first = &won[i]
|
||||
}
|
||||
if c.Before == k[1] && second == nil {
|
||||
second = &won[i]
|
||||
}
|
||||
said = append(said, fmt.Sprintf("%s puts %s first", c.Why, c.Before))
|
||||
}
|
||||
pairs = append(pairs, orderPair{Before: first.Before, After: first.After, Why: first.Why},
|
||||
orderPair{Before: second.Before, After: second.After, Why: second.Why})
|
||||
a, b := byID[k[0]], byID[k[1]]
|
||||
how := fmt.Sprintf("neither outranks the other: declare the order with a line `after: %s` in %s's "+
|
||||
"description, or `after: %s` in %s's", a.Repository, pullName(b), b.Repository, pullName(a))
|
||||
if top == orderRank(orderDeclared) {
|
||||
how = "each pull request declares it goes after the other: remove one of the `after:` lines"
|
||||
}
|
||||
contradictions = append(contradictions, orderContradiction{Members: []string{k[0], k[1]}, Rules: won,
|
||||
Said: fmt.Sprintf("%s and %s: %s; %s", k[0], k[1], strings.Join(said, ", "), how)})
|
||||
}
|
||||
return pairs, contradictions
|
||||
}
|
||||
|
||||
// pullName is a member as a person finds it: its pull request, or its id when no number was said.
|
||||
func pullName(m orderMember) string {
|
||||
if m.Number > 0 {
|
||||
return fmt.Sprintf("%s#%d", m.Repository, m.Number)
|
||||
}
|
||||
return m.ID
|
||||
}
|
||||
|
||||
// reachOfMembers is each member's reach, as the planner says it.
|
||||
func reachOfMembers(members []orderMember, entries []inventory.Entry, read map[string][]inventory.ReadRepository,
|
||||
edges []inventory.Edge) map[string]orderReach {
|
||||
@@ -891,6 +764,8 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
removed := set.String("removed", "", "the files it deletes, comma-separated")
|
||||
membersJSON := set.String("members", "", "a group's members, as JSON")
|
||||
group := set.String("group", "", "a delivery group's id")
|
||||
ready := set.Bool("ready", false, "the head's pull request is marked ready: its check is queued before every other check "+
|
||||
"(novox/hq ADR 0406 §3)")
|
||||
by := set.String("by", catalogue.DeliverySeat, "who says it")
|
||||
why := set.String("why", "", "why")
|
||||
without := set.String("without", "", "merges a stop leaves out, <repository>@<commit>, comma-separated: the walk's "+
|
||||
@@ -906,6 +781,12 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
if err := json.Unmarshal([]byte(*membersJSON), &members); err != nil {
|
||||
return fmt.Errorf("the members are not readable: %w", err)
|
||||
}
|
||||
// Marked ready by the verb's word for every member it names, besides any member that says so itself.
|
||||
if *ready {
|
||||
for i := range members {
|
||||
members[i].Ready = true
|
||||
}
|
||||
}
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
@@ -947,7 +828,11 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return answer(orderOf(members, reachOfMembers(members, entries, read, edges), edges))
|
||||
changes, err := changesOf(ctx, changeSourcesOf(members, entries, reachOfMembers(members, entries, read, edges)))
|
||||
if err != nil {
|
||||
return fmt.Errorf("what the members say of their manifests cannot be read: %w", err)
|
||||
}
|
||||
return answer(orderOf(members, reachOfMembers(members, entries, read, edges), edges, changes))
|
||||
case "check":
|
||||
if *group == "" && len(members) == 1 {
|
||||
// One head, checked again as the forge's announcement would have it (a recheck): the controller's
|
||||
@@ -976,7 +861,7 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s (%s at %s) is let go by %s; its first tier is asked at the next pass\n", p.ID, p.Repository,
|
||||
fmt.Printf("%s (%s at %s) is let go by %s; its first tier is asked when the controller next looks at the plans, which it does as this ends\n", p.ID, p.Repository,
|
||||
short(p.Commit), *by)
|
||||
return nil
|
||||
case "stop":
|
||||
@@ -999,6 +884,19 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
fmt.Printf("%s stopped by %s at tier %d of %d; what was asked still builds and registers, nothing further "+
|
||||
"is asked or sent\n", p.ID, *by, p.Tier, len(p.Tiers))
|
||||
return nil
|
||||
case "backups-again":
|
||||
if len(positionals) != 1 || strings.TrimSpace(*why) == "" {
|
||||
return errors.New("delivery backups-again <plan> --why <text> [--by <who>]")
|
||||
}
|
||||
var p inventory.Plan
|
||||
var again []inventory.PlanBackup
|
||||
if err := sayingOnTheBus(ctx, inv, func() (err error) {
|
||||
p, again, err = backupsAgain(ctx, inv, positionals[0], *by, strings.TrimSpace(*why))
|
||||
return err
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
return answer(map[string]any{"plan": p.ID, "asked_again": again})
|
||||
case "walks":
|
||||
var walks []inventory.Plan
|
||||
if *planID != "" {
|
||||
@@ -1039,7 +937,7 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
return answer(map[string]any{"held": deliverySeatHeld(entries), "walks": walks,
|
||||
"own-path": sortedKeysOf(ownPathWords())})
|
||||
}
|
||||
return fmt.Errorf("delivery %s: plan, order, check, go, stop or walks", sub)
|
||||
return fmt.Errorf("delivery %s: plan, order, check, go, stop, backups-again or walks", sub)
|
||||
}
|
||||
|
||||
// appliedFrom answers a machine's own report of the send a walk made to it from the controller's `apply` durations
|
||||
@@ -1129,6 +1027,46 @@ func theGraph(ctx context.Context, inv *inventory.Inventory) ([]inventory.Entry,
|
||||
return entries, read, edges, nil
|
||||
}
|
||||
|
||||
// cycleVerdict is the failing verdict of a group whose modules are ordered in a cycle, and whether there is one: the
|
||||
// group's composed check, said without a build. The manifests' change is read from the forge; an answer it cannot give
|
||||
// is an error, never read as no cycle.
|
||||
func cycleVerdict(ctx context.Context, group string, members []orderMember, reach map[string]orderReach,
|
||||
edges []inventory.Edge, entries []inventory.Entry) (link.Checked, bool, error) {
|
||||
changes, err := changesOf(ctx, changeSourcesOf(members, entries, reach))
|
||||
if err != nil {
|
||||
return link.Checked{}, false, fmt.Errorf("what the group's members say of their manifests cannot be read: %w", err)
|
||||
}
|
||||
var moved []string
|
||||
for _, m := range members {
|
||||
moved = appendNew(moved, reach[m.ID].Moved...)
|
||||
}
|
||||
said := orderEdges(moved, changes, edges).CycleSaid()
|
||||
if said == "" {
|
||||
return link.Checked{}, false, nil
|
||||
}
|
||||
first := members[0].pull()
|
||||
c := link.Checked{ID: link.NewBuildID(time.Now()), Owner: first.Owner, Repo: first.Repo, Number: first.Number,
|
||||
Commit: first.Commit, Verdict: "fail", Summary: said, Report: said, Group: group,
|
||||
Gate: &link.CheckLayer{Verdict: "fail", Summary: said}}
|
||||
for _, m := range members {
|
||||
if group == "" {
|
||||
break // a lone pull request's verdict names no group and no member
|
||||
}
|
||||
p := m.pull()
|
||||
c.Members = append(c.Members, link.CheckedMember{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Commit: p.Commit,
|
||||
Trunk: twelve(p.BaseSHA)})
|
||||
}
|
||||
return c, true, nil
|
||||
}
|
||||
|
||||
// twelve is a commit as twelve characters.
|
||||
func twelve(c string) string {
|
||||
if len(c) > 12 {
|
||||
return c[:12]
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// groupPrimary is the head a group's composed check is run from: the one moving the controller, which then
|
||||
// judges the group by itself; else the one moving the node-engine, whose validator judges; else the first.
|
||||
func groupPrimary(members []orderMember, reach map[string]orderReach) int {
|
||||
@@ -1151,6 +1089,14 @@ func askGroupCheck(ctx context.Context, open *stores, group string, members []or
|
||||
return "", err
|
||||
}
|
||||
reach := reachOfMembers(members, entries, read, edges)
|
||||
// **A cycle of edges fails the group's composed check, naming every module and edge** (novox/hq ADR 0419 clause 6),
|
||||
// before the build seat is asked: no composition of the heads has an order to be sent in.
|
||||
if failing, cyclic, err := cycleVerdict(ctx, group, members, reach, edges, entries); err != nil {
|
||||
return "", err
|
||||
} else if cyclic {
|
||||
sayChecked(ctx, failing)
|
||||
return failing.ID, nil
|
||||
}
|
||||
primary := groupPrimary(members, reach)
|
||||
p := members[primary].pull()
|
||||
scope := pullScope(p, entries, read, edges)
|
||||
@@ -1165,6 +1111,8 @@ func askGroupCheck(ctx context.Context, open *stores, group string, members []or
|
||||
return "", err
|
||||
}
|
||||
request.Check.Group = group
|
||||
// A group's check is queued as ready when any member's pull request is marked ready.
|
||||
request.Check.Ready = slices.ContainsFunc(members, func(m orderMember) bool { return m.Ready })
|
||||
world, err := theRestOfTheMesh(ctx, open.inventory, shelfOf(entries), "")
|
||||
if err != nil {
|
||||
return "", err
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"path"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// What a change says of the manifests it touches, read from the forge (novox/hq ADR 0419 clause 3, the lead's settlement
|
||||
// of issue 977 question 1): the controller asks the forge for each changed module.json at the commit and at what it
|
||||
// replaced, once per commit. The composed check's result is not used: the plan is cut after the merge.
|
||||
|
||||
// changeSource is one change to a repository whose manifests are read: the commit after it and, when known, what it
|
||||
// replaced (a pull request's base commit or branch). An empty Old is the first parent of New: the merge's base.
|
||||
type changeSource struct {
|
||||
Owner, Repo string
|
||||
New, Old string
|
||||
Paths []string
|
||||
Truncated bool
|
||||
Removed []string
|
||||
// Since is the first commit of the repository in the change, when the change is several merges: what it replaced
|
||||
// is the first parent of that one, so a verb an earlier merge added is added by the change too.
|
||||
Since string
|
||||
}
|
||||
|
||||
// manifestAt reads a repository's module.json at a ref from the forge; the bool says it exists there. A variable so a
|
||||
// test needs no forge.
|
||||
var manifestAt = func(ctx context.Context, owner, repo, file, ref string) (catalogue.Manifest, bool, error) {
|
||||
var found struct {
|
||||
Result struct {
|
||||
Content string `json:"content"`
|
||||
Encoding string `json:"encoding"`
|
||||
} `json:"result"`
|
||||
}
|
||||
missing, err := askForgeApi(ctx, "/repos/"+owner+"/"+repo+"/contents/"+escapedPath(file)+"?ref="+url.QueryEscape(ref), &found)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, false, err
|
||||
}
|
||||
if missing {
|
||||
return catalogue.Manifest{}, false, nil
|
||||
}
|
||||
body := []byte(found.Result.Content)
|
||||
if found.Result.Encoding == "base64" {
|
||||
if body, err = base64.StdEncoding.DecodeString(strings.ReplaceAll(found.Result.Content, "\n", "")); err != nil {
|
||||
return catalogue.Manifest{}, false, fmt.Errorf("%s/%s %s at %s cannot be read: %w", owner, repo, file, ref, err)
|
||||
}
|
||||
}
|
||||
var m catalogue.Manifest
|
||||
if err := json.Unmarshal(body, &m); err != nil {
|
||||
return catalogue.Manifest{}, false, fmt.Errorf("%s/%s %s at %s is not a manifest: %w", owner, repo, file, ref, err)
|
||||
}
|
||||
return m, true, nil
|
||||
}
|
||||
|
||||
// parentOf is the first parent of a commit, from the forge. A variable so a test needs no forge.
|
||||
var parentOf = func(ctx context.Context, owner, repo, commit string) (string, error) {
|
||||
var found struct {
|
||||
Result struct {
|
||||
Parents []struct {
|
||||
SHA string `json:"sha"`
|
||||
} `json:"parents"`
|
||||
} `json:"result"`
|
||||
}
|
||||
if _, err := askForgeApi(ctx, "/repos/"+owner+"/"+repo+"/git/commits/"+url.PathEscape(commit), &found); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(found.Result.Parents) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
return found.Result.Parents[0].SHA, nil
|
||||
}
|
||||
|
||||
func escapedPath(p string) string {
|
||||
parts := strings.Split(p, "/")
|
||||
for i := range parts {
|
||||
parts[i] = url.PathEscape(parts[i])
|
||||
}
|
||||
return strings.Join(parts, "/")
|
||||
}
|
||||
|
||||
// askForgeApi makes one read of the forge's API through its module's tool on the bus. It answers true when the forge
|
||||
// says there is no such thing (404), and an error for anything else it cannot answer.
|
||||
func askForgeApi(ctx context.Context, apiPath string, into any) (bool, error) {
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer js.Close()
|
||||
raw, _ := json.Marshal(map[string]any{"path": apiPath})
|
||||
answer, err := link.Ask(ctx, link.OverNATS{Conn: js.Conn()}, "gitea", "gitea_api", raw, 30*time.Second)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
if strings.Contains(answer.Error, " 404 ") {
|
||||
return true, nil
|
||||
}
|
||||
return false, fmt.Errorf("the forge did not answer %s: %s", apiPath, answer.Error)
|
||||
}
|
||||
if err := json.Unmarshal(answer.Result, into); err != nil {
|
||||
return false, fmt.Errorf("the forge's answer to %s cannot be read: %w", apiPath, err)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// manifestsRead keeps what the forge said of a manifest at a commit, which never changes: one ask per commit.
|
||||
var (
|
||||
manifestsRead = map[string]manifestRead{}
|
||||
manifestsReadMu sync.Mutex
|
||||
)
|
||||
|
||||
type manifestRead struct {
|
||||
manifest catalogue.Manifest
|
||||
exists bool
|
||||
}
|
||||
|
||||
func manifestCached(ctx context.Context, owner, repo, file, ref string) (catalogue.Manifest, bool, error) {
|
||||
key := strings.Join([]string{owner, repo, file, ref}, "\x00")
|
||||
if len(ref) == 40 {
|
||||
manifestsReadMu.Lock()
|
||||
got, ok := manifestsRead[key]
|
||||
manifestsReadMu.Unlock()
|
||||
if ok {
|
||||
return got.manifest, got.exists, nil
|
||||
}
|
||||
}
|
||||
m, exists, err := manifestAt(ctx, owner, repo, file, ref)
|
||||
if err == nil && len(ref) == 40 {
|
||||
manifestsReadMu.Lock()
|
||||
if len(manifestsRead) > 512 {
|
||||
clear(manifestsRead)
|
||||
}
|
||||
manifestsRead[key] = manifestRead{m, exists}
|
||||
manifestsReadMu.Unlock()
|
||||
}
|
||||
return m, exists, err
|
||||
}
|
||||
|
||||
// verbsServed are the seat verbs a manifest serves, as "<seat>.<verb>": those of the seats it defines, those of the
|
||||
// seats it claims, and its own tools as "<module>.<tool>".
|
||||
func verbsServed(m catalogue.Manifest) []string {
|
||||
var out []string
|
||||
for _, s := range m.DefinesSeats {
|
||||
for _, v := range s.Serves {
|
||||
out = appendNew(out, s.Name+"."+v.Name)
|
||||
}
|
||||
}
|
||||
for _, c := range m.Claims {
|
||||
for _, v := range c.ServesFor(m) {
|
||||
out = appendNew(out, c.Name+"."+v)
|
||||
}
|
||||
}
|
||||
for _, t := range m.Tools {
|
||||
out = appendNew(out, m.Module+"."+t)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// verbsInvoked are the verbs a manifest calls, as "<seat>.<verb>" or "<module>.<tool>".
|
||||
func verbsInvoked(m catalogue.Manifest) []string {
|
||||
var out []string
|
||||
for _, i := range m.Invokes {
|
||||
if i = strings.TrimPrefix(i, "seat:"); i != "*" && i != "" {
|
||||
out = appendNew(out, i)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func added(now, was []string) []string {
|
||||
var out []string
|
||||
for _, v := range now {
|
||||
if !slices.Contains(was, v) {
|
||||
out = append(out, v)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// changesOf reads, for each manifest the changes touch, what they add to the verbs its module serves and invokes, by
|
||||
// module. A change whose paths are unsaid or truncated names no manifest: nothing is read for it. An error is the
|
||||
// forge's, never read as "no change": the plan is not cut on a guess.
|
||||
func changesOf(ctx context.Context, sources []changeSource) (map[string]moduleChange, error) {
|
||||
out := map[string]moduleChange{}
|
||||
for _, s := range sources {
|
||||
if s.Truncated {
|
||||
continue
|
||||
}
|
||||
for _, p := range s.Paths {
|
||||
if path.Base(p) != "module.json" || slices.Contains(s.Removed, p) {
|
||||
continue
|
||||
}
|
||||
now, exists, err := manifestCached(ctx, s.Owner, s.Repo, p, s.New)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !exists || now.Module == "" {
|
||||
continue
|
||||
}
|
||||
old := s.Old
|
||||
if old == "" {
|
||||
first := s.Since
|
||||
if first == "" {
|
||||
first = s.New
|
||||
}
|
||||
if old, err = parentOf(ctx, s.Owner, s.Repo, first); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var was catalogue.Manifest
|
||||
if old != "" {
|
||||
if was, _, err = manifestCached(ctx, s.Owner, s.Repo, p, old); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
c := out[now.Module]
|
||||
c.Manifest = true
|
||||
for _, v := range added(verbsServed(now), verbsServed(was)) {
|
||||
c.Serves = appendNew(c.Serves, v)
|
||||
}
|
||||
for _, v := range added(verbsInvoked(now), verbsInvoked(was)) {
|
||||
c.Invokes = appendNew(c.Invokes, v)
|
||||
}
|
||||
out[now.Module] = c
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// truncatedPaths are the manifests a change that names only a beginning of its files is read as changing: those of the
|
||||
// modules it moves that are built from its repository. Said, never skipped: no edge may be lost to a long list.
|
||||
func truncatedPaths(entries []inventory.Entry, owner, repo string, moved []string) []string {
|
||||
var out []string
|
||||
for _, e := range entries {
|
||||
if slices.Contains(moved, e.Manifest.Module) && sameRepository(e.Source.Repository, link.SourceMoved{Owner: owner, Repo: repo}) {
|
||||
out = append(out, path.Join(strings.Trim(e.Source.Path, "/"), moduleManifestFile))
|
||||
}
|
||||
}
|
||||
fmt.Printf(" %s/%s changed more files than were listed: the manifests of its modules (%s) are read as changed\n",
|
||||
owner, repo, strings.Join(out, ", "))
|
||||
return out
|
||||
}
|
||||
|
||||
// changeSourcesOf are a group's members as changes to read: each head over what its pull request merges into.
|
||||
func changeSourcesOf(members []orderMember, entries []inventory.Entry, reach map[string]orderReach) []changeSource {
|
||||
var out []changeSource
|
||||
for _, m := range members {
|
||||
p := m.pull()
|
||||
if p.PathsTruncated {
|
||||
p.Paths, p.PathsTruncated = truncatedPaths(entries, p.Owner, p.Repo, reach[m.ID].Moved), false
|
||||
}
|
||||
old := p.BaseSHA
|
||||
if old == "" {
|
||||
old = p.Base
|
||||
}
|
||||
out = append(out, changeSource{Owner: p.Owner, Repo: p.Repo, New: p.Commit, Old: old, Paths: p.Paths,
|
||||
Truncated: p.PathsTruncated, Removed: p.Removed})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The order of a plan's modules is read from the module graph and from what the change says of the manifests
|
||||
// (novox/hq ADR 0419), per module and never per repository: no `after:` line is read, and no member of a group
|
||||
// is ordered against another. Pure, so a test needs no store.
|
||||
|
||||
// The rules that draw an edge of a plan's order, in the words the plan shows.
|
||||
const (
|
||||
ruleVerbBeforeCaller = "verb before caller"
|
||||
ruleVersionSkew = "version skew"
|
||||
ruleEngineFirst = "engine before controller"
|
||||
)
|
||||
|
||||
// The two modules the rollout rules name.
|
||||
const (
|
||||
theController = "mesh-controller"
|
||||
theEngine = "mesh-host"
|
||||
)
|
||||
|
||||
// moduleChange is what a change says of one module's manifest: the seat verbs ("<seat>.<verb>") it adds to what the
|
||||
// module serves, the seat verbs it adds to what the module invokes, and whether the manifest changes at all.
|
||||
type moduleChange struct {
|
||||
Serves []string
|
||||
Invokes []string
|
||||
Manifest bool
|
||||
}
|
||||
|
||||
// planOrder is the edges the rules draw among a plan's modules, and the cycle they make when they make one.
|
||||
type planOrder struct {
|
||||
Edges []inventory.Edge
|
||||
// Cycle is every edge of a cycle among the order's edges and the graph's, the module graph's included.
|
||||
Cycle []inventory.Edge
|
||||
}
|
||||
|
||||
// CycleSaid names every module and edge of the cycle, for a check to fail with (ADR 0419 clause 6).
|
||||
func (o planOrder) CycleSaid() string {
|
||||
if len(o.Cycle) == 0 {
|
||||
return ""
|
||||
}
|
||||
var parts []string
|
||||
for _, e := range o.Cycle {
|
||||
parts = append(parts, edgeSaid(e))
|
||||
}
|
||||
return "the modules of the plan are ordered in a cycle: " + strings.Join(parts, "; ")
|
||||
}
|
||||
|
||||
// edgeSaid is an edge as a person reads it: the module after, the module before, and why.
|
||||
func edgeSaid(e inventory.Edge) string {
|
||||
why := e.Why
|
||||
if why == "" {
|
||||
why = e.Kind
|
||||
}
|
||||
return fmt.Sprintf("%s after %s (%s)", e.From, e.To, why)
|
||||
}
|
||||
|
||||
// orders is whether an edge of the module graph orders a plan's tiers (tiersOf).
|
||||
func orders(e inventory.Edge) bool {
|
||||
return e.Kind != inventory.EdgePackages && e.Kind != inventory.EdgeBuiltBy && e.From != e.To
|
||||
}
|
||||
|
||||
// orderEdges draws the edges of a plan's order among its moved modules (ADR 0419 clause 3, 4): a module whose change
|
||||
// adds a verb to a seat goes before one whose change adds a call of that verb; the controller, when the plan moves
|
||||
// it, goes before every module whose manifest the plan changes, the node-engine excepted; and the node-engine goes
|
||||
// before the controller by default, when the plan moves both and no edge orders them either way, transitively. An
|
||||
// edge is "From after To". graph is the module graph, whose edges order too and are never drawn again.
|
||||
func orderEdges(moved []string, changes map[string]moduleChange, graph []inventory.Edge) planOrder {
|
||||
var out planOrder
|
||||
sorted := append([]string(nil), moved...)
|
||||
sort.Strings(sorted)
|
||||
for _, server := range sorted {
|
||||
for _, caller := range sorted {
|
||||
if server == caller {
|
||||
continue
|
||||
}
|
||||
for _, verb := range changes[server].Serves {
|
||||
if slices.Contains(changes[caller].Invokes, verb) {
|
||||
out.Edges = append(out.Edges, inventory.Edge{From: caller, To: server, Kind: edgeChangeOrder,
|
||||
Why: ruleVerbBeforeCaller + ": " + verb})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if slices.Contains(sorted, theController) {
|
||||
for _, m := range sorted {
|
||||
if m != theController && m != theEngine && changes[m].Manifest {
|
||||
out.Edges = append(out.Edges, inventory.Edge{From: m, To: theController, Kind: edgeChangeOrder,
|
||||
Why: ruleVersionSkew})
|
||||
}
|
||||
}
|
||||
}
|
||||
if slices.Contains(sorted, theController) && slices.Contains(sorted, theEngine) {
|
||||
all := orderingEdges(graph, out.Edges)
|
||||
switch {
|
||||
case pathBetween(all, theController, theEngine) != nil:
|
||||
// An edge already puts the controller after the engine: the default says the same.
|
||||
case pathBetween(all, theEngine, theController) != nil:
|
||||
// An edge puts the engine after the controller: it overrules the default, and says so.
|
||||
path := pathBetween(all, theEngine, theController)
|
||||
for i := range out.Edges {
|
||||
if out.Edges[i] == path[0] {
|
||||
out.Edges[i].Why += " (overrules " + ruleEngineFirst + ")"
|
||||
break
|
||||
}
|
||||
}
|
||||
default:
|
||||
out.Edges = append(out.Edges, inventory.Edge{From: theController, To: theEngine, Kind: edgeChangeOrder,
|
||||
Why: ruleEngineFirst})
|
||||
}
|
||||
}
|
||||
all := orderingEdges(graph, out.Edges)
|
||||
for _, e := range out.Edges {
|
||||
if back := pathBetween(all, e.To, e.From); back != nil {
|
||||
out.Cycle = append(out.Cycle, e)
|
||||
out.Cycle = append(out.Cycle, back...)
|
||||
break
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// orderingEdges is the graph's edges that order and the edges drawn, together.
|
||||
func orderingEdges(graph, drawn []inventory.Edge) []inventory.Edge {
|
||||
var out []inventory.Edge
|
||||
for _, e := range graph {
|
||||
if orders(e) {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
return append(out, drawn...)
|
||||
}
|
||||
|
||||
// pathBetween is the edges that lead from a module to another along "depends on" (From to To), the shortest, or nil
|
||||
// when the first does not depend on the second.
|
||||
func pathBetween(edges []inventory.Edge, from, to string) []inventory.Edge {
|
||||
if from == to {
|
||||
return nil
|
||||
}
|
||||
via := map[string]inventory.Edge{}
|
||||
seen := map[string]bool{from: true}
|
||||
queue := []string{from}
|
||||
for len(queue) > 0 {
|
||||
at := queue[0]
|
||||
queue = queue[1:]
|
||||
for _, e := range edges {
|
||||
if e.From != at || seen[e.To] {
|
||||
continue
|
||||
}
|
||||
seen[e.To] = true
|
||||
via[e.To] = e
|
||||
if e.To == to {
|
||||
var path []inventory.Edge
|
||||
for m := to; m != from; m = via[m].From {
|
||||
path = append([]inventory.Edge{via[m]}, path...)
|
||||
}
|
||||
return path
|
||||
}
|
||||
queue = append(queue, e.To)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The order of a plan's modules is read from the module graph and the manifests' change alone (novox/hq ADR 0419):
|
||||
// no `after:` line, no member of a group. Pure, like orderOf was.
|
||||
|
||||
func hasEdge(edges []inventory.Edge, from, to string) (inventory.Edge, bool) {
|
||||
for _, e := range edges {
|
||||
if e.From == from && e.To == to {
|
||||
return e, true
|
||||
}
|
||||
}
|
||||
return inventory.Edge{}, false
|
||||
}
|
||||
|
||||
// A module whose change adds a verb to a seat goes before the module whose change adds a call of it: the caller
|
||||
// depends on the server, the plan puts them in two tiers, and the edge says why.
|
||||
func TestAVerbIsServedBeforeItsCallerIsSent(t *testing.T) {
|
||||
changes := map[string]moduleChange{
|
||||
"server": {Serves: []string{"mesh-delivery.groups"}, Manifest: true},
|
||||
"caller": {Invokes: []string{"mesh-delivery.groups"}, Manifest: true},
|
||||
}
|
||||
got := orderEdges([]string{"caller", "server"}, changes, nil)
|
||||
e, ok := hasEdge(got.Edges, "caller", "server")
|
||||
if !ok || !strings.Contains(e.Why, "verb before caller") || !strings.Contains(e.Why, "mesh-delivery.groups") {
|
||||
t.Fatalf("the caller does not follow the server by the verb: %+v", got.Edges)
|
||||
}
|
||||
if len(got.Cycle) != 0 {
|
||||
t.Fatalf("a cycle: %+v", got.Cycle)
|
||||
}
|
||||
if tiers := tiersOf([]string{"caller", "server"}, got.Edges); len(tiers) != 2 || tiers[0][0] != "server" {
|
||||
t.Fatalf("tiers %v, want the server first", tiers)
|
||||
}
|
||||
}
|
||||
|
||||
// Edges of one rule both ways are a cycle: every module and edge is named.
|
||||
func TestEdgesBothWaysAreACycleNamingEveryModuleAndEdge(t *testing.T) {
|
||||
changes := map[string]moduleChange{
|
||||
"x": {Serves: []string{"s.a"}, Invokes: []string{"s.b"}},
|
||||
"y": {Serves: []string{"s.b"}, Invokes: []string{"s.a"}},
|
||||
}
|
||||
got := orderEdges([]string{"x", "y"}, changes, nil)
|
||||
if len(got.Cycle) < 2 {
|
||||
t.Fatalf("no cycle found: %+v", got)
|
||||
}
|
||||
said := got.CycleSaid()
|
||||
for _, want := range []string{"x", "y", "s.a", "s.b"} {
|
||||
if !strings.Contains(said, want) {
|
||||
t.Fatalf("the cycle is said as %q, without %s", said, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The node-engine goes before the controller by default, and says so; an edge of the manifests the other way
|
||||
// overrules it, and the edge that did is said.
|
||||
func TestTheEngineGoesBeforeTheControllerUnlessAnEdgeSaysOtherwise(t *testing.T) {
|
||||
got := orderEdges([]string{"mesh-controller", "mesh-host"}, nil, nil)
|
||||
e, ok := hasEdge(got.Edges, "mesh-controller", "mesh-host")
|
||||
if !ok || e.Why != "engine before controller" {
|
||||
t.Fatalf("the engine does not go first: %+v", got.Edges)
|
||||
}
|
||||
changes := map[string]moduleChange{
|
||||
"mesh-host": {Serves: []string{"s.v"}},
|
||||
"mesh-controller": {Invokes: []string{"s.v"}},
|
||||
}
|
||||
got = orderEdges([]string{"mesh-controller", "mesh-host"}, changes, nil)
|
||||
if _, ok := hasEdge(got.Edges, "mesh-controller", "mesh-host"); !ok || len(got.Edges) != 1 ||
|
||||
!strings.Contains(got.Edges[0].Why, "verb before caller") {
|
||||
t.Fatalf("the verb edge does not place the controller after the engine: %+v", got.Edges)
|
||||
}
|
||||
// The other way: the engine calls what the controller brings.
|
||||
changes = map[string]moduleChange{
|
||||
"mesh-controller": {Serves: []string{"s.v"}},
|
||||
"mesh-host": {Invokes: []string{"s.v"}},
|
||||
}
|
||||
got = orderEdges([]string{"mesh-controller", "mesh-host"}, changes, nil)
|
||||
e, ok = hasEdge(got.Edges, "mesh-host", "mesh-controller")
|
||||
if !ok || len(got.Edges) != 1 || !strings.Contains(e.Why, "overrules engine before controller") {
|
||||
t.Fatalf("the verb edge does not say it overruled the default: %+v", got.Edges)
|
||||
}
|
||||
}
|
||||
|
||||
// The controller goes before every module whose manifest the plan changes (version skew), the node-engine excepted.
|
||||
func TestTheControllerGoesBeforeEveryChangedManifest(t *testing.T) {
|
||||
changes := map[string]moduleChange{"app": {Manifest: true}, "other": {}}
|
||||
got := orderEdges([]string{"mesh-controller", "app", "other"}, changes, nil)
|
||||
if e, ok := hasEdge(got.Edges, "app", "mesh-controller"); !ok || e.Why != "version skew" {
|
||||
t.Fatalf("the changed manifest does not follow the controller: %+v", got.Edges)
|
||||
}
|
||||
if _, ok := hasEdge(got.Edges, "other", "mesh-controller"); ok {
|
||||
t.Fatalf("a module with an unchanged manifest was ordered: %+v", got.Edges)
|
||||
}
|
||||
if got := orderEdges([]string{"app", "other"}, changes, nil); len(got.Edges) != 0 {
|
||||
t.Fatalf("version skew ordered a plan that does not move the controller: %+v", got.Edges)
|
||||
}
|
||||
}
|
||||
|
||||
// The order is per module, not per repository: of two modules of one repository only the one with the edge moves.
|
||||
func TestTheOrderIsPerModuleNotPerRepository(t *testing.T) {
|
||||
changes := map[string]moduleChange{
|
||||
"a": {Serves: []string{"s.v"}},
|
||||
"c": {Invokes: []string{"s.v"}},
|
||||
}
|
||||
got := orderEdges([]string{"a", "b", "c"}, changes, nil)
|
||||
tiers := tiersOf([]string{"a", "b", "c"}, got.Edges)
|
||||
if len(tiers) != 2 || strings.Join(tiers[0], ",") != "a,b" || strings.Join(tiers[1], ",") != "c" {
|
||||
t.Fatalf("tiers %v, want [a b] then [c]", tiers)
|
||||
}
|
||||
}
|
||||
|
||||
// The edges a plan's order was drawn with are kept on the plan, and its gates read them: a later tier waits for the
|
||||
// earlier to be applied although the catalogue's graph holds no such edge (ADR 0419, the lead's settlement Q4).
|
||||
func TestAPlansOwnOrderGatesItsTiersWhateverTheStoreGraphHolds(t *testing.T) {
|
||||
p := inventory.Plan{Tiers: [][]string{{"server"}, {"caller"}}, Tier: 0,
|
||||
Delivery: &inventory.PlanDelivery{Order: []inventory.Edge{{From: "caller", To: "server", Kind: edgeChangeOrder,
|
||||
Why: ruleVerbBeforeCaller + ": s.v"}}}}
|
||||
if got := gates(p, nil, func(string) bool { return true }); len(got) != 1 || got[0] != "server" {
|
||||
t.Fatalf("the caller's tier waits for %v; want the server", got)
|
||||
}
|
||||
p.Delivery.Order = nil
|
||||
if got := gates(p, nil, func(string) bool { return true }); len(got) != 0 {
|
||||
t.Fatalf("a plan with no order waits for %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A verb an earlier merge of the same repository in the batch added is added by the batch: the change is compared with
|
||||
// what the repository held before the batch, not with the first parent of its latest merge.
|
||||
func TestAVerbAnEarlierMergeOfTheBatchAddedDrawsItsEdge(t *testing.T) {
|
||||
was, wasParent := manifestAt, parentOf
|
||||
t.Cleanup(func() { manifestAt, parentOf = was, wasParent })
|
||||
manifestsReadMu.Lock()
|
||||
clear(manifestsRead)
|
||||
manifestsReadMu.Unlock()
|
||||
const first, second, before = "1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222", "0000000000000000000000000000000000000000"
|
||||
parentOf = func(_ context.Context, _, _, commit string) (string, error) {
|
||||
if commit == first {
|
||||
return before, nil
|
||||
}
|
||||
return first, nil // the latest merge's parent holds the verb already
|
||||
}
|
||||
manifestAt = func(_ context.Context, _, _, _, ref string) (catalogue.Manifest, bool, error) {
|
||||
m := catalogue.Manifest{Module: "server"}
|
||||
if ref != before {
|
||||
m.DefinesSeats = []catalogue.SeatDeclaration{{Name: "s", Serves: []catalogue.Verb{{Name: "v"}}}}
|
||||
}
|
||||
return m, true, nil
|
||||
}
|
||||
src := changeSource{Owner: "novox", Repo: "r", New: second, Paths: []string{"module.json"}}
|
||||
if got, err := changesOf(context.Background(), []changeSource{src}); err != nil || len(got["server"].Serves) != 0 {
|
||||
t.Fatalf("against the first parent of the latest merge: %+v %v", got, err)
|
||||
}
|
||||
src.Since = first
|
||||
got, err := changesOf(context.Background(), []changeSource{src})
|
||||
if err != nil || len(got["server"].Serves) != 1 || got["server"].Serves[0] != "s.v" {
|
||||
t.Fatalf("against what the batch found: %+v %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A change that names only a beginning of its files is read as changing the manifests of the modules it moves.
|
||||
func TestATruncatedChangeIsReadAsChangingTheManifestsOfWhatItMoves(t *testing.T) {
|
||||
entries := []inventory.Entry{
|
||||
{Manifest: catalogue.Manifest{Module: "a"}, Source: inventory.Source{Repository: "novox/mesh-catalog", Path: "modules/a"}},
|
||||
{Manifest: catalogue.Manifest{Module: "b"}, Source: inventory.Source{Repository: "novox/mesh-catalog", Path: "modules/b"}},
|
||||
{Manifest: catalogue.Manifest{Module: "c"}, Source: inventory.Source{Repository: "novox/other", Path: "modules/c"}},
|
||||
}
|
||||
got := truncatedPaths(entries, "novox", "mesh-catalog", []string{"a", "c"})
|
||||
if len(got) != 1 || got[0] != "modules/a/module.json" {
|
||||
t.Fatalf("read as changed: %v", got)
|
||||
}
|
||||
members := []orderMember{{ID: "m", Repository: "novox/mesh-catalog", Head: "h", PathsTruncated: true, Paths: []string{"x"}}}
|
||||
src := changeSourcesOf(members, entries, map[string]orderReach{"m": {Moved: []string{"a"}}})
|
||||
if len(src) != 1 || src[0].Truncated || len(src[0].Paths) != 1 || src[0].Paths[0] != "modules/a/module.json" {
|
||||
t.Fatalf("the member's change is read as %+v", src)
|
||||
}
|
||||
}
|
||||
|
||||
// A lone pull request's cycle fails its check, naming every module and edge, with no group and no members.
|
||||
func TestALonePullRequestInACycleFailsWithNoGroup(t *testing.T) {
|
||||
was, wasParent := manifestAt, parentOf
|
||||
t.Cleanup(func() { manifestAt, parentOf = was, wasParent })
|
||||
manifestsReadMu.Lock()
|
||||
clear(manifestsRead)
|
||||
manifestsReadMu.Unlock()
|
||||
const head = "3333333333333333333333333333333333333333"
|
||||
parentOf = func(context.Context, string, string, string) (string, error) { return "", nil }
|
||||
manifestAt = func(_ context.Context, _, _, file, ref string) (catalogue.Manifest, bool, error) {
|
||||
if ref == "main" {
|
||||
return catalogue.Manifest{}, false, nil
|
||||
}
|
||||
if file == "modules/x/module.json" {
|
||||
return catalogue.Manifest{Module: "mx", DefinesSeats: []catalogue.SeatDeclaration{{Name: "s", Serves: []catalogue.Verb{{Name: "a"}}}},
|
||||
Invokes: []string{"seat:s.b"}}, true, nil
|
||||
}
|
||||
return catalogue.Manifest{Module: "my", DefinesSeats: []catalogue.SeatDeclaration{{Name: "s", Serves: []catalogue.Verb{{Name: "b"}}}},
|
||||
Invokes: []string{"seat:s.a"}}, true, nil
|
||||
}
|
||||
m := orderMember{ID: "novox/r@" + head[:12], Repository: "novox/r", Base: "main", Head: head, Number: 5,
|
||||
Paths: []string{"modules/x/module.json", "modules/y/module.json"}}
|
||||
c, cyclic, err := cycleVerdict(context.Background(), "", []orderMember{m}, map[string]orderReach{m.ID: {Moved: []string{"mx", "my"}}}, nil, nil)
|
||||
if err != nil || !cyclic || c.Verdict != "fail" || c.Group != "" || len(c.Members) != 0 || !strings.Contains(c.Summary, "s.a") ||
|
||||
!strings.Contains(c.Summary, "my") {
|
||||
t.Fatalf("the verdict is %+v (cyclic %v, %v)", c, cyclic, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asked := asksRecorded(t)
|
||||
withConditionsInMemory(t)
|
||||
for _, name := range []string{"app", "mesh-delivery"} {
|
||||
m := catalogue.Manifest{Module: name, Version: "1"}
|
||||
if name == "mesh-delivery" {
|
||||
m.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
|
||||
Path: "modules/" + name, Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
merge := func(commit string, paths ...string) inventory.Plan {
|
||||
t.Helper()
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit, Paths: paths,
|
||||
ModuleDirs: []string{"modules/app", "modules/mesh-delivery"}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recent, err := inv.RecentPlans(ctx, 1)
|
||||
if err != nil || len(recent) != 1 || recent[0].Commit != commit {
|
||||
t.Fatalf("no plan for %s: %v %v", commit, recent, err)
|
||||
}
|
||||
return recent[0]
|
||||
}
|
||||
|
||||
// finish ends a walk as done, so the next merge has nothing of it to take over.
|
||||
finish := func(id string) {
|
||||
t.Helper()
|
||||
w, err := inv.PlanByID(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.State = inventory.PlanDone
|
||||
if err := inv.SavePlan(ctx, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// No holder on record: the merge starts its walk, as before.
|
||||
p := merge("c1aaaaaaaa", "modules/app/index.ts")
|
||||
if p.Waiting() || len(*asked) != 1 {
|
||||
t.Fatalf("with no holder on record the walk waited (%v) or asked %v", p.Waiting(), *asked)
|
||||
}
|
||||
|
||||
// The holder on record: the next merge waits, asking nothing, and an advance asks nothing either. One walk
|
||||
// is open at a time (novox/hq ADR 0276): the first ends before the next merge's batch is cut.
|
||||
finish(p.ID)
|
||||
if _, err := inv.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p = merge("c2bbbbbbbb", "modules/app/index.ts")
|
||||
if !p.Waiting() || len(*asked) != 1 || !strings.Contains(p.Note, "plans go "+p.ID) {
|
||||
t.Fatalf("the walk did not wait for its word: waiting %v, asked %v, note %q", p.Waiting(), *asked, p.Note)
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) != 1 {
|
||||
t.Fatalf("a waiting walk was advanced into asking: %v", *asked)
|
||||
}
|
||||
if line := planLine(p, p.Created); strings.Contains(line, "LATE") || !strings.Contains(line, "waits for") {
|
||||
t.Errorf("a waiting walk reads %q", line)
|
||||
}
|
||||
|
||||
// The delivery's word starts it.
|
||||
if err := deliveryCommand(ctx, []string{"go", p.ID, "--by", catalogue.DeliverySeat, "--why", "its turn"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := deliveryCommand(ctx, []string{"go", p.ID}); err == nil {
|
||||
t.Fatal("a walk was let go twice")
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) != 2 {
|
||||
t.Fatalf("the word did not start the walk: %v", *asked)
|
||||
}
|
||||
got, err := inv.PlanByID(ctx, p.ID)
|
||||
if err != nil || got.Delivery == nil || got.Delivery.By != catalogue.DeliverySeat || got.Delivery.Why != "its turn" {
|
||||
t.Fatalf("the word was not kept: %+v %v", got.Delivery, err)
|
||||
}
|
||||
|
||||
// The delivery's owner's own merge never waits for it. A walk that started is never taken over (novox/hq
|
||||
// ADR 0276): the merge's batch is cut once it ended.
|
||||
finish(p.ID)
|
||||
p = merge("c3cccccccc", "modules/mesh-delivery/main.go")
|
||||
if p.Waiting() || len(*asked) != 3 {
|
||||
t.Fatalf("mesh-delivery's own walk waited for mesh-delivery: %v %v", p.Waiting(), *asked)
|
||||
}
|
||||
|
||||
// The owner down: a person starts a waiting walk, with why.
|
||||
finish(p.ID)
|
||||
p = merge("c4dddddddd", "modules/app/index.ts")
|
||||
if !p.Waiting() {
|
||||
t.Fatal("the walk did not wait")
|
||||
}
|
||||
if err := plansCommand(ctx, []string{"go", p.ID}); err == nil || !strings.Contains(err.Error(), "--why") {
|
||||
t.Fatalf("a walk was started by hand without why: %v", err)
|
||||
}
|
||||
if err := plansCommand(ctx, []string{"go", p.ID, "--why", "mesh-delivery is down"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
got, _ = inv.PlanByID(ctx, p.ID)
|
||||
if got.Waiting() || !strings.HasPrefix(got.Delivery.By, "a person") || len(*asked) < 4 {
|
||||
t.Fatalf("a person's word did not start the walk: %+v, asked %v", got.Delivery, *asked)
|
||||
}
|
||||
|
||||
// The owner's stop: failed, said as stopped by it.
|
||||
finish(p.ID)
|
||||
p = merge("c5eeeeeeee", "modules/app/index.ts")
|
||||
if err := deliveryCommand(ctx, []string{"stop", p.ID, "--why", "the operator stopped it", "--by",
|
||||
"mesh-delivery for jochen"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ = inv.PlanByID(ctx, p.ID)
|
||||
if got.State != inventory.PlanFailed || got.Delivery.Stopped != "mesh-delivery for jochen" ||
|
||||
!strings.Contains(got.Note, "the operator stopped it") {
|
||||
t.Fatalf("the stop was kept as %s %+v %q", got.State, got.Delivery, got.Note)
|
||||
}
|
||||
|
||||
// Unassigned: the mesh is back on the controller's own path.
|
||||
if err := inv.Unassign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asks := len(*asked)
|
||||
if p = merge("c6ffffffff", "modules/app/index.ts"); p.Waiting() || len(*asked) != asks+1 {
|
||||
t.Fatalf("with the holder gone the walk waited: %v", p.Waiting())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWalkLetGoByAVerbIsSaidAsPlanMoved(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
url := testbus.URL(t)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := handActConn
|
||||
handActConn = js.Conn()
|
||||
t.Cleanup(func() { handActConn = before })
|
||||
heard := make(chan *nats.Msg, 4)
|
||||
sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), heard)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
|
||||
waiting := inventory.Plan{ID: "plan-waits", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c7c7c7c7",
|
||||
Created: time.Now().UTC(), State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
|
||||
Modules: map[string]*inventory.PlanModule{"app": {}},
|
||||
Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}}
|
||||
if err := open.inventory.SavePlan(ctx, &waiting); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case m := <-heard:
|
||||
var said inventory.Plan
|
||||
if err := json.Unmarshal(m.Data, &said); err != nil || said.ID != waiting.ID || said.Delivery == nil ||
|
||||
said.Delivery.Go == nil {
|
||||
t.Fatalf("plan-moved said %s (%v)", m.Data, err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("a walk let go by a verb was not said")
|
||||
}
|
||||
if checkEvents != nil || inventory.PlanSaved != nil {
|
||||
t.Fatal("the command's own bus was left in place")
|
||||
}
|
||||
}
|
||||
@@ -1,50 +1,44 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0239 decision 4: a group's order, from the graph and the pull requests, the same on every
|
||||
// reading — the build agent before what it builds, the controller before a manifest that needs it, the
|
||||
// node-engine before the controller, a declared `after:` — and a contradiction named, never ordered.
|
||||
func TestAGroupIsOrderedByTheGraphAndWhatItsPullRequestsSay(t *testing.T) {
|
||||
// novox/hq ADR 0419: a group's order, from the module graph and the manifests' change alone, the same on every
|
||||
// reading — what a module stands on before it, the controller before a manifest that needs it, the node-engine
|
||||
// before the controller, a verb's server before its caller — and no `after:` line read.
|
||||
func TestAGroupIsOrderedByTheGraphAndTheManifestsChange(t *testing.T) {
|
||||
members := []orderMember{
|
||||
{ID: "cat", Repository: "novox/mesh-catalog"},
|
||||
{ID: "ctl", Repository: "novox/mesh-controller"},
|
||||
{ID: "host", Repository: "novox/mesh-host"},
|
||||
{ID: "agent", Repository: "novox/build-agent"},
|
||||
{ID: "app", Repository: "novox/app", After: []string{"lab"}},
|
||||
{ID: "base", Repository: "novox/base"},
|
||||
{ID: "app", Repository: "novox/app"},
|
||||
{ID: "lab", Repository: "novox/lab"},
|
||||
}
|
||||
reach := map[string]orderReach{
|
||||
"cat": {Moved: []string{"gitea"}, Manifests: []string{"modules/gitea/module.json"}},
|
||||
"ctl": {Moved: []string{"mesh-controller"}, Manifests: []string{"module.json"}},
|
||||
"host": {Moved: []string{"mesh-host"}, Manifests: []string{"module.json"}},
|
||||
"agent": {Moved: []string{"build-agent"}},
|
||||
"app": {Moved: []string{"app"}},
|
||||
"lab": {Moved: []string{"lab"}},
|
||||
"cat": {Moved: []string{"gitea"}, Manifests: []string{"modules/gitea/module.json"}},
|
||||
"ctl": {Moved: []string{"mesh-controller"}, Manifests: []string{"module.json"}},
|
||||
"host": {Moved: []string{"mesh-host"}, Manifests: []string{"module.json"}},
|
||||
"base": {Moved: []string{"base"}},
|
||||
"app": {Moved: []string{"app"}},
|
||||
"lab": {Moved: []string{"lab"}},
|
||||
}
|
||||
edges := []inventory.Edge{{From: "app", To: "build-agent", Kind: inventory.EdgeBuiltBy},
|
||||
edges := []inventory.Edge{{From: "app", To: "base", Kind: inventory.EdgeStandsOn},
|
||||
{From: "gitea", To: "postgres", Kind: inventory.EdgeDeclared}}
|
||||
got := orderOf(members, reach, edges)
|
||||
changes := map[string]moduleChange{"gitea": {Manifest: true}, "mesh-controller": {Manifest: true},
|
||||
"mesh-host": {Manifest: true}, "lab": {Serves: []string{"seat.v"}}, "app": {Invokes: []string{"seat.v"}}}
|
||||
got := orderOf(members, reach, edges, changes)
|
||||
if len(got.Cycle) > 0 {
|
||||
t.Fatalf("a cycle where there is none: %v", got.Cycle)
|
||||
}
|
||||
// By repository among those with nothing before them: the build agent, the lab, then what waited on both.
|
||||
want := []string{"agent", "lab", "app", "host", "ctl", "cat"}
|
||||
// By repository among those with nothing before them: app waits for base and lab; gitea's catalogue for the controller.
|
||||
want := []string{"base", "lab", "app", "host", "ctl", "cat"}
|
||||
if !reflect.DeepEqual(got.Order, want) {
|
||||
t.Fatalf("ordered %v, wanted %v; pairs %+v", got.Order, want, got.Pairs)
|
||||
}
|
||||
@@ -52,95 +46,44 @@ func TestAGroupIsOrderedByTheGraphAndWhatItsPullRequestsSay(t *testing.T) {
|
||||
for _, p := range got.Pairs {
|
||||
why[p.Before+">"+p.After] = p.Why
|
||||
}
|
||||
for pair, reason := range map[string]string{"host>ctl": orderEngineFirst, "ctl>cat": orderVersionSkew,
|
||||
"ctl>host": "", "agent>app": orderBuiltBy, "lab>app": orderDeclared} {
|
||||
for pair, reason := range map[string]string{"host>ctl": ruleEngineFirst, "ctl>cat": ruleVersionSkew,
|
||||
"ctl>host": "", "base>app": orderBuiltBy, "lab>app": ruleVerbBeforeCaller + ": seat.v"} {
|
||||
if why[pair] != reason {
|
||||
t.Errorf("%s is ordered %q, wanted %q", pair, why[pair], reason)
|
||||
}
|
||||
}
|
||||
// The same members in another order read the same.
|
||||
shuffled := []orderMember{members[5], members[3], members[0], members[4], members[2], members[1]}
|
||||
if again := orderOf(shuffled, reach, edges); !reflect.DeepEqual(again.Order, want) {
|
||||
if again := orderOf(shuffled, reach, edges, changes); !reflect.DeepEqual(again.Order, want) {
|
||||
t.Fatalf("another reading ordered %v", again.Order)
|
||||
}
|
||||
|
||||
// A declared order against an inferred one wins (novox/hq ADR 0249), and the pair says what it won over.
|
||||
members[1].After = []string{"mesh-catalog"}
|
||||
got = orderOf(members, reach, edges)
|
||||
if len(got.Cycle) > 0 || len(got.Contradictions) > 0 {
|
||||
t.Fatalf("a declared order did not win: cycle %v, %+v", got.Cycle, got.Contradictions)
|
||||
}
|
||||
if want := []string{"agent", "lab", "app", "cat", "host", "ctl"}; !reflect.DeepEqual(got.Order, want) {
|
||||
t.Fatalf("ordered %v, wanted %v", got.Order, want)
|
||||
}
|
||||
if !slices.ContainsFunc(got.Pairs, func(p orderPair) bool {
|
||||
return p.Before == "cat" && p.After == "ctl" && p.Why == orderDeclared && reflect.DeepEqual(p.Over, []string{orderVersionSkew})
|
||||
}) || slices.ContainsFunc(got.Pairs, func(p orderPair) bool { return p.Before == "ctl" && p.After == "cat" }) {
|
||||
t.Fatalf("the declared pair does not say it won over version skew: %+v", got.Pairs)
|
||||
}
|
||||
|
||||
// Declared both ways is a contradiction no precedence resolves: a cycle, and how to end it.
|
||||
members[0].After = []string{"novox/mesh-controller"}
|
||||
got = orderOf(members, reach, edges)
|
||||
if !reflect.DeepEqual(got.Cycle, []string{"cat", "ctl"}) || len(got.Contradictions) != 1 {
|
||||
t.Fatalf("the cycle is %v, ordered %v, contradictions %+v", got.Cycle, got.Order, got.Contradictions)
|
||||
}
|
||||
if c := got.Contradictions[0]; !reflect.DeepEqual(c.Members, []string{"cat", "ctl"}) ||
|
||||
!strings.Contains(c.Said, "remove one of the `after:` lines") {
|
||||
t.Fatalf("the contradiction is said %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0249: when two rules order one pair both ways, the rule of higher precedence wins and the
|
||||
// pair says what it overruled; rules of one rank both ways are refused, naming both and how to declare.
|
||||
func TestAnOrderRuleYieldsToAStrongerOne(t *testing.T) {
|
||||
// A cycle is the one refusal, and it names every module and edge. A description's `after:` line changes nothing.
|
||||
func TestAGroupInACycleIsNamedAndNothingElseIsRefused(t *testing.T) {
|
||||
members := []orderMember{
|
||||
{ID: "ctl", Repository: "novox/mesh-controller", Number: 7},
|
||||
{ID: "host", Repository: "novox/mesh-host", Number: 9},
|
||||
{ID: "x", Repository: "novox/x", Number: 7},
|
||||
{ID: "y", Repository: "novox/y", Number: 9},
|
||||
}
|
||||
reach := map[string]orderReach{
|
||||
"ctl": {Moved: []string{"mesh-controller", "build-agent"}},
|
||||
"host": {Moved: []string{"mesh-host"}},
|
||||
reach := map[string]orderReach{"x": {Moved: []string{"mx"}}, "y": {Moved: []string{"my"}}}
|
||||
changes := map[string]moduleChange{
|
||||
"mx": {Serves: []string{"s.a"}, Invokes: []string{"s.b"}},
|
||||
"my": {Serves: []string{"s.b"}, Invokes: []string{"s.a"}},
|
||||
}
|
||||
builtBy := []inventory.Edge{{From: "mesh-host", To: "build-agent", Kind: inventory.EdgeBuiltBy}}
|
||||
|
||||
// Built by outranks engine before controller.
|
||||
got := orderOf(members, reach, builtBy)
|
||||
if !reflect.DeepEqual(got.Order, []string{"ctl", "host"}) || len(got.Cycle) > 0 {
|
||||
t.Fatalf("ordered %v, cycle %v", got.Order, got.Cycle)
|
||||
got := orderOf(members, reach, nil, changes)
|
||||
if !reflect.DeepEqual(got.Cycle, []string{"x", "y"}) || len(got.Order) > 0 {
|
||||
t.Fatalf("a cycle was ordered: %v, cycle %v", got.Order, got.Cycle)
|
||||
}
|
||||
if want := []orderPair{{Before: "ctl", After: "host", Why: orderBuiltBy, Over: []string{orderEngineFirst}}}; !reflect.DeepEqual(got.Pairs, want) {
|
||||
t.Fatalf("pairs %+v, wanted %+v", got.Pairs, want)
|
||||
}
|
||||
|
||||
// A declared line outranks both.
|
||||
members[0].After = []string{"mesh-host"}
|
||||
got = orderOf(members, reach, builtBy)
|
||||
if !reflect.DeepEqual(got.Order, []string{"host", "ctl"}) || len(got.Cycle) > 0 ||
|
||||
!reflect.DeepEqual(got.Pairs[0].Over, []string{orderBuiltBy}) || got.Pairs[0].Why != orderDeclared {
|
||||
t.Fatalf("a declared order did not win: %v %+v", got.Order, got.Pairs)
|
||||
}
|
||||
|
||||
// Without the build agent, engine before controller stands alone.
|
||||
members[0].After = nil
|
||||
got = orderOf(members, map[string]orderReach{"ctl": {Moved: []string{"mesh-controller"}}, "host": reach["host"]}, builtBy)
|
||||
if !reflect.DeepEqual(got.Order, []string{"host", "ctl"}) || got.Pairs[0].Why != orderEngineFirst || len(got.Pairs[0].Over) > 0 {
|
||||
t.Fatalf("engine before controller alone: %v %+v", got.Order, got.Pairs)
|
||||
}
|
||||
|
||||
// Built by both ways is one rank against itself: refused, both rules named, and how to declare.
|
||||
both := append(builtBy, inventory.Edge{From: "build-agent", To: "mesh-host", Kind: inventory.EdgeStandsOn})
|
||||
got = orderOf(members, reach, both)
|
||||
if !reflect.DeepEqual(got.Cycle, []string{"ctl", "host"}) || len(got.Order) > 0 || len(got.Contradictions) != 1 {
|
||||
t.Fatalf("a contradiction was ordered: %v, cycle %v, %+v", got.Order, got.Cycle, got.Contradictions)
|
||||
}
|
||||
said := got.Contradictions[0].Said
|
||||
for _, w := range []string{"built by puts ctl first", "built by puts host first",
|
||||
"`after: novox/mesh-controller` in novox/mesh-host#9's description", "`after: novox/mesh-host` in novox/mesh-controller#7's"} {
|
||||
if !strings.Contains(said, w) {
|
||||
t.Errorf("the contradiction does not say %q: %s", w, said)
|
||||
for _, w := range []string{"mx", "my", "s.a", "s.b"} {
|
||||
if !strings.Contains(got.CycleSaid, w) {
|
||||
t.Errorf("the cycle does not say %q: %s", w, got.CycleSaid)
|
||||
}
|
||||
}
|
||||
// Without the edges the same members are ordered by repository.
|
||||
got = orderOf(members, reach, nil, nil)
|
||||
if !reflect.DeepEqual(got.Order, []string{"x", "y"}) || len(got.Cycle) > 0 || len(got.Pairs) > 0 {
|
||||
t.Fatalf("members with no edge: %v %v %+v", got.Order, got.Cycle, got.Pairs)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0239 decision 8: a walk waits for the delivery's owner only while the seat has a holder on
|
||||
@@ -178,137 +121,6 @@ func TestAWalkWaitsOnlyWhileTheDeliverySeatIsHeldAndNeverForTheCore(t *testing.T
|
||||
// novox/hq ADR 0239: with mesh-delivery on record, a merge opens its walk and asks nothing until the
|
||||
// delivery's word; the word starts it; the core's own merge never waits; a person starts a waiting walk by
|
||||
// hand when the owner is down, and the owner's stop ends one as stopped.
|
||||
func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asked := asksRecorded(t)
|
||||
withConditionsInMemory(t)
|
||||
for _, name := range []string{"app", "mesh-delivery"} {
|
||||
m := catalogue.Manifest{Module: name, Version: "1"}
|
||||
if name == "mesh-delivery" {
|
||||
m.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
|
||||
Path: "modules/" + name, Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
merge := func(commit string, paths ...string) inventory.Plan {
|
||||
t.Helper()
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit, Paths: paths,
|
||||
ModuleDirs: []string{"modules/app", "modules/mesh-delivery"}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recent, err := inv.RecentPlans(ctx, 1)
|
||||
if err != nil || len(recent) != 1 || recent[0].Commit != commit {
|
||||
t.Fatalf("no plan for %s: %v %v", commit, recent, err)
|
||||
}
|
||||
return recent[0]
|
||||
}
|
||||
|
||||
// finish ends a walk as done, so the next merge has nothing of it to take over.
|
||||
finish := func(id string) {
|
||||
t.Helper()
|
||||
w, err := inv.PlanByID(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.State = inventory.PlanDone
|
||||
if err := inv.SavePlan(ctx, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// No holder on record: the merge starts its walk, as before.
|
||||
p := merge("c1aaaaaaaa", "modules/app/index.ts")
|
||||
if p.Waiting() || len(*asked) != 1 {
|
||||
t.Fatalf("with no holder on record the walk waited (%v) or asked %v", p.Waiting(), *asked)
|
||||
}
|
||||
|
||||
// The holder on record: the next merge waits, asking nothing, and an advance asks nothing either. One walk
|
||||
// is open at a time (novox/hq ADR 0276): the first ends before the next merge's batch is cut.
|
||||
finish(p.ID)
|
||||
if _, err := inv.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p = merge("c2bbbbbbbb", "modules/app/index.ts")
|
||||
if !p.Waiting() || len(*asked) != 1 || !strings.Contains(p.Note, "plans go "+p.ID) {
|
||||
t.Fatalf("the walk did not wait for its word: waiting %v, asked %v, note %q", p.Waiting(), *asked, p.Note)
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) != 1 {
|
||||
t.Fatalf("a waiting walk was advanced into asking: %v", *asked)
|
||||
}
|
||||
if line := planLine(p, p.Created); strings.Contains(line, "LATE") || !strings.Contains(line, "waits for") {
|
||||
t.Errorf("a waiting walk reads %q", line)
|
||||
}
|
||||
|
||||
// The delivery's word starts it.
|
||||
if err := deliveryCommand(ctx, []string{"go", p.ID, "--by", catalogue.DeliverySeat, "--why", "its turn"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := deliveryCommand(ctx, []string{"go", p.ID}); err == nil {
|
||||
t.Fatal("a walk was let go twice")
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) != 2 {
|
||||
t.Fatalf("the word did not start the walk: %v", *asked)
|
||||
}
|
||||
got, err := inv.PlanByID(ctx, p.ID)
|
||||
if err != nil || got.Delivery == nil || got.Delivery.By != catalogue.DeliverySeat || got.Delivery.Why != "its turn" {
|
||||
t.Fatalf("the word was not kept: %+v %v", got.Delivery, err)
|
||||
}
|
||||
|
||||
// The delivery's owner's own merge never waits for it. A walk that started is never taken over (novox/hq
|
||||
// ADR 0276): the merge's batch is cut once it ended.
|
||||
finish(p.ID)
|
||||
p = merge("c3cccccccc", "modules/mesh-delivery/main.go")
|
||||
if p.Waiting() || len(*asked) != 3 {
|
||||
t.Fatalf("mesh-delivery's own walk waited for mesh-delivery: %v %v", p.Waiting(), *asked)
|
||||
}
|
||||
|
||||
// The owner down: a person starts a waiting walk, with why.
|
||||
finish(p.ID)
|
||||
p = merge("c4dddddddd", "modules/app/index.ts")
|
||||
if !p.Waiting() {
|
||||
t.Fatal("the walk did not wait")
|
||||
}
|
||||
if err := plansCommand(ctx, []string{"go", p.ID}); err == nil || !strings.Contains(err.Error(), "--why") {
|
||||
t.Fatalf("a walk was started by hand without why: %v", err)
|
||||
}
|
||||
if err := plansCommand(ctx, []string{"go", p.ID, "--why", "mesh-delivery is down"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
got, _ = inv.PlanByID(ctx, p.ID)
|
||||
if got.Waiting() || !strings.HasPrefix(got.Delivery.By, "a person") || len(*asked) < 4 {
|
||||
t.Fatalf("a person's word did not start the walk: %+v, asked %v", got.Delivery, *asked)
|
||||
}
|
||||
|
||||
// The owner's stop: failed, said as stopped by it.
|
||||
finish(p.ID)
|
||||
p = merge("c5eeeeeeee", "modules/app/index.ts")
|
||||
if err := deliveryCommand(ctx, []string{"stop", p.ID, "--why", "the operator stopped it", "--by",
|
||||
"mesh-delivery for jochen"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ = inv.PlanByID(ctx, p.ID)
|
||||
if got.State != inventory.PlanFailed || got.Delivery.Stopped != "mesh-delivery for jochen" ||
|
||||
!strings.Contains(got.Note, "the operator stopped it") {
|
||||
t.Fatalf("the stop was kept as %s %+v %q", got.State, got.Delivery, got.Note)
|
||||
}
|
||||
|
||||
// Unassigned: the mesh is back on the controller's own path.
|
||||
if err := inv.Unassign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asks := len(*asked)
|
||||
if p = merge("c6ffffffff", "modules/app/index.ts"); p.Waiting() || len(*asked) != asks+1 {
|
||||
t.Fatalf("with the holder gone the walk waited: %v", p.Waiting())
|
||||
}
|
||||
}
|
||||
|
||||
// The verbs mesh-delivery asks with become the commands they name, and nothing a caller sends is passed over.
|
||||
func TestTheDeliveryVerbsComposeTheirCommands(t *testing.T) {
|
||||
@@ -345,49 +157,3 @@ func TestTheDeliveryVerbsComposeTheirCommands(t *testing.T) {
|
||||
|
||||
// A verb runs as a command of its own: what it keeps of a walk is still said as plan-moved, on a bus of the
|
||||
// command's own, so the delivery's owner hears it at once and not only when it reads the walks back.
|
||||
func TestAWalkLetGoByAVerbIsSaidAsPlanMoved(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
url := testbus.URL(t)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := handActConn
|
||||
handActConn = js.Conn()
|
||||
t.Cleanup(func() { handActConn = before })
|
||||
heard := make(chan *nats.Msg, 4)
|
||||
sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), heard)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
|
||||
waiting := inventory.Plan{ID: "plan-waits", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c7c7c7c7",
|
||||
Created: time.Now().UTC(), State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
|
||||
Modules: map[string]*inventory.PlanModule{"app": {}},
|
||||
Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}}
|
||||
if err := open.inventory.SavePlan(ctx, &waiting); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case m := <-heard:
|
||||
var said inventory.Plan
|
||||
if err := json.Unmarshal(m.Data, &said); err != nil || said.ID != waiting.ID || said.Delivery == nil ||
|
||||
said.Delivery.Go == nil {
|
||||
t.Fatalf("plan-moved said %s (%v)", m.Data, err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("a walk let go by a verb was not said")
|
||||
}
|
||||
if checkEvents != nil || inventory.PlanSaved != nil {
|
||||
t.Fatal("the command's own bus was left in place")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func TestASecretValueIsNeverAcceptedThroughAVerb(t *testing.T) {
|
||||
t.Setenv(verbVar, "mesh-controller.command")
|
||||
for _, args := range [][]string{
|
||||
{"accept", "anchor", "telegram", "telegram-token", "--from", "/dev/null"},
|
||||
{"accept", "anchor", "app", "db", "--from", "/dev/null", "--provider", "store"},
|
||||
} {
|
||||
err := secretCommand(context.Background(), args)
|
||||
if err == nil || !strings.Contains(err.Error(), "never through a verb") {
|
||||
t.Errorf("%v: %v", args, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestATrustedPartysSecretGivenAtTheTerminalIsAnnouncedBeforeItIsKept(t *testing.T) {
|
||||
var order []string
|
||||
announce := func(fail bool) func() error {
|
||||
return func() error {
|
||||
order = append(order, "announce")
|
||||
if fail {
|
||||
return errors.New("no channel")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
keep := func() (bool, error) { order = append(order, "keep"); return false, nil }
|
||||
|
||||
order = nil
|
||||
if _, unannounced, err := keepGiven(true, announce(false), keep); err != nil || unannounced != nil ||
|
||||
strings.Join(order, ",") != "announce,keep" {
|
||||
t.Errorf("trusted: %v %v, order %v; want announced, then kept", unannounced, err, order)
|
||||
}
|
||||
order = nil
|
||||
if _, _, err := keepGiven(true, announce(true), keep); err == nil || strings.Join(order, ",") != "announce" {
|
||||
t.Errorf("trusted, announcement failed: %v, order %v; want refused and nothing kept", err, order)
|
||||
}
|
||||
order = nil
|
||||
if _, unannounced, err := keepGiven(false, announce(true), keep); err != nil || unannounced == nil ||
|
||||
strings.Join(order, ",") != "keep,announce" {
|
||||
t.Errorf("not trusted: %v %v, order %v; want kept, then the failed announcement said", unannounced, err, order)
|
||||
}
|
||||
order = nil
|
||||
failing := func() (bool, error) { order = append(order, "keep"); return false, errors.New("store away") }
|
||||
if _, _, err := keepGiven(false, announce(false), failing); err == nil || strings.Join(order, ",") != "keep" {
|
||||
t.Errorf("not trusted, keep failed: %v, order %v; want refused and nothing announced", err, order)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASecretAskIsNeverASecretRead(t *testing.T) {
|
||||
t.Setenv(verbVar, "mesh-controller.secret-ask")
|
||||
for _, args := range [][]string{
|
||||
{"ask", "anchor", "telegram", "telegram-token", "the-value"},
|
||||
{"ask", "anchor", "telegram"},
|
||||
{"ask", "anchor", "telegram", "telegram-token", "--from", "/dev/null"},
|
||||
} {
|
||||
if err := secretCommand(context.Background(), args); err == nil {
|
||||
t.Errorf("secret %v was taken", args)
|
||||
}
|
||||
}
|
||||
for _, args := range [][]string{{"recover", "anchor", "telegram", "telegram-token"}, {"export"}} {
|
||||
if err := terminalOnly(append([]string{"secret"}, args...)); err == nil {
|
||||
t.Errorf("secret %v passed the terminal rule", args)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestATrustedPartysSecretAtTheTerminalIsNotKeptWhenTellAndTheConditionBothFail(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
askOperatorChannelFake(t, func(map[string]any) (json.RawMessage, error) { return nil, link.ErrNothingServes })
|
||||
// conditionsFrom is left nil and there is no bus to dial here, so the fallback condition cannot be
|
||||
// raised either: both of ADR 0380's ways to tell the operator fail.
|
||||
kept := false
|
||||
if _, _, err := keepGiven(true,
|
||||
func() error {
|
||||
return announceSecretGiven(ctx, "anchor", "telegram", "telegram-token", "at the controller's terminal", "me")
|
||||
},
|
||||
func() (bool, error) { kept = true; return false, nil }); err == nil {
|
||||
t.Error("a trusted party's secret was kept though telling the operator and raising the condition both failed")
|
||||
}
|
||||
if kept {
|
||||
t.Error("the value was kept despite the failure")
|
||||
}
|
||||
}
|
||||
@@ -263,18 +263,6 @@ func TestOnlyTheControllerMayAskTheDesksPrompt(t *testing.T) {
|
||||
|
||||
// A value for a secret comes from the terminal or the desk, never through a verb (the review of 2026-10-09,
|
||||
// M4): `secret accept` with a value, run for a verb, is refused before anything is read.
|
||||
func TestASecretValueIsNeverAcceptedThroughAVerb(t *testing.T) {
|
||||
t.Setenv(verbVar, "mesh-controller.command")
|
||||
for _, args := range [][]string{
|
||||
{"accept", "anchor", "telegram", "telegram-token", "--from", "/dev/null"},
|
||||
{"accept", "anchor", "app", "db", "--from", "/dev/null", "--provider", "store"},
|
||||
} {
|
||||
err := secretCommand(context.Background(), args)
|
||||
if err == nil || !strings.Contains(err.Error(), "never through a verb") {
|
||||
t.Errorf("%v: %v", args, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The `give` and `secret-ask` verbs' own line passes the terminal-only rule of ADR 0266, and no `secret accept`
|
||||
// does: a value, a file, a provider or an extra word is still the terminal's alone (novox/hq ADR 0277).
|
||||
@@ -355,39 +343,6 @@ func TestOnlyTheDeskMachinesLauncherMayAnswerItsPrompt(t *testing.T) {
|
||||
// N1-give at the controller's terminal (the confirmation review of 2026-10-09): a trusted party's secret is
|
||||
// announced before it is kept, and not kept when the announcement fails; another module's is kept first and
|
||||
// a failed announcement is said, not undone.
|
||||
func TestATrustedPartysSecretGivenAtTheTerminalIsAnnouncedBeforeItIsKept(t *testing.T) {
|
||||
var order []string
|
||||
announce := func(fail bool) func() error {
|
||||
return func() error {
|
||||
order = append(order, "announce")
|
||||
if fail {
|
||||
return errors.New("no channel")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
keep := func() (bool, error) { order = append(order, "keep"); return false, nil }
|
||||
|
||||
order = nil
|
||||
if _, unannounced, err := keepGiven(true, announce(false), keep); err != nil || unannounced != nil ||
|
||||
strings.Join(order, ",") != "announce,keep" {
|
||||
t.Errorf("trusted: %v %v, order %v; want announced, then kept", unannounced, err, order)
|
||||
}
|
||||
order = nil
|
||||
if _, _, err := keepGiven(true, announce(true), keep); err == nil || strings.Join(order, ",") != "announce" {
|
||||
t.Errorf("trusted, announcement failed: %v, order %v; want refused and nothing kept", err, order)
|
||||
}
|
||||
order = nil
|
||||
if _, unannounced, err := keepGiven(false, announce(true), keep); err != nil || unannounced == nil ||
|
||||
strings.Join(order, ",") != "keep,announce" {
|
||||
t.Errorf("not trusted: %v %v, order %v; want kept, then the failed announcement said", unannounced, err, order)
|
||||
}
|
||||
order = nil
|
||||
failing := func() (bool, error) { order = append(order, "keep"); return false, errors.New("store away") }
|
||||
if _, _, err := keepGiven(false, announce(false), failing); err == nil || strings.Join(order, ",") != "keep" {
|
||||
t.Errorf("not trusted, keep failed: %v, order %v; want refused and nothing announced", err, order)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine the mesh does not know, as the secret's or as the desk, is refused before anybody is asked to type.
|
||||
func TestAGiveNamingAMachineTheMeshDoesNotKnowAsksNobody(t *testing.T) {
|
||||
@@ -539,23 +494,6 @@ func TestASecretAskIsBoundedAndItsEndRecorded(t *testing.T) {
|
||||
|
||||
// A secret ask cannot be turned into a secret read: the line carries no value, the answer carries none, and every
|
||||
// other `secret` line is the terminal's.
|
||||
func TestASecretAskIsNeverASecretRead(t *testing.T) {
|
||||
t.Setenv(verbVar, "mesh-controller.secret-ask")
|
||||
for _, args := range [][]string{
|
||||
{"ask", "anchor", "telegram", "telegram-token", "the-value"},
|
||||
{"ask", "anchor", "telegram"},
|
||||
{"ask", "anchor", "telegram", "telegram-token", "--from", "/dev/null"},
|
||||
} {
|
||||
if err := secretCommand(context.Background(), args); err == nil {
|
||||
t.Errorf("secret %v was taken", args)
|
||||
}
|
||||
}
|
||||
for _, args := range [][]string{{"recover", "anchor", "telegram", "telegram-token"}, {"export"}} {
|
||||
if err := terminalOnly(append([]string{"secret"}, args...)); err == nil {
|
||||
t.Errorf("secret %v passed the terminal rule", args)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// askOperatorChannelFake replaces askOperatorChannel for the test and records every call it was given.
|
||||
func askOperatorChannelFake(t *testing.T, answer func(args map[string]any) (json.RawMessage, error)) *[]map[string]any {
|
||||
@@ -664,23 +602,6 @@ func TestASuccessfulTellRetiresEveryOpenSecretGivenCondition(t *testing.T) {
|
||||
// novox/hq ADR 0380 clause 7: a trusted party's secret given at the controller's terminal is announced
|
||||
// before it is kept, and not kept when the announcement fails. It still fails when telling the operator
|
||||
// AND raising the fallback condition both fail — nothing silently keeps a trusted party's value unheard.
|
||||
func TestATrustedPartysSecretAtTheTerminalIsNotKeptWhenTellAndTheConditionBothFail(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
askOperatorChannelFake(t, func(map[string]any) (json.RawMessage, error) { return nil, link.ErrNothingServes })
|
||||
// conditionsFrom is left nil and there is no bus to dial here, so the fallback condition cannot be
|
||||
// raised either: both of ADR 0380's ways to tell the operator fail.
|
||||
kept := false
|
||||
if _, _, err := keepGiven(true,
|
||||
func() error {
|
||||
return announceSecretGiven(ctx, "anchor", "telegram", "telegram-token", "at the controller's terminal", "me")
|
||||
},
|
||||
func() (bool, error) { kept = true; return false, nil }); err == nil {
|
||||
t.Error("a trusted party's secret was kept though telling the operator and raising the condition both failed")
|
||||
}
|
||||
if kept {
|
||||
t.Error("the value was kept despite the failure")
|
||||
}
|
||||
}
|
||||
|
||||
// Review round 2 of mesh-controller (feat/mail-seats, head 5f24a47d): ADR 0380 §9's fallback — raising the
|
||||
// secret-given condition — answers only link.ErrNothingServes and link.ErrGrantRefused. Any other failure of
|
||||
|
||||
@@ -174,6 +174,10 @@ var probeRegistry = []probe{
|
||||
{ID: busStepProbe, Asserts: "a bus upgrade a person started is said while it runs, and is followed by the bus's " +
|
||||
"health within its bound — or is said failed, with its snapshot as the way back", From: "ADR 0236, to-be 45 §8",
|
||||
Kind: kindBusMaintenance, Raises: []string{kindBusUpgradeFailed}, Phase: 4, run: probeBusStep},
|
||||
// A delivery plan held for a backup that failed or stayed silent (ADR 0404 decision 4): one condition per plan.
|
||||
{ID: backupHeldProbe, Asserts: "no delivery plan is held for a backup before its send that failed or did not " +
|
||||
"answer in ten minutes", From: "ADR 0404 decision 4, issue 1066", Kind: kindBackupHeld, Phase: 1,
|
||||
run: probeBackupHeld},
|
||||
}
|
||||
|
||||
// probeVerdict is one probe's outcome in a run.
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
func TestARunKeepsWhatEachProbeFoundAndSaysItsHeartbeat(t *testing.T) {
|
||||
failing := conditions.Observation{Scope: conditions.ScopeMachine, ID: "anchor", Token: "refused",
|
||||
Severity: conditions.Urgent, Summary: "anchor's node-engine would refuse its declaration"}
|
||||
var broken atomic.Bool
|
||||
broken.Store(true)
|
||||
withProbes(t,
|
||||
probe{ID: "P1", Asserts: "passes", Kind: "never", Phase: 1,
|
||||
run: func(context.Context, *doctor) ([]conditions.Observation, error) { return nil, nil }},
|
||||
probe{ID: "P2", Asserts: "finds a fault", Kind: "declaration-refused", Phase: 1,
|
||||
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
|
||||
if broken.Load() {
|
||||
return []conditions.Observation{failing}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}},
|
||||
probe{ID: "P3", Asserts: "cannot run", Kind: "x", Phase: 1,
|
||||
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
|
||||
if broken.Load() {
|
||||
return nil, errors.New("the store is away")
|
||||
}
|
||||
return nil, nil
|
||||
}},
|
||||
probe{ID: "P4", Asserts: "hangs", Kind: "x", Phase: 1,
|
||||
run: func(ctx context.Context, _ *doctor) ([]conditions.Observation, error) {
|
||||
if broken.Load() {
|
||||
<-ctx.Done()
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
return nil, nil
|
||||
}},
|
||||
probe{ID: "P5", Asserts: "later", Kind: "x", Phase: 2, Deferred: "not yet"},
|
||||
)
|
||||
before := probeWithin
|
||||
probeWithin = 200 * time.Millisecond
|
||||
t.Cleanup(func() { probeWithin = before })
|
||||
|
||||
store := conditions.NewInMemory()
|
||||
told := &conditions.Told{}
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||
defer k.Close(context.Background())
|
||||
d := &doctor{keeper: k, teller: told, host: "anchor"}
|
||||
// A probe that cannot run is said in the verdict at once, and as a condition when the next run cannot
|
||||
// run it either (novox/hq issue 277): one timed-out question is not a probe gone blind.
|
||||
first := d.runOnce(t.Context(), "a test")
|
||||
if first.Counts != (doctorCounts{Passed: 1, Failed: 1, FailedToRun: 2, Deferred: 1}) {
|
||||
t.Fatalf("counted %+v", first.Counts)
|
||||
}
|
||||
if open, _ := k.Open(t.Context()); len(open) != 1 || open[0].Key != "machine.anchor.refused" {
|
||||
t.Fatalf("after one run, open: %+v", open)
|
||||
}
|
||||
run := d.runOnce(t.Context(), "a test")
|
||||
if run.Counts != (doctorCounts{Passed: 1, Failed: 1, FailedToRun: 2, Deferred: 1}) {
|
||||
t.Fatalf("counted %+v", run.Counts)
|
||||
}
|
||||
open, err := k.Open(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keys []string
|
||||
for _, c := range open {
|
||||
keys = append(keys, c.Key+"="+c.Kind)
|
||||
}
|
||||
for _, want := range []string{"machine.anchor.refused=declaration-refused", "probe.P3.failed=probe-failed",
|
||||
"probe.P4.failed=probe-failed"} {
|
||||
if !slices.Contains(keys, want) {
|
||||
t.Errorf("%s is not open: %v", want, keys)
|
||||
}
|
||||
}
|
||||
// The heartbeat, in the shape mesh-watcher reads (the contract with the operator's channel).
|
||||
if len(told.Names) != 2 || told.Names[1] != conditions.HeartbeatEvent {
|
||||
t.Fatalf("said %v", told.Names)
|
||||
}
|
||||
if d.lastRunEnded().IsZero() || d.lastRun().Run != run.Run {
|
||||
t.Fatal("the run is not the last verdict")
|
||||
}
|
||||
body, _ := json.Marshal(run)
|
||||
var shape map[string]any
|
||||
_ = json.Unmarshal(body, &shape)
|
||||
for _, field := range []string{"run", "at", "interval-seconds", "counts", "probes", "controller"} {
|
||||
if _, ok := shape[field]; !ok {
|
||||
t.Errorf("the heartbeat carries no %q: %s", field, body)
|
||||
}
|
||||
}
|
||||
|
||||
// Mended: the next run clears every one of them.
|
||||
broken.Store(false)
|
||||
d.runOnce(t.Context(), "a test")
|
||||
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||
t.Fatalf("a passing run left open %+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEveryMachineOfAHealthyMeshComposesAndValidates(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
got, err := probeDeclarations(t.Context(), &doctor{open: open})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 0 {
|
||||
t.Fatalf("a healthy mesh failed D1: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMachineWhoseDeclarationDoesNotComposeIsSaid(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
one, two := rivals()
|
||||
register(t, open, one)
|
||||
register(t, open, two)
|
||||
for _, m := range []string{"rival-one", "rival-two"} {
|
||||
if _, err := assign(ctx, open, "laptop", m); err != nil && m == "rival-one" {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, err := probeDeclarations(ctx, &doctor{open: open})
|
||||
linted(got)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "the-seat") {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNatsTheBusIsWhatTheControllerDefines(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
open := aMesh(t)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
|
||||
_ = js.Context().DeleteStream(s)
|
||||
}
|
||||
if _, err := assertBusObjects(t.Context(), open.inventory, js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &doctor{open: open, js: js}
|
||||
for _, p := range []func(context.Context, *doctor) ([]conditions.Observation, error){probeConsumers, probeStreams} {
|
||||
got, err := p(t.Context(), d)
|
||||
if err != nil || len(got) != 0 {
|
||||
t.Fatalf("a bus just raised fails: %+v %v", got, err)
|
||||
}
|
||||
}
|
||||
if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := js.Context().StreamInfo("EVENTS")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := info.Config
|
||||
cfg.MaxMsgsPerSubject = 3
|
||||
if _, err := js.Context().UpdateStream(&cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
consumers, err := probeConsumers(t.Context(), d)
|
||||
if err != nil || len(consumers) != 1 || consumers[0].Key() != "bus.NODES.laptop.missing" {
|
||||
t.Fatalf("the deleted consumer: %+v %v", consumers, err)
|
||||
}
|
||||
streams, err := probeStreams(t.Context(), d)
|
||||
if err != nil || len(streams) != 1 || !strings.Contains(streams[0].Summary, "per subject") {
|
||||
t.Fatalf("the redefined stream: %+v %v", streams, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNatsTheBusSaysAConsumerGaveUpAndOneWasDeleted(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
heard := make(chan *nats.Msg, 16)
|
||||
for _, subject := range broker.BusAdvisories {
|
||||
if _, err := conn.ChanSubscribe(subject, heard); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
api, _ := jetstream.New(conn)
|
||||
_ = api.DeleteStream(t.Context(), "SEAT_ADVISED")
|
||||
stream, err := api.CreateStream(t.Context(), jetstream.StreamConfig{Name: "SEAT_ADVISED", Subjects: []string{"advised.>"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = api.DeleteStream(context.Background(), "SEAT_ADVISED") }()
|
||||
consumer, err := stream.CreateConsumer(t.Context(), jetstream.ConsumerConfig{Durable: "SEAT_ADVISED_worker",
|
||||
AckPolicy: jetstream.AckExplicitPolicy, MaxDeliver: 1, AckWait: 100 * time.Millisecond})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := api.Publish(t.Context(), "advised.x", []byte("x")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := consumer.Fetch(1, jetstream.FetchMaxWait(time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A seat's worker, so the deletion is of a consumer the mesh names (link.MeshNamed).
|
||||
// Not acknowledged: after its one delivery the consumer gives up on it — on the next fetch.
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
_, _ = consumer.Fetch(1, jetstream.FetchMaxWait(300*time.Millisecond))
|
||||
if err := stream.DeleteConsumer(t.Context(), "SEAT_ADVISED_worker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kinds := map[string]string{}
|
||||
deadline := time.After(5 * time.Second)
|
||||
for len(kinds) < 2 {
|
||||
select {
|
||||
case m := <-heard:
|
||||
if a, ok := link.ReadAdvisory(m.Subject, m.Data); ok {
|
||||
kinds[a.Kind] = a.Said
|
||||
}
|
||||
case <-deadline:
|
||||
t.Fatalf("the bus said only %v", kinds)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(kinds["max-deliveries"], "gave up") || !strings.Contains(kinds["consumer-lost"], "was deleted") {
|
||||
t.Fatalf("%v", kinds)
|
||||
}
|
||||
}
|
||||
@@ -2,25 +2,18 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"golang.org/x/net/dns/dnsmessage"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// The self-check (novox/hq to-be 45 §4): a probe that fails raises its condition, one that cannot run
|
||||
@@ -34,98 +27,6 @@ func withProbes(t *testing.T, probes ...probe) {
|
||||
t.Cleanup(func() { probeRegistry = before })
|
||||
}
|
||||
|
||||
func TestARunKeepsWhatEachProbeFoundAndSaysItsHeartbeat(t *testing.T) {
|
||||
failing := conditions.Observation{Scope: conditions.ScopeMachine, ID: "anchor", Token: "refused",
|
||||
Severity: conditions.Urgent, Summary: "anchor's node-engine would refuse its declaration"}
|
||||
var broken atomic.Bool
|
||||
broken.Store(true)
|
||||
withProbes(t,
|
||||
probe{ID: "P1", Asserts: "passes", Kind: "never", Phase: 1,
|
||||
run: func(context.Context, *doctor) ([]conditions.Observation, error) { return nil, nil }},
|
||||
probe{ID: "P2", Asserts: "finds a fault", Kind: "declaration-refused", Phase: 1,
|
||||
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
|
||||
if broken.Load() {
|
||||
return []conditions.Observation{failing}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}},
|
||||
probe{ID: "P3", Asserts: "cannot run", Kind: "x", Phase: 1,
|
||||
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
|
||||
if broken.Load() {
|
||||
return nil, errors.New("the store is away")
|
||||
}
|
||||
return nil, nil
|
||||
}},
|
||||
probe{ID: "P4", Asserts: "hangs", Kind: "x", Phase: 1,
|
||||
run: func(ctx context.Context, _ *doctor) ([]conditions.Observation, error) {
|
||||
if broken.Load() {
|
||||
<-ctx.Done()
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
return nil, nil
|
||||
}},
|
||||
probe{ID: "P5", Asserts: "later", Kind: "x", Phase: 2, Deferred: "not yet"},
|
||||
)
|
||||
before := probeWithin
|
||||
probeWithin = 200 * time.Millisecond
|
||||
t.Cleanup(func() { probeWithin = before })
|
||||
|
||||
store := conditions.NewInMemory()
|
||||
told := &conditions.Told{}
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||
defer k.Close(context.Background())
|
||||
d := &doctor{keeper: k, teller: told, host: "anchor"}
|
||||
// A probe that cannot run is said in the verdict at once, and as a condition when the next run cannot
|
||||
// run it either (novox/hq issue 277): one timed-out question is not a probe gone blind.
|
||||
first := d.runOnce(t.Context(), "a test")
|
||||
if first.Counts != (doctorCounts{Passed: 1, Failed: 1, FailedToRun: 2, Deferred: 1}) {
|
||||
t.Fatalf("counted %+v", first.Counts)
|
||||
}
|
||||
if open, _ := k.Open(t.Context()); len(open) != 1 || open[0].Key != "machine.anchor.refused" {
|
||||
t.Fatalf("after one run, open: %+v", open)
|
||||
}
|
||||
run := d.runOnce(t.Context(), "a test")
|
||||
if run.Counts != (doctorCounts{Passed: 1, Failed: 1, FailedToRun: 2, Deferred: 1}) {
|
||||
t.Fatalf("counted %+v", run.Counts)
|
||||
}
|
||||
open, err := k.Open(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keys []string
|
||||
for _, c := range open {
|
||||
keys = append(keys, c.Key+"="+c.Kind)
|
||||
}
|
||||
for _, want := range []string{"machine.anchor.refused=declaration-refused", "probe.P3.failed=probe-failed",
|
||||
"probe.P4.failed=probe-failed"} {
|
||||
if !slices.Contains(keys, want) {
|
||||
t.Errorf("%s is not open: %v", want, keys)
|
||||
}
|
||||
}
|
||||
// The heartbeat, in the shape mesh-watcher reads (the contract with the operator's channel).
|
||||
if len(told.Names) != 2 || told.Names[1] != conditions.HeartbeatEvent {
|
||||
t.Fatalf("said %v", told.Names)
|
||||
}
|
||||
if d.lastRunEnded().IsZero() || d.lastRun().Run != run.Run {
|
||||
t.Fatal("the run is not the last verdict")
|
||||
}
|
||||
body, _ := json.Marshal(run)
|
||||
var shape map[string]any
|
||||
_ = json.Unmarshal(body, &shape)
|
||||
for _, field := range []string{"run", "at", "interval-seconds", "counts", "probes", "controller"} {
|
||||
if _, ok := shape[field]; !ok {
|
||||
t.Errorf("the heartbeat carries no %q: %s", field, body)
|
||||
}
|
||||
}
|
||||
|
||||
// Mended: the next run clears every one of them.
|
||||
broken.Store(false)
|
||||
d.runOnce(t.Context(), "a test")
|
||||
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||
t.Fatalf("a passing run left open %+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
// **The registry says what each probe asserts**, and a probe not built says why and when.
|
||||
func TestTheRegistryIsTheDesignsLiveForm(t *testing.T) {
|
||||
seen := map[string]bool{}
|
||||
@@ -164,38 +65,8 @@ func TestWatchdogsThatStoppedAreSaid(t *testing.T) {
|
||||
}
|
||||
|
||||
// **D1 composes every machine of a healthy mesh and the host's own validator takes each.**
|
||||
func TestEveryMachineOfAHealthyMeshComposesAndValidates(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
got, err := probeDeclarations(t.Context(), &doctor{open: open})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 0 {
|
||||
t.Fatalf("a healthy mesh failed D1: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **D1 names a machine nothing can be sent to**, and the network that cannot be computed for it.
|
||||
func TestAMachineWhoseDeclarationDoesNotComposeIsSaid(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
one, two := rivals()
|
||||
register(t, open, one)
|
||||
register(t, open, two)
|
||||
for _, m := range []string{"rival-one", "rival-two"} {
|
||||
if _, err := assign(ctx, open, "laptop", m); err != nil && m == "rival-one" {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, err := probeDeclarations(ctx, &doctor{open: open})
|
||||
linted(got)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "the-seat") {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **D2: a resolver answering NXDOMAIN for IPv6 is wrong** — musl takes it as no such name (issue 262).
|
||||
func TestAResolverAnsweringNoSuchNameForIPv6IsWrong(t *testing.T) {
|
||||
@@ -250,107 +121,8 @@ func TestAResolverAnsweringNoSuchNameForIPv6IsWrong(t *testing.T) {
|
||||
|
||||
// **D6, D7: what the controller defines is what it finds**, and a consumer deleted or a stream
|
||||
// redefined is said — against a real bus, raised by the same derivation the controller starts with.
|
||||
func TestNatsTheBusIsWhatTheControllerDefines(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
open := aMesh(t)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
|
||||
_ = js.Context().DeleteStream(s)
|
||||
}
|
||||
if _, err := assertBusObjects(t.Context(), open.inventory, js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &doctor{open: open, js: js}
|
||||
for _, p := range []func(context.Context, *doctor) ([]conditions.Observation, error){probeConsumers, probeStreams} {
|
||||
got, err := p(t.Context(), d)
|
||||
if err != nil || len(got) != 0 {
|
||||
t.Fatalf("a bus just raised fails: %+v %v", got, err)
|
||||
}
|
||||
}
|
||||
if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := js.Context().StreamInfo("EVENTS")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := info.Config
|
||||
cfg.MaxMsgsPerSubject = 3
|
||||
if _, err := js.Context().UpdateStream(&cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
consumers, err := probeConsumers(t.Context(), d)
|
||||
if err != nil || len(consumers) != 1 || consumers[0].Key() != "bus.NODES.laptop.missing" {
|
||||
t.Fatalf("the deleted consumer: %+v %v", consumers, err)
|
||||
}
|
||||
streams, err := probeStreams(t.Context(), d)
|
||||
if err != nil || len(streams) != 1 || !strings.Contains(streams[0].Summary, "per subject") {
|
||||
t.Fatalf("the redefined stream: %+v %v", streams, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **S9 hears the bus**: a consumer that gives up on a message, and one deleted, as the server says.
|
||||
func TestNatsTheBusSaysAConsumerGaveUpAndOneWasDeleted(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
heard := make(chan *nats.Msg, 16)
|
||||
for _, subject := range broker.BusAdvisories {
|
||||
if _, err := conn.ChanSubscribe(subject, heard); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
api, _ := jetstream.New(conn)
|
||||
_ = api.DeleteStream(t.Context(), "SEAT_ADVISED")
|
||||
stream, err := api.CreateStream(t.Context(), jetstream.StreamConfig{Name: "SEAT_ADVISED", Subjects: []string{"advised.>"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = api.DeleteStream(context.Background(), "SEAT_ADVISED") }()
|
||||
consumer, err := stream.CreateConsumer(t.Context(), jetstream.ConsumerConfig{Durable: "SEAT_ADVISED_worker",
|
||||
AckPolicy: jetstream.AckExplicitPolicy, MaxDeliver: 1, AckWait: 100 * time.Millisecond})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := api.Publish(t.Context(), "advised.x", []byte("x")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := consumer.Fetch(1, jetstream.FetchMaxWait(time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A seat's worker, so the deletion is of a consumer the mesh names (link.MeshNamed).
|
||||
// Not acknowledged: after its one delivery the consumer gives up on it — on the next fetch.
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
_, _ = consumer.Fetch(1, jetstream.FetchMaxWait(300*time.Millisecond))
|
||||
if err := stream.DeleteConsumer(t.Context(), "SEAT_ADVISED_worker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kinds := map[string]string{}
|
||||
deadline := time.After(5 * time.Second)
|
||||
for len(kinds) < 2 {
|
||||
select {
|
||||
case m := <-heard:
|
||||
if a, ok := link.ReadAdvisory(m.Subject, m.Data); ok {
|
||||
kinds[a.Kind] = a.Said
|
||||
}
|
||||
case <-deadline:
|
||||
t.Fatalf("the bus said only %v", kinds)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(kinds["max-deliveries"], "gave up") || !strings.Contains(kinds["consumer-lost"], "was deleted") {
|
||||
t.Fatalf("%v", kinds)
|
||||
}
|
||||
}
|
||||
|
||||
// **D10 compares a node-engine with what it is delivered as, not with its commit** (2026-10-06: every
|
||||
// machine read as behind right after a push sent it the current build — it says the digest-named
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func TestADryRunsOutcomeIsTakenInByNothing(t *testing.T) {
|
||||
err := builds{}.Built(t.Context(), link.BuildResult{
|
||||
ID: "build-1", Repository: "ssh://forge/app.git", Ref: "unreviewed", Module: "app", DryRun: true,
|
||||
Manifest: json.RawMessage(`{"module":"app","version":"1"}`),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("a dry run's outcome was not simply set aside: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -9,15 +9,6 @@ import (
|
||||
|
||||
// A dry run's outcome is looked at, never taken in (novox/hq issue 240). The daemon here holds no
|
||||
// store at all, so anything that tried to record or register would fail rather than pass quietly.
|
||||
func TestADryRunsOutcomeIsTakenInByNothing(t *testing.T) {
|
||||
err := builds{}.Built(t.Context(), link.BuildResult{
|
||||
ID: "build-1", Repository: "ssh://forge/app.git", Ref: "unreviewed", Module: "app", DryRun: true,
|
||||
Manifest: json.RawMessage(`{"module":"app","version":"1"}`),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("a dry run's outcome was not simply set aside: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The mark survives the wire both ways: asked as a dry run, answered as one.
|
||||
func TestTheDryRunMarkTravelsWithTheBuild(t *testing.T) {
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
|
||||
@@ -287,6 +287,12 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
m.Architecture, m.Kernel = reported.Architecture, reported.Kernel
|
||||
// Whether its node-engine takes handed files is its contract (issue 969).
|
||||
if stated, had, err := inv.HealthOf(ctx, n.Name); err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
} else if had {
|
||||
m.Contract = stated.Contract
|
||||
}
|
||||
outward, err := inv.OutwardLinksOf(ctx, n.Name)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
found time.Duration // when the directory was found, against now
|
||||
passes bool
|
||||
}{
|
||||
{"found a day before the send", -24 * time.Hour, true},
|
||||
{"found by this send", time.Hour, false},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
|
||||
return map[string]bool{"anchor": true, "laptop": true}, nil
|
||||
}
|
||||
backlogFacts := gatherGateFacts
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
f, err := backlogFacts(ctx, open, component)
|
||||
// The used-as-found condition, raised after the send (its second statement): its own kind, never a
|
||||
// fault the gate reads as the build's.
|
||||
f.judged, f.openErr = true, nil
|
||||
f.open = append(f.open, conditions.Condition{Key: usedAsFoundKey("app", "anchor"), Kind: kindUsedAsFound,
|
||||
Subject: conditions.Subject{Scope: conditions.ScopeModule, ID: "app.anchor", Machine: "anchor"},
|
||||
Raised: time.Now()})
|
||||
f.health = map[string]inventory.NodeHealth{}
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
f.health[n] = inventory.NodeHealth{Node: n, HeardAt: time.Now(), Resources: []inventory.ResourceHealth{
|
||||
{Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy},
|
||||
foundDirectory("app", time.Now().Add(c.found)),
|
||||
{Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateHealthy}}}
|
||||
}
|
||||
return f, err
|
||||
}
|
||||
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
|
||||
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
|
||||
wasWindow := gateWitnessWindow
|
||||
gateWitnessWindow = 0
|
||||
t.Cleanup(func() { gateWitnessWindow = wasWindow })
|
||||
t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
|
||||
gateWithin, gateEvery, gateBound = time.Hour, 0, 300*time.Millisecond
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
advancePlans(ctx, b.open)
|
||||
if p := b.release(t); p.State != inventory.PlanRolling {
|
||||
break
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
p := b.release(t)
|
||||
v, found, err := inv.GateOf(ctx, "build-app-c2")
|
||||
if c.passes {
|
||||
if p.State != inventory.PlanDone || err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("the walk is %s (%s); app's verdict %+v: want the fix through", p.State, p.Note, v)
|
||||
}
|
||||
if !strings.Contains(v.Why+p.Note, "hand it over") {
|
||||
t.Errorf("the wait is not carried: verdict %q, walk %q", v.Why, p.Note)
|
||||
}
|
||||
return
|
||||
}
|
||||
if p.State == inventory.PlanDone {
|
||||
t.Fatalf("a directory this send found let the walk through: %s", p.Note)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestADirectoryUsedAsFoundIsItsOwnCondition(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
k, _ := withConditionsInMemory(t)
|
||||
ctx := t.Context()
|
||||
rs := map[string][]inventory.ResourceHealth{"notes": {foundDirectory("notes", time.Now().Add(-time.Hour))}}
|
||||
for i := 0; i < 2; i++ {
|
||||
if err := judgeModuleHealth(ctx, nil, k, "laptop", rs, map[string]int{"notes": i + 1}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
open, _ := k.Open(ctx)
|
||||
var got *conditions.Condition
|
||||
for i, c := range open {
|
||||
if c.Key == usedAsFoundKey("notes", "laptop") {
|
||||
got = &open[i]
|
||||
}
|
||||
if c.Kind == kindModuleUnhealthy {
|
||||
t.Fatalf("raised as a fault: %+v", c)
|
||||
}
|
||||
}
|
||||
if got == nil || got.Resolver != conditions.ResolverOperator || got.Severity == conditions.Urgent ||
|
||||
!strings.Contains(got.Summary, "notes.data") {
|
||||
t.Fatalf("the condition: %+v", got)
|
||||
}
|
||||
// Long open is still not urgent: only a person can hand it over, and nothing is broken by the wait.
|
||||
if err := judgeModuleHealth(ctx, nil, k, "laptop", rs, map[string]int{"notes": 3}, time.Now().Add(48*time.Hour)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open, _ = k.Open(ctx)
|
||||
for _, c := range open {
|
||||
if c.Key == usedAsFoundKey("notes", "laptop") && c.Severity == conditions.Urgent {
|
||||
t.Fatal("a directory used as found became urgent")
|
||||
}
|
||||
}
|
||||
if err := judgeModuleHealth(ctx, nil, k, "laptop", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open, _ = k.Open(ctx)
|
||||
for _, c := range open {
|
||||
if c.Key == usedAsFoundKey("notes", "laptop") {
|
||||
t.Fatal("not cleared once handed over")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,10 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
@@ -48,116 +46,5 @@ func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) {
|
||||
|
||||
// The whole walk: a module whose directory was used as found long before still gets its fix to every machine,
|
||||
// its pass kept and the wait said; a directory this very send found holds it and puts it back.
|
||||
func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
found time.Duration // when the directory was found, against now
|
||||
passes bool
|
||||
}{
|
||||
{"found a day before the send", -24 * time.Hour, true},
|
||||
{"found by this send", time.Hour, false},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
|
||||
return map[string]bool{"anchor": true, "laptop": true}, nil
|
||||
}
|
||||
backlogFacts := gatherGateFacts
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
f, err := backlogFacts(ctx, open, component)
|
||||
// The used-as-found condition, raised after the send (its second statement): its own kind, never a
|
||||
// fault the gate reads as the build's.
|
||||
f.judged, f.openErr = true, nil
|
||||
f.open = append(f.open, conditions.Condition{Key: usedAsFoundKey("app", "anchor"), Kind: kindUsedAsFound,
|
||||
Subject: conditions.Subject{Scope: conditions.ScopeModule, ID: "app.anchor", Machine: "anchor"},
|
||||
Raised: time.Now()})
|
||||
f.health = map[string]inventory.NodeHealth{}
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
f.health[n] = inventory.NodeHealth{Node: n, HeardAt: time.Now(), Resources: []inventory.ResourceHealth{
|
||||
{Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy},
|
||||
foundDirectory("app", time.Now().Add(c.found)),
|
||||
{Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateHealthy}}}
|
||||
}
|
||||
return f, err
|
||||
}
|
||||
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
|
||||
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
|
||||
wasWindow := gateWitnessWindow
|
||||
gateWitnessWindow = 0
|
||||
t.Cleanup(func() { gateWitnessWindow = wasWindow })
|
||||
t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
|
||||
gateWithin, gateEvery, gateBound = time.Hour, 0, 300*time.Millisecond
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
advancePlans(ctx, b.open)
|
||||
if p := b.release(t); p.State != inventory.PlanRolling {
|
||||
break
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
p := b.release(t)
|
||||
v, found, err := inv.GateOf(ctx, "build-app-c2")
|
||||
if c.passes {
|
||||
if p.State != inventory.PlanDone || err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("the walk is %s (%s); app's verdict %+v: want the fix through", p.State, p.Note, v)
|
||||
}
|
||||
if !strings.Contains(v.Why+p.Note, "hand it over") {
|
||||
t.Errorf("the wait is not carried: verdict %q, walk %q", v.Why, p.Note)
|
||||
}
|
||||
return
|
||||
}
|
||||
if p.State == inventory.PlanDone {
|
||||
t.Fatalf("a directory this send found let the walk through: %s", p.Note)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The condition says the wait in its own kind: the operator's, never urgent, and cleared once handed over.
|
||||
func TestADirectoryUsedAsFoundIsItsOwnCondition(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
k, _ := withConditionsInMemory(t)
|
||||
ctx := t.Context()
|
||||
rs := map[string][]inventory.ResourceHealth{"notes": {foundDirectory("notes", time.Now().Add(-time.Hour))}}
|
||||
for i := 0; i < 2; i++ {
|
||||
if err := judgeModuleHealth(ctx, nil, k, "laptop", rs, map[string]int{"notes": i + 1}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
open, _ := k.Open(ctx)
|
||||
var got *conditions.Condition
|
||||
for i, c := range open {
|
||||
if c.Key == usedAsFoundKey("notes", "laptop") {
|
||||
got = &open[i]
|
||||
}
|
||||
if c.Kind == kindModuleUnhealthy {
|
||||
t.Fatalf("raised as a fault: %+v", c)
|
||||
}
|
||||
}
|
||||
if got == nil || got.Resolver != conditions.ResolverOperator || got.Severity == conditions.Urgent ||
|
||||
!strings.Contains(got.Summary, "notes.data") {
|
||||
t.Fatalf("the condition: %+v", got)
|
||||
}
|
||||
// Long open is still not urgent: only a person can hand it over, and nothing is broken by the wait.
|
||||
if err := judgeModuleHealth(ctx, nil, k, "laptop", rs, map[string]int{"notes": 3}, time.Now().Add(48*time.Hour)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open, _ = k.Open(ctx)
|
||||
for _, c := range open {
|
||||
if c.Key == usedAsFoundKey("notes", "laptop") && c.Severity == conditions.Urgent {
|
||||
t.Fatal("a directory used as found became urgent")
|
||||
}
|
||||
}
|
||||
if err := judgeModuleHealth(ctx, nil, k, "laptop", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open, _ = k.Open(ctx)
|
||||
for _, c := range open {
|
||||
if c.Key == usedAsFoundKey("notes", "laptop") {
|
||||
t.Fatal("not cleared once handed over")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
|
||||
@@ -0,0 +1,482 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func aGateMesh(t *testing.T) *gateMesh {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
g := &gateMesh{open: open, health: map[string]health{}}
|
||||
// The gate's own tests judge a first machine, as walks a controller before novox/hq ADR 0360 started still are.
|
||||
wasGated := gateFirstMachine
|
||||
gateFirstMachine = true
|
||||
t.Cleanup(func() { gateFirstMachine = wasGated })
|
||||
g.keeper, _ = withConditionsInMemory(t)
|
||||
g.told = &conditions.Told{}
|
||||
was := doctorFrom
|
||||
doctorFrom = &doctor{open: open, keeper: g.keeper, teller: g.told}
|
||||
t.Cleanup(func() { doctorFrom = was })
|
||||
|
||||
for _, b := range []inventory.Build{
|
||||
{ID: "build-1", Module: "app", Commit: "c1", Repository: "novox/mesh-catalog", Path: "modules/app",
|
||||
Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)},
|
||||
{ID: "build-2", Module: "app", Commit: "c2", Repository: "novox/mesh-catalog", Path: "modules/app",
|
||||
Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)},
|
||||
} {
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: b.Commit})
|
||||
b.Manifest = manifest
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
register := func(commit string, asked time.Time) {
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: commit},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: commit,
|
||||
Head: commit, Asked: asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
register("c1", time.Now().Add(-2*time.Hour))
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.Assign(ctx, n, "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"app": "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
register("c2", time.Now().Add(-time.Minute))
|
||||
|
||||
// Every send: recorded with the build the module is at, applied and reported by the machine.
|
||||
n := 0
|
||||
wasSend := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
g.sent = append(g.sent, append([]string(nil), names...))
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, node := range names {
|
||||
n++
|
||||
digest := fmt.Sprintf("d-%s-%d", node, n)
|
||||
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, map[string]string{"app": current["app"].Commit}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
|
||||
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = wasSend })
|
||||
|
||||
// What a judging reads: the store's reports, and a health the test says per machine.
|
||||
wasGather := gatherGateFacts
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
|
||||
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
|
||||
reports, err := open.inventory.LastReports(ctx)
|
||||
if err != nil {
|
||||
return f, err
|
||||
}
|
||||
for _, r := range reports {
|
||||
if g.health[r.Node] == healthBroken {
|
||||
r.Outcome = inventory.OutcomeFailed
|
||||
}
|
||||
f.reports[r.Node] = r
|
||||
}
|
||||
for node, h := range g.health {
|
||||
if h == healthNotYet {
|
||||
// Not reported on the send: neither the send made last, nor the gate's own (issue 352).
|
||||
f.rolledBack[node] = nil
|
||||
r := f.reports[node]
|
||||
r.Current, r.Declared, r.ReportedSequence = false, "", 0
|
||||
f.reports[node] = r
|
||||
}
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
t.Cleanup(func() { gatherGateFacts = wasGather })
|
||||
|
||||
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
|
||||
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
|
||||
wasWindow := gateWitnessWindow
|
||||
gateWitnessWindow = 0
|
||||
t.Cleanup(func() { gateWitnessWindow = wasWindow })
|
||||
// One judging per advance until a test says otherwise: a pass waits gateEvery for the next.
|
||||
gateWithin, gateEvery = time.Hour, time.Hour
|
||||
t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
|
||||
|
||||
built := time.Now().UTC()
|
||||
plan := inventory.Plan{ID: "plan-gate", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
|
||||
Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
|
||||
Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2",
|
||||
Build: "build-2"}}}
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
func TestABuildThatFailsItsGateIsRolledBackOnItsFirstMachineAndGoesNoFurther(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := g.open.inventory
|
||||
|
||||
advancePlans(ctx, g.open) // the first machine is sent the new build
|
||||
if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}}) {
|
||||
t.Fatalf("sent %v, not the first machine alone", g.sent)
|
||||
}
|
||||
if p := g.plan(t); p.Modules["app"].Previous != "c1" {
|
||||
t.Fatalf("the build the first machine ran before was not kept: %+v", p.Modules["app"])
|
||||
}
|
||||
|
||||
g.health["anchor"] = healthBroken // the new build breaks its first machine, after one good judging
|
||||
gateEvery = 0
|
||||
advancePlans(ctx, g.open)
|
||||
|
||||
p := g.plan(t)
|
||||
gate := p.Modules["app"].Gate
|
||||
if p.State != inventory.PlanFailed || gate == nil || gate.Verdict != inventory.GateFailed ||
|
||||
gate.Rollback != inventory.RolledBack {
|
||||
t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate)
|
||||
}
|
||||
if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"anchor"}}) {
|
||||
t.Fatalf("sent %v: the rollback goes to the first machine, and nothing reaches laptop", g.sent)
|
||||
}
|
||||
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
|
||||
t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit)
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c1" {
|
||||
t.Fatalf("anchor was last sent %v, not the previous build", sent)
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["app"] != "c1" {
|
||||
t.Fatalf("laptop was sent the failed build: %v", sent)
|
||||
}
|
||||
if failed, err := inv.GateFailed(ctx, "build-2"); err != nil || !failed {
|
||||
t.Fatalf("the build is not marked failed at its gate: %v %v", failed, err)
|
||||
}
|
||||
|
||||
// Said: a condition for the operator, and the event.
|
||||
open, err := g.keeper.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var key string
|
||||
for _, c := range open {
|
||||
if c.Kind == kindRolledBack {
|
||||
key = c.Key
|
||||
}
|
||||
}
|
||||
if key != "build.app.anchor.rolled-back" {
|
||||
t.Fatalf("no rolled-back condition for app on anchor: %+v", open)
|
||||
}
|
||||
saidIt := false
|
||||
for _, e := range g.told.Said() {
|
||||
saidIt = saidIt || e.Event == link.KeyRolledBack
|
||||
}
|
||||
if !saidIt {
|
||||
t.Fatalf("the rollback was not said as an event: %+v", g.told.Said())
|
||||
}
|
||||
|
||||
// Never again: more passes send nothing, a second judging rolls nothing back, and the failed build
|
||||
// heard again is not registered.
|
||||
advancePlans(ctx, g.open)
|
||||
state := p.Modules["app"]
|
||||
gateFailed(ctx, g.open, &p, "app", state, []string{"anchor"}, "again")
|
||||
if len(g.sent) != 2 {
|
||||
t.Fatalf("sent again after the rollback: %v", g.sent)
|
||||
}
|
||||
_, _, err = takeIn(ctx, inv, link.BuildResult{ID: "build-2", Repository: "novox/mesh-catalog", Path: "modules/app",
|
||||
Commit: "c2", Manifest: mustJSON(t, catalogue.Manifest{Module: "app", Version: "c2"}),
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}})
|
||||
if err == nil || !strings.Contains(err.Error(), "failed its gate") {
|
||||
t.Fatalf("the failed build was registered again: %v", err)
|
||||
}
|
||||
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
|
||||
t.Fatalf("the module moved to %s", current["app"].Commit)
|
||||
}
|
||||
if _, err := retryPlan(ctx, g.open, "plan-gate"); err == nil || !strings.Contains(err.Error(), "failed its gate") {
|
||||
t.Fatalf("a plan stopped at a failed gate was retried: %v", err)
|
||||
}
|
||||
|
||||
// The probe keeps the condition while the newest verdict is the failure.
|
||||
obs, err := probeGates(ctx, doctorFrom)
|
||||
if err != nil || len(obs) != 1 || obs[0].Key() != key {
|
||||
t.Fatalf("the gate's probe found %+v %v", obs, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
gateEvery = 0
|
||||
for i := 0; i < 6; i++ {
|
||||
advancePlans(ctx, g.open)
|
||||
}
|
||||
p := g.plan(t)
|
||||
if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"laptop"}}) {
|
||||
t.Fatalf("sent %v", g.sent)
|
||||
}
|
||||
gate := p.Modules["app"].Gate
|
||||
if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gateWanted(gate) {
|
||||
t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate)
|
||||
}
|
||||
if v, found, err := g.open.inventory.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("the verdict was not kept: %+v %v %v", v, found, err)
|
||||
}
|
||||
if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 {
|
||||
t.Fatalf("a passing build left a condition: %+v %v", obs, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABuildRollsToEveryMachineAtOnceWithNoGate(t *testing.T) {
|
||||
g := aGateMesh(t)
|
||||
gateFirstMachine = false
|
||||
ctx := t.Context()
|
||||
judged := false
|
||||
wasGather := gatherGateFacts
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
judged = true
|
||||
return wasGather(ctx, open, component)
|
||||
}
|
||||
t.Cleanup(func() { gatherGateFacts = wasGather })
|
||||
for i := 0; i < 6; i++ {
|
||||
advancePlans(ctx, g.open)
|
||||
}
|
||||
p := g.plan(t)
|
||||
if !reflect.DeepEqual(g.sent, [][]string{{"anchor", "laptop"}}) {
|
||||
t.Fatalf("sent %v, not both machines in one send", g.sent)
|
||||
}
|
||||
if s := p.Modules["app"]; p.State != inventory.PlanDone || s.Gate != nil || s.FirstAt != nil || judged {
|
||||
t.Fatalf("the plan is %s (%s), app %+v, judged %v", p.State, p.Note, s, judged)
|
||||
}
|
||||
if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 {
|
||||
t.Fatalf("a rolled build left a condition: %+v %v", obs, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestALeafModulePassesOnOneHealthyReading(t *testing.T) {
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
gateWithin = time.Hour // a core module could not pass on fewer readings for an hour
|
||||
advancePlans(ctx, g.open) // the first machine is sent the new build
|
||||
advancePlans(ctx, g.open) // one healthy judging
|
||||
p := g.plan(t)
|
||||
gate := p.Modules["app"].Gate
|
||||
if gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes != 1 || gate.Wants != 1 {
|
||||
t.Fatalf("a leaf module's gate after one healthy judging: %+v (plan %s: %s)", gate, p.State, p.Note)
|
||||
}
|
||||
if !strings.Contains(gate.Why, "healthy 1 times") {
|
||||
t.Errorf("the verdict says %q", gate.Why)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
advancePlans(ctx, g.open)
|
||||
g.health["anchor"] = healthNotYet
|
||||
gateEvery = 0
|
||||
advancePlans(ctx, g.open)
|
||||
if p := g.plan(t); !p.Open() || len(g.sent) != 1 {
|
||||
t.Fatalf("judged before the bound: %s %v", p.State, g.sent)
|
||||
}
|
||||
gateBound = -time.Second
|
||||
advancePlans(ctx, g.open)
|
||||
p := g.plan(t)
|
||||
if gate := p.Modules["app"].Gate; p.State != inventory.PlanFailed || gate.Verdict != inventory.GateFailed ||
|
||||
!strings.Contains(gate.Why, "not healthy within") || gate.Rollback != inventory.RolledBack {
|
||||
t.Fatalf("the plan is %s, its gate %+v", p.State, gate)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheBusIsNeverRolledOutAutomatically(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
bus := catalogue.Manifest{Module: "nats", Version: "2", Provides: []catalogue.Offer{{Name: "mesh-bus"}},
|
||||
Upgrade: &catalogue.UpgradePolicy{Policy: catalogue.PolicyRoll}}
|
||||
if err := inv.RegisterModule(ctx, bus, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
|
||||
Path: "modules/nats", BuiltFrom: "n2", Head: "n2"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "anchor", "nats"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"nats": "n1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
u, err := inv.UpgradeOf(ctx, "nats")
|
||||
if err != nil || u.RollOut || u.From != catalogue.FromBus {
|
||||
t.Fatalf("the bus's policy is %+v %v", u, err)
|
||||
}
|
||||
if err := inv.SetUpgradeOf(ctx, "nats", inventory.Upgrade{RollOut: true}); !errors.Is(err, inventory.ErrBusIsPlanned) {
|
||||
t.Fatalf("a person rolled the bus out: %v", err)
|
||||
}
|
||||
var sent [][]string
|
||||
was := sendRollout
|
||||
sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) {
|
||||
sent = append(sent, names)
|
||||
return names, nil
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = was })
|
||||
now := time.Now().UTC()
|
||||
plan := inventory.Plan{ID: "plan-bus", Repository: "novox/mesh-catalog", Commit: "n2", Created: now,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"nats"}},
|
||||
Modules: map[string]*inventory.PlanModule{"nats": {State: "built", BuiltAt: &now, Commit: "n2", Build: "b"}}}
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, open)
|
||||
if p, _ := inv.PlanByID(ctx, "plan-bus"); p.State != inventory.PlanDone || len(sent) != 0 {
|
||||
t.Fatalf("a plan sent the bus: %s %v", p.State, sent)
|
||||
}
|
||||
held, err := busHeld(ctx, inv, []string{"anchor", "laptop"})
|
||||
if err != nil || !strings.Contains(held["anchor"], "planned step") || held["laptop"] != "" {
|
||||
t.Fatalf("a push may send the bus's machine: %v %v", held, err)
|
||||
}
|
||||
// Nor may any other send — a plan's for another module on that machine carried the bus with it.
|
||||
if _, err := sendToEach(ctx, open, []string{"laptop", "anchor"}); !errors.Is(err, errBusWaits) {
|
||||
t.Fatalf("a send to the bus's machine was not refused: %v", err)
|
||||
}
|
||||
// A rebuild that made the same artifacts is no move.
|
||||
for _, b := range []inventory.Build{{ID: "nb1", Module: "nats", Commit: "n1"}, {ID: "nb2", Module: "nats", Commit: "n2"}} {
|
||||
b.Made = []inventory.Artifact{{Name: "server", Kind: "image", Reference: "registry/nats@sha256:same"}}
|
||||
b.Asked, b.At = time.Now(), time.Now()
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if held, err := busHeld(ctx, inv, []string{"anchor"}); err != nil || len(held) != 0 {
|
||||
t.Fatalf("a rebuild that changes nothing held the bus's machine: %v %v", held, err)
|
||||
}
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "nb3", Module: "nats", Commit: "n2", Asked: time.Now().Add(time.Second),
|
||||
At: time.Now().Add(time.Second), Made: []inventory.Artifact{{Name: "server", Kind: "image",
|
||||
Reference: "registry/nats@sha256:new"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The planned step refuses to start without its word on reversibility, and without a snapshot taken
|
||||
// first by the bus machine's backup holder.
|
||||
if err := busCommand(ctx, []string{"upgrade", "--why", "2.11"}); err == nil || !strings.Contains(err.Error(), "reversible") {
|
||||
t.Fatalf("a bus upgrade started without saying whether it can be reverted: %v", err)
|
||||
}
|
||||
wasSnapshot := takeBusSnapshot
|
||||
t.Cleanup(func() { takeBusSnapshot = wasSnapshot })
|
||||
takeBusSnapshot = func(context.Context, string, string) (string, error) { return "", errors.New("no holder answers") }
|
||||
if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "snapshotted") || len(sent) != 0 {
|
||||
t.Fatalf("a bus upgrade started without its snapshot: %v, sent %v", err, sent)
|
||||
}
|
||||
takeBusSnapshot = func(_ context.Context, module, node string) (string, error) {
|
||||
return node + "'s restore point of " + module, nil
|
||||
}
|
||||
if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(sent, [][]string{{"anchor"}}) {
|
||||
t.Fatalf("the step sent %v, not the bus's machine", sent)
|
||||
}
|
||||
s, found, err := inv.LatestBusStep(ctx)
|
||||
if err != nil || !found || s.Snapshot != "anchor's restore point of nats" || s.Ended != nil ||
|
||||
!reflect.DeepEqual(s.Machines, []string{"anchor"}) {
|
||||
t.Fatalf("the step is %+v %v %v", s, found, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMergeThatDeletesAModulePlansNothingToBuildForIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asked := asksRecorded(t)
|
||||
for _, name := range []string{"gone", "kept"} {
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: name, Version: "1"}, inventory.Source{
|
||||
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + name, BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1deadbeef",
|
||||
Paths: []string{"modules/gone/module.json", "modules/gone/index.ts"},
|
||||
Removed: []string{"modules/gone/module.json", "modules/gone/index.ts"}}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(*asked) != 0 {
|
||||
t.Fatalf("a deleted module was asked to build: %v", *asked)
|
||||
}
|
||||
if plans, _ := inv.OpenPlans(ctx); len(plans) != 0 {
|
||||
t.Fatalf("a merge that only deleted a module made a plan: %+v", plans)
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, still := shelf["gone"]; still {
|
||||
t.Fatal("a deleted module nothing holds was not forgotten")
|
||||
}
|
||||
|
||||
// Without the announcer saying what went: the build finds no manifest, and the plan goes on.
|
||||
asked0 := time.Now().UTC().Add(-time.Minute)
|
||||
plan := inventory.Plan{ID: "plan-deleted", Repository: "novox/mesh-catalog", Commit: "c2", Created: asked0,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"kept"}},
|
||||
Modules: map[string]*inventory.PlanModule{"kept": {State: "asked", AskedAt: &asked0, Build: "build-k"}}}
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
planBuilt(ctx, open, "kept", "", "http://forge/novox/mesh-catalog.git has no module.json at modules/kept, so "+
|
||||
"there is nothing saying what it is: open …/module.json: no such file or directory", asked0, "build-k")
|
||||
p, _ := inv.PlanByID(ctx, "plan-deleted")
|
||||
if p.State == inventory.PlanFailed || p.Modules["kept"].State != planDeleted {
|
||||
t.Fatalf("a module deleted at its source failed the plan: %s %+v", p.State, p.Modules["kept"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWitnessesVerdictIsAConditionWhileItsReportsSayIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
d := &doctor{open: open}
|
||||
at := time.Now().UTC()
|
||||
witnessed.heard("control", []lease.Rollback{
|
||||
{Component: lease.ComponentController, From: "sha256:new", To: "sha256:old", Outcome: lease.OutcomeRolledBack,
|
||||
Why: "the new controller did not take the lease within 60s", At: at},
|
||||
{Component: lease.ComponentNodeTools, From: "sha256:t2", Outcome: lease.OutcomeUnwitnessed, Why: "no grant", At: at},
|
||||
}, at)
|
||||
t.Cleanup(func() { witnessed.heard("control", nil, time.Now()) })
|
||||
obs, err := probeGates(t.Context(), d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := map[string]conditions.Severity{}
|
||||
for _, o := range obs {
|
||||
got[o.Key()] = o.Severity
|
||||
}
|
||||
want := map[string]conditions.Severity{"core.controller.control.rolled-back": conditions.Urgent,
|
||||
"core.node-tools.control.unwitnessed": conditions.Warning}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("the witness's verdicts are %v", got)
|
||||
}
|
||||
witnessed.heard("control", nil, time.Now())
|
||||
if obs, err := probeGates(t.Context(), d); err != nil || len(obs) != 0 {
|
||||
t.Fatalf("a verdict the reports no longer carry is still said: %+v %v", obs, err)
|
||||
}
|
||||
}
|
||||
@@ -4,8 +4,6 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -30,124 +28,6 @@ type gateMesh struct {
|
||||
told *conditions.Told
|
||||
}
|
||||
|
||||
func aGateMesh(t *testing.T) *gateMesh {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
g := &gateMesh{open: open, health: map[string]health{}}
|
||||
// The gate's own tests judge a first machine, as walks a controller before novox/hq ADR 0360 started still are.
|
||||
wasGated := gateFirstMachine
|
||||
gateFirstMachine = true
|
||||
t.Cleanup(func() { gateFirstMachine = wasGated })
|
||||
g.keeper, _ = withConditionsInMemory(t)
|
||||
g.told = &conditions.Told{}
|
||||
was := doctorFrom
|
||||
doctorFrom = &doctor{open: open, keeper: g.keeper, teller: g.told}
|
||||
t.Cleanup(func() { doctorFrom = was })
|
||||
|
||||
for _, b := range []inventory.Build{
|
||||
{ID: "build-1", Module: "app", Commit: "c1", Repository: "novox/mesh-catalog", Path: "modules/app",
|
||||
Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)},
|
||||
{ID: "build-2", Module: "app", Commit: "c2", Repository: "novox/mesh-catalog", Path: "modules/app",
|
||||
Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)},
|
||||
} {
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: b.Commit})
|
||||
b.Manifest = manifest
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
register := func(commit string, asked time.Time) {
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: commit},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: commit,
|
||||
Head: commit, Asked: asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
register("c1", time.Now().Add(-2*time.Hour))
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.Assign(ctx, n, "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"app": "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
register("c2", time.Now().Add(-time.Minute))
|
||||
|
||||
// Every send: recorded with the build the module is at, applied and reported by the machine.
|
||||
n := 0
|
||||
wasSend := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
g.sent = append(g.sent, append([]string(nil), names...))
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, node := range names {
|
||||
n++
|
||||
digest := fmt.Sprintf("d-%s-%d", node, n)
|
||||
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, map[string]string{"app": current["app"].Commit}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
|
||||
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = wasSend })
|
||||
|
||||
// What a judging reads: the store's reports, and a health the test says per machine.
|
||||
wasGather := gatherGateFacts
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
|
||||
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
|
||||
reports, err := open.inventory.LastReports(ctx)
|
||||
if err != nil {
|
||||
return f, err
|
||||
}
|
||||
for _, r := range reports {
|
||||
if g.health[r.Node] == healthBroken {
|
||||
r.Outcome = inventory.OutcomeFailed
|
||||
}
|
||||
f.reports[r.Node] = r
|
||||
}
|
||||
for node, h := range g.health {
|
||||
if h == healthNotYet {
|
||||
// Not reported on the send: neither the send made last, nor the gate's own (issue 352).
|
||||
f.rolledBack[node] = nil
|
||||
r := f.reports[node]
|
||||
r.Current, r.Declared, r.ReportedSequence = false, "", 0
|
||||
f.reports[node] = r
|
||||
}
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
t.Cleanup(func() { gatherGateFacts = wasGather })
|
||||
|
||||
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
|
||||
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
|
||||
wasWindow := gateWitnessWindow
|
||||
gateWitnessWindow = 0
|
||||
t.Cleanup(func() { gateWitnessWindow = wasWindow })
|
||||
// One judging per advance until a test says otherwise: a pass waits gateEvery for the next.
|
||||
gateWithin, gateEvery = time.Hour, time.Hour
|
||||
t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
|
||||
|
||||
built := time.Now().UTC()
|
||||
plan := inventory.Plan{ID: "plan-gate", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
|
||||
Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
|
||||
Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2",
|
||||
Build: "build-2"}}}
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
func (g *gateMesh) plan(t *testing.T) inventory.Plan {
|
||||
t.Helper()
|
||||
p, err := g.open.inventory.PlanByID(t.Context(), "plan-gate")
|
||||
@@ -160,187 +40,17 @@ func (g *gateMesh) plan(t *testing.T) inventory.Plan {
|
||||
// A module's build that fails its gate on the first machine is put back there — the previous build
|
||||
// registered and sent to it again — and never reaches the second machine; it is marked, said as a
|
||||
// condition and an event, never registered or rolled back again.
|
||||
func TestABuildThatFailsItsGateIsRolledBackOnItsFirstMachineAndGoesNoFurther(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := g.open.inventory
|
||||
|
||||
advancePlans(ctx, g.open) // the first machine is sent the new build
|
||||
if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}}) {
|
||||
t.Fatalf("sent %v, not the first machine alone", g.sent)
|
||||
}
|
||||
if p := g.plan(t); p.Modules["app"].Previous != "c1" {
|
||||
t.Fatalf("the build the first machine ran before was not kept: %+v", p.Modules["app"])
|
||||
}
|
||||
|
||||
g.health["anchor"] = healthBroken // the new build breaks its first machine, after one good judging
|
||||
gateEvery = 0
|
||||
advancePlans(ctx, g.open)
|
||||
|
||||
p := g.plan(t)
|
||||
gate := p.Modules["app"].Gate
|
||||
if p.State != inventory.PlanFailed || gate == nil || gate.Verdict != inventory.GateFailed ||
|
||||
gate.Rollback != inventory.RolledBack {
|
||||
t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate)
|
||||
}
|
||||
if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"anchor"}}) {
|
||||
t.Fatalf("sent %v: the rollback goes to the first machine, and nothing reaches laptop", g.sent)
|
||||
}
|
||||
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
|
||||
t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit)
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c1" {
|
||||
t.Fatalf("anchor was last sent %v, not the previous build", sent)
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["app"] != "c1" {
|
||||
t.Fatalf("laptop was sent the failed build: %v", sent)
|
||||
}
|
||||
if failed, err := inv.GateFailed(ctx, "build-2"); err != nil || !failed {
|
||||
t.Fatalf("the build is not marked failed at its gate: %v %v", failed, err)
|
||||
}
|
||||
|
||||
// Said: a condition for the operator, and the event.
|
||||
open, err := g.keeper.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var key string
|
||||
for _, c := range open {
|
||||
if c.Kind == kindRolledBack {
|
||||
key = c.Key
|
||||
}
|
||||
}
|
||||
if key != "build.app.anchor.rolled-back" {
|
||||
t.Fatalf("no rolled-back condition for app on anchor: %+v", open)
|
||||
}
|
||||
saidIt := false
|
||||
for _, e := range g.told.Said() {
|
||||
saidIt = saidIt || e.Event == link.KeyRolledBack
|
||||
}
|
||||
if !saidIt {
|
||||
t.Fatalf("the rollback was not said as an event: %+v", g.told.Said())
|
||||
}
|
||||
|
||||
// Never again: more passes send nothing, a second judging rolls nothing back, and the failed build
|
||||
// heard again is not registered.
|
||||
advancePlans(ctx, g.open)
|
||||
state := p.Modules["app"]
|
||||
gateFailed(ctx, g.open, &p, "app", state, []string{"anchor"}, "again")
|
||||
if len(g.sent) != 2 {
|
||||
t.Fatalf("sent again after the rollback: %v", g.sent)
|
||||
}
|
||||
_, _, err = takeIn(ctx, inv, link.BuildResult{ID: "build-2", Repository: "novox/mesh-catalog", Path: "modules/app",
|
||||
Commit: "c2", Manifest: mustJSON(t, catalogue.Manifest{Module: "app", Version: "c2"}),
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}})
|
||||
if err == nil || !strings.Contains(err.Error(), "failed its gate") {
|
||||
t.Fatalf("the failed build was registered again: %v", err)
|
||||
}
|
||||
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
|
||||
t.Fatalf("the module moved to %s", current["app"].Commit)
|
||||
}
|
||||
if _, err := retryPlan(ctx, g.open, "plan-gate"); err == nil || !strings.Contains(err.Error(), "failed its gate") {
|
||||
t.Fatalf("a plan stopped at a failed gate was retried: %v", err)
|
||||
}
|
||||
|
||||
// The probe keeps the condition while the newest verdict is the failure.
|
||||
obs, err := probeGates(ctx, doctorFrom)
|
||||
if err != nil || len(obs) != 1 || obs[0].Key() != key {
|
||||
t.Fatalf("the gate's probe found %+v %v", obs, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A passing build rolls everywhere with no hand: judged healthy on its first machine three times, then
|
||||
// the rest are sent, the verdict kept, and the plan done.
|
||||
func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
gateEvery = 0
|
||||
for i := 0; i < 6; i++ {
|
||||
advancePlans(ctx, g.open)
|
||||
}
|
||||
p := g.plan(t)
|
||||
if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"laptop"}}) {
|
||||
t.Fatalf("sent %v", g.sent)
|
||||
}
|
||||
gate := p.Modules["app"].Gate
|
||||
if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gateWanted(gate) {
|
||||
t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate)
|
||||
}
|
||||
if v, found, err := g.open.inventory.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("the verdict was not kept: %+v %v %v", v, found, err)
|
||||
}
|
||||
if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 {
|
||||
t.Fatalf("a passing build left a condition: %+v %v", obs, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A build rolls everywhere with no hand and no gate (novox/hq ADR 0360 §1): both machines in one send, no health
|
||||
// judged, and the walk done once both reported on the send. Before, anchor was judged first, then laptop sent.
|
||||
func TestABuildRollsToEveryMachineAtOnceWithNoGate(t *testing.T) {
|
||||
g := aGateMesh(t)
|
||||
gateFirstMachine = false
|
||||
ctx := t.Context()
|
||||
judged := false
|
||||
wasGather := gatherGateFacts
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
judged = true
|
||||
return wasGather(ctx, open, component)
|
||||
}
|
||||
t.Cleanup(func() { gatherGateFacts = wasGather })
|
||||
for i := 0; i < 6; i++ {
|
||||
advancePlans(ctx, g.open)
|
||||
}
|
||||
p := g.plan(t)
|
||||
if !reflect.DeepEqual(g.sent, [][]string{{"anchor", "laptop"}}) {
|
||||
t.Fatalf("sent %v, not both machines in one send", g.sent)
|
||||
}
|
||||
if s := p.Modules["app"]; p.State != inventory.PlanDone || s.Gate != nil || s.FirstAt != nil || judged {
|
||||
t.Fatalf("the plan is %s (%s), app %+v, judged %v", p.State, p.Note, s, judged)
|
||||
}
|
||||
if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 {
|
||||
t.Fatalf("a rolled build left a condition: %+v %v", obs, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A leaf module passes its gate on one reading, as soon as its first machine is healthy, without waiting out the
|
||||
// core's settle time (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it, issue 524).
|
||||
func TestALeafModulePassesOnOneHealthyReading(t *testing.T) {
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
gateWithin = time.Hour // a core module could not pass on fewer readings for an hour
|
||||
advancePlans(ctx, g.open) // the first machine is sent the new build
|
||||
advancePlans(ctx, g.open) // one healthy judging
|
||||
p := g.plan(t)
|
||||
gate := p.Modules["app"].Gate
|
||||
if gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes != 1 || gate.Wants != 1 {
|
||||
t.Fatalf("a leaf module's gate after one healthy judging: %+v (plan %s: %s)", gate, p.State, p.Note)
|
||||
}
|
||||
if !strings.Contains(gate.Why, "healthy 1 times") {
|
||||
t.Errorf("the verdict says %q", gate.Why)
|
||||
}
|
||||
}
|
||||
|
||||
// A first machine that never becomes healthy fails its gate at the bound, and is put back.
|
||||
func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
advancePlans(ctx, g.open)
|
||||
g.health["anchor"] = healthNotYet
|
||||
gateEvery = 0
|
||||
advancePlans(ctx, g.open)
|
||||
if p := g.plan(t); !p.Open() || len(g.sent) != 1 {
|
||||
t.Fatalf("judged before the bound: %s %v", p.State, g.sent)
|
||||
}
|
||||
gateBound = -time.Second
|
||||
advancePlans(ctx, g.open)
|
||||
p := g.plan(t)
|
||||
if gate := p.Modules["app"].Gate; p.State != inventory.PlanFailed || gate.Verdict != inventory.GateFailed ||
|
||||
!strings.Contains(gate.Why, "not healthy within") || gate.Rollback != inventory.RolledBack {
|
||||
t.Fatalf("the plan is %s, its gate %+v", p.State, gate)
|
||||
}
|
||||
}
|
||||
|
||||
// The health a judging finds, per core component and for a module.
|
||||
func TestTheHealthDefinitions(t *testing.T) {
|
||||
@@ -458,147 +168,9 @@ func TestTheHealthDefinitions(t *testing.T) {
|
||||
|
||||
// The bus is never rolled out: its policy records whatever is said, a person's roll-out is refused,
|
||||
// a plan builds it and sends nothing, and a push naming its machine is refused while a new build waits.
|
||||
func TestTheBusIsNeverRolledOutAutomatically(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
bus := catalogue.Manifest{Module: "nats", Version: "2", Provides: []catalogue.Offer{{Name: "mesh-bus"}},
|
||||
Upgrade: &catalogue.UpgradePolicy{Policy: catalogue.PolicyRoll}}
|
||||
if err := inv.RegisterModule(ctx, bus, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
|
||||
Path: "modules/nats", BuiltFrom: "n2", Head: "n2"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "anchor", "nats"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"nats": "n1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
u, err := inv.UpgradeOf(ctx, "nats")
|
||||
if err != nil || u.RollOut || u.From != catalogue.FromBus {
|
||||
t.Fatalf("the bus's policy is %+v %v", u, err)
|
||||
}
|
||||
if err := inv.SetUpgradeOf(ctx, "nats", inventory.Upgrade{RollOut: true}); !errors.Is(err, inventory.ErrBusIsPlanned) {
|
||||
t.Fatalf("a person rolled the bus out: %v", err)
|
||||
}
|
||||
var sent [][]string
|
||||
was := sendRollout
|
||||
sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) {
|
||||
sent = append(sent, names)
|
||||
return names, nil
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = was })
|
||||
now := time.Now().UTC()
|
||||
plan := inventory.Plan{ID: "plan-bus", Repository: "novox/mesh-catalog", Commit: "n2", Created: now,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"nats"}},
|
||||
Modules: map[string]*inventory.PlanModule{"nats": {State: "built", BuiltAt: &now, Commit: "n2", Build: "b"}}}
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, open)
|
||||
if p, _ := inv.PlanByID(ctx, "plan-bus"); p.State != inventory.PlanDone || len(sent) != 0 {
|
||||
t.Fatalf("a plan sent the bus: %s %v", p.State, sent)
|
||||
}
|
||||
held, err := busHeld(ctx, inv, []string{"anchor", "laptop"})
|
||||
if err != nil || !strings.Contains(held["anchor"], "planned step") || held["laptop"] != "" {
|
||||
t.Fatalf("a push may send the bus's machine: %v %v", held, err)
|
||||
}
|
||||
// Nor may any other send — a plan's for another module on that machine carried the bus with it.
|
||||
if _, err := sendToEach(ctx, open, []string{"laptop", "anchor"}); !errors.Is(err, errBusWaits) {
|
||||
t.Fatalf("a send to the bus's machine was not refused: %v", err)
|
||||
}
|
||||
// A rebuild that made the same artifacts is no move.
|
||||
for _, b := range []inventory.Build{{ID: "nb1", Module: "nats", Commit: "n1"}, {ID: "nb2", Module: "nats", Commit: "n2"}} {
|
||||
b.Made = []inventory.Artifact{{Name: "server", Kind: "image", Reference: "registry/nats@sha256:same"}}
|
||||
b.Asked, b.At = time.Now(), time.Now()
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if held, err := busHeld(ctx, inv, []string{"anchor"}); err != nil || len(held) != 0 {
|
||||
t.Fatalf("a rebuild that changes nothing held the bus's machine: %v %v", held, err)
|
||||
}
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "nb3", Module: "nats", Commit: "n2", Asked: time.Now().Add(time.Second),
|
||||
At: time.Now().Add(time.Second), Made: []inventory.Artifact{{Name: "server", Kind: "image",
|
||||
Reference: "registry/nats@sha256:new"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The planned step refuses to start without its word on reversibility, and without a snapshot taken
|
||||
// first by the bus machine's backup holder.
|
||||
if err := busCommand(ctx, []string{"upgrade", "--why", "2.11"}); err == nil || !strings.Contains(err.Error(), "reversible") {
|
||||
t.Fatalf("a bus upgrade started without saying whether it can be reverted: %v", err)
|
||||
}
|
||||
wasSnapshot := takeBusSnapshot
|
||||
t.Cleanup(func() { takeBusSnapshot = wasSnapshot })
|
||||
takeBusSnapshot = func(context.Context, string, string) (string, error) { return "", errors.New("no holder answers") }
|
||||
if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "snapshotted") || len(sent) != 0 {
|
||||
t.Fatalf("a bus upgrade started without its snapshot: %v, sent %v", err, sent)
|
||||
}
|
||||
takeBusSnapshot = func(_ context.Context, module, node string) (string, error) {
|
||||
return node + "'s restore point of " + module, nil
|
||||
}
|
||||
if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(sent, [][]string{{"anchor"}}) {
|
||||
t.Fatalf("the step sent %v, not the bus's machine", sent)
|
||||
}
|
||||
s, found, err := inv.LatestBusStep(ctx)
|
||||
if err != nil || !found || s.Snapshot != "anchor's restore point of nats" || s.Ended != nil ||
|
||||
!reflect.DeepEqual(s.Machines, []string{"anchor"}) {
|
||||
t.Fatalf("the step is %+v %v %v", s, found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge that deletes a module's directory builds nothing for it: the module is forgotten where
|
||||
// nothing holds it, and a plan whose build finds no manifest goes on instead of failing.
|
||||
func TestAMergeThatDeletesAModulePlansNothingToBuildForIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asked := asksRecorded(t)
|
||||
for _, name := range []string{"gone", "kept"} {
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: name, Version: "1"}, inventory.Source{
|
||||
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + name, BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1deadbeef",
|
||||
Paths: []string{"modules/gone/module.json", "modules/gone/index.ts"},
|
||||
Removed: []string{"modules/gone/module.json", "modules/gone/index.ts"}}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(*asked) != 0 {
|
||||
t.Fatalf("a deleted module was asked to build: %v", *asked)
|
||||
}
|
||||
if plans, _ := inv.OpenPlans(ctx); len(plans) != 0 {
|
||||
t.Fatalf("a merge that only deleted a module made a plan: %+v", plans)
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, still := shelf["gone"]; still {
|
||||
t.Fatal("a deleted module nothing holds was not forgotten")
|
||||
}
|
||||
|
||||
// Without the announcer saying what went: the build finds no manifest, and the plan goes on.
|
||||
asked0 := time.Now().UTC().Add(-time.Minute)
|
||||
plan := inventory.Plan{ID: "plan-deleted", Repository: "novox/mesh-catalog", Commit: "c2", Created: asked0,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"kept"}},
|
||||
Modules: map[string]*inventory.PlanModule{"kept": {State: "asked", AskedAt: &asked0, Build: "build-k"}}}
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
planBuilt(ctx, open, "kept", "", "http://forge/novox/mesh-catalog.git has no module.json at modules/kept, so "+
|
||||
"there is nothing saying what it is: open …/module.json: no such file or directory", asked0, "build-k")
|
||||
p, _ := inv.PlanByID(ctx, "plan-deleted")
|
||||
if p.State == inventory.PlanFailed || p.Modules["kept"].State != planDeleted {
|
||||
t.Fatalf("a module deleted at its source failed the plan: %s %+v", p.State, p.Modules["kept"])
|
||||
}
|
||||
}
|
||||
|
||||
func mustJSON(t *testing.T, v any) []byte {
|
||||
t.Helper()
|
||||
@@ -621,34 +193,6 @@ func nodeID(t *testing.T, open *stores, name string) string {
|
||||
// What a machine's witness says in its reports is a condition while it says it, by the host's words:
|
||||
// `core.<component>.<node>.<outcome>`, urgent for a rollback, a warning when it could not judge — and
|
||||
// gone with the first report that no longer carries it.
|
||||
func TestAWitnessesVerdictIsAConditionWhileItsReportsSayIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
d := &doctor{open: open}
|
||||
at := time.Now().UTC()
|
||||
witnessed.heard("control", []lease.Rollback{
|
||||
{Component: lease.ComponentController, From: "sha256:new", To: "sha256:old", Outcome: lease.OutcomeRolledBack,
|
||||
Why: "the new controller did not take the lease within 60s", At: at},
|
||||
{Component: lease.ComponentNodeTools, From: "sha256:t2", Outcome: lease.OutcomeUnwitnessed, Why: "no grant", At: at},
|
||||
}, at)
|
||||
t.Cleanup(func() { witnessed.heard("control", nil, time.Now()) })
|
||||
obs, err := probeGates(t.Context(), d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := map[string]conditions.Severity{}
|
||||
for _, o := range obs {
|
||||
got[o.Key()] = o.Severity
|
||||
}
|
||||
want := map[string]conditions.Severity{"core.controller.control.rolled-back": conditions.Urgent,
|
||||
"core.node-tools.control.unwitnessed": conditions.Warning}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("the witness's verdicts are %v", got)
|
||||
}
|
||||
witnessed.heard("control", nil, time.Now())
|
||||
if obs, err := probeGates(t.Context(), d); err != nil || len(obs) != 0 {
|
||||
t.Fatalf("a verdict the reports no longer carry is still said: %+v %v", obs, err)
|
||||
}
|
||||
}
|
||||
|
||||
// judgeLeavesAsTheCore makes a test's leaf modules judged as a core module is, three readings after the settle
|
||||
// time, for a test of the gate's other rules (a fault, the bound, a supersession) that needs a judging still
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
func TestAConsumerPinnedBackIsNoLongerGrantedByTheProviderItLeft(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
for _, m := range storeManifests() {
|
||||
register(t, open, m)
|
||||
}
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.HoldSeat(ctx, "mesh-store", catalogue.ScopeMesh, "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, a := range [][2]string{{"laptop", "store"}, {"laptop", "board"}} {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
|
||||
// Bound to the anchor's store, and sent so: the credential from the anchor is made and recorded.
|
||||
if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, _, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := need(t, plan, "board", "postgres-database"); n.From != "anchor" {
|
||||
t.Fatalf("pinned to the anchor and bound to %s", n.From)
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
grants, _, unbound, err := grantsFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); !ok || g.From != "board" || len(unbound) != 0 {
|
||||
t.Fatalf("a consumer bound to the anchor is not granted there: %+v, unbound %+v", grants, unbound)
|
||||
}
|
||||
|
||||
// Pinned back beside its data, as the operator did.
|
||||
if err := inv.PinProvision(ctx, "laptop", "postgres-database", "laptop", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, _, err = planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := need(t, plan, "board", "postgres-database"); n.From != "laptop" {
|
||||
t.Fatalf("pinned back to the laptop and bound to %s", n.From)
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
holders, err := inv.HoldersOf(ctx, "postgres-database", "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(holders) != 2 {
|
||||
t.Fatalf("today's state is two credentials on record, one from each store: %+v", holders)
|
||||
}
|
||||
|
||||
// The store it left: no longer granted, and said.
|
||||
grants, _, unbound, err = grantsFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); ok && g.From != "" {
|
||||
t.Fatalf("the anchor's store is still granted a consumer bound to the laptop's: %+v", g)
|
||||
}
|
||||
if len(unbound) != 1 || unbound[0].Module != "board" || unbound[0].Provider != "anchor" ||
|
||||
strings.Join(unbound[0].BoundTo, ",") != "laptop" {
|
||||
t.Fatalf("the consumer that moved is not the one said: %+v", unbound)
|
||||
}
|
||||
planned, settings, err := planFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationFor(ctx, open, "anchor", planned, settings)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := declared.Received["store"]["postgres-database"]; len(got) != 0 {
|
||||
t.Fatalf("the anchor's store is still told about %+v", got)
|
||||
}
|
||||
said := printed(t, func() error { reportLeftOut("anchor", declared); return nil })
|
||||
if !strings.Contains(said, "board on laptop is bound to laptop for postgres-database, not to anchor") ||
|
||||
!strings.Contains(said, "cleanup delete") {
|
||||
t.Fatalf("the push does not say whom the anchor no longer grants:\n%s", said)
|
||||
}
|
||||
|
||||
// The store it is bound to: granted.
|
||||
grants, _, unbound, err = grantsFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); !ok || g.From != "board" || len(unbound) != 0 {
|
||||
t.Fatalf("the consumer is not granted by the store it is bound to: %+v, unbound %+v", grants, unbound)
|
||||
}
|
||||
|
||||
// And the credential from the store it left is kept: the key to the login and data held there.
|
||||
if holders, err = inv.HoldersOf(ctx, "postgres-database", "laptop"); err != nil || len(holders) != 2 {
|
||||
t.Fatalf("a credential was forgotten while its provider still holds the login: %+v, %v", holders, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAConsumerWhoseResolutionCannotBeReadIsNotWithdrawn(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
for _, m := range storeManifests() {
|
||||
register(t, open, m)
|
||||
}
|
||||
for _, a := range [][2]string{{"anchor", "store"}, {"laptop", "board"}} {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
if _, _, err := planFor(ctx, open, "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The laptop's key changes to one nothing can seal to: its resolution cannot be completed, and
|
||||
// that is not its set failing to compose.
|
||||
laptop, err := inv.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSealingKey(ctx, laptop.ID, "not a key"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := planFor(ctx, open, "laptop"); err == nil || unresolvable(err) {
|
||||
t.Fatalf("the seam this test relies on moved: %v", err)
|
||||
}
|
||||
grants, _, unbound, err := grantsFor(ctx, open, "anchor")
|
||||
if err == nil {
|
||||
t.Fatalf("an unreadable consumer was answered: grants %+v, unbound %+v", grants, unbound)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "what laptop asked of postgres-database cannot be read") {
|
||||
t.Fatalf("the error does not say whose resolution could not be read: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -26,154 +26,9 @@ func grantOf(grants []catalogue.Grant, provision, consumer, module string) (cata
|
||||
// credentials are on record. The store it left is no longer granted it — so it retires it and keeps
|
||||
// its data (ADR 0230) — and says so; the store it is bound to keeps its grant; and the credential from
|
||||
// the store it left stays on record.
|
||||
func TestAConsumerPinnedBackIsNoLongerGrantedByTheProviderItLeft(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
for _, m := range storeManifests() {
|
||||
register(t, open, m)
|
||||
}
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.HoldSeat(ctx, "mesh-store", catalogue.ScopeMesh, "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, a := range [][2]string{{"laptop", "store"}, {"laptop", "board"}} {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
|
||||
// Bound to the anchor's store, and sent so: the credential from the anchor is made and recorded.
|
||||
if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, _, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := need(t, plan, "board", "postgres-database"); n.From != "anchor" {
|
||||
t.Fatalf("pinned to the anchor and bound to %s", n.From)
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
grants, _, unbound, err := grantsFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); !ok || g.From != "board" || len(unbound) != 0 {
|
||||
t.Fatalf("a consumer bound to the anchor is not granted there: %+v, unbound %+v", grants, unbound)
|
||||
}
|
||||
|
||||
// Pinned back beside its data, as the operator did.
|
||||
if err := inv.PinProvision(ctx, "laptop", "postgres-database", "laptop", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, _, err = planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := need(t, plan, "board", "postgres-database"); n.From != "laptop" {
|
||||
t.Fatalf("pinned back to the laptop and bound to %s", n.From)
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
holders, err := inv.HoldersOf(ctx, "postgres-database", "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(holders) != 2 {
|
||||
t.Fatalf("today's state is two credentials on record, one from each store: %+v", holders)
|
||||
}
|
||||
|
||||
// The store it left: no longer granted, and said.
|
||||
grants, _, unbound, err = grantsFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); ok && g.From != "" {
|
||||
t.Fatalf("the anchor's store is still granted a consumer bound to the laptop's: %+v", g)
|
||||
}
|
||||
if len(unbound) != 1 || unbound[0].Module != "board" || unbound[0].Provider != "anchor" ||
|
||||
strings.Join(unbound[0].BoundTo, ",") != "laptop" {
|
||||
t.Fatalf("the consumer that moved is not the one said: %+v", unbound)
|
||||
}
|
||||
planned, settings, err := planFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationFor(ctx, open, "anchor", planned, settings)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := declared.Received["store"]["postgres-database"]; len(got) != 0 {
|
||||
t.Fatalf("the anchor's store is still told about %+v", got)
|
||||
}
|
||||
said := printed(t, func() error { reportLeftOut("anchor", declared); return nil })
|
||||
if !strings.Contains(said, "board on laptop is bound to laptop for postgres-database, not to anchor") ||
|
||||
!strings.Contains(said, "cleanup delete") {
|
||||
t.Fatalf("the push does not say whom the anchor no longer grants:\n%s", said)
|
||||
}
|
||||
|
||||
// The store it is bound to: granted.
|
||||
grants, _, unbound, err = grantsFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); !ok || g.From != "board" || len(unbound) != 0 {
|
||||
t.Fatalf("the consumer is not granted by the store it is bound to: %+v, unbound %+v", grants, unbound)
|
||||
}
|
||||
|
||||
// And the credential from the store it left is kept: the key to the login and data held there.
|
||||
if holders, err = inv.HoldersOf(ctx, "postgres-database", "laptop"); err != nil || len(holders) != 2 {
|
||||
t.Fatalf("a credential was forgotten while its provider still holds the login: %+v, %v", holders, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A consumer whose resolution cannot be read is an error, never a consumer bound nowhere — withdrawing
|
||||
// a grant on that reading would take its access away (issue 152).
|
||||
func TestAConsumerWhoseResolutionCannotBeReadIsNotWithdrawn(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
for _, m := range storeManifests() {
|
||||
register(t, open, m)
|
||||
}
|
||||
for _, a := range [][2]string{{"anchor", "store"}, {"laptop", "board"}} {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
if _, _, err := planFor(ctx, open, "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The laptop's key changes to one nothing can seal to: its resolution cannot be completed, and
|
||||
// that is not its set failing to compose.
|
||||
laptop, err := inv.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSealingKey(ctx, laptop.ID, "not a key"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := planFor(ctx, open, "laptop"); err == nil || unresolvable(err) {
|
||||
t.Fatalf("the seam this test relies on moved: %v", err)
|
||||
}
|
||||
grants, _, unbound, err := grantsFor(ctx, open, "anchor")
|
||||
if err == nil {
|
||||
t.Fatalf("an unreadable consumer was answered: grants %+v, unbound %+v", grants, unbound)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "what laptop asked of postgres-database cannot be read") {
|
||||
t.Fatalf("the error does not say whose resolution could not be read: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The rule itself, on a resolution: bound is the provider a need for exactly that credential is
|
||||
// answered by, never one answered by a record, and a different local name is a different credential.
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// The store test's manifests are manifests the controller takes in: checked here, where no store is needed,
|
||||
// so a fixture the module rules refuse fails before the store test is run (repo-check of #230 at 325575b2).
|
||||
func TestTheGrantsStepFixturesAreManifests(t *testing.T) {
|
||||
for name, m := range map[string]map[string]any{"nats": natsFixture(), "restic-c1": resticFixture(false),
|
||||
"restic-c2": resticFixture(true), "wallpaper": wallpaperFixture()} {
|
||||
m["module"], m["version"] = strings.SplitN(name, "-", 2)[0], "1"
|
||||
raw, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := catalogue.ParseManifest(raw); err != nil {
|
||||
t.Errorf("%s: %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
|
||||
@@ -177,22 +177,6 @@ func TestTheGrantsStepKeepsOneMachinesBuilds(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The store test's manifests are manifests the controller takes in: checked here, where no store is needed,
|
||||
// so a fixture the module rules refuse fails before the store test is run (repo-check of #230 at 325575b2).
|
||||
func TestTheGrantsStepFixturesAreManifests(t *testing.T) {
|
||||
for name, m := range map[string]map[string]any{"nats": natsFixture(), "restic-c1": resticFixture(false),
|
||||
"restic-c2": resticFixture(true), "wallpaper": wallpaperFixture()} {
|
||||
m["module"], m["version"] = strings.SplitN(name, "-", 2)[0], "1"
|
||||
raw, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := catalogue.ParseManifest(raw); err != nil {
|
||||
t.Errorf("%s: %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The delivery plan says the step before the merge.
|
||||
func TestAChangePlanSaysTheGrantsStep(t *testing.T) {
|
||||
const catalogue_ = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A group is one batch and one plan (novox/hq ADR 0418 clauses 3 and 6): the batch holding a merge of a group with a
|
||||
// member still to merge does not close by the merge window, however long that member takes, and a group with a
|
||||
// member of each kind is one batch whatever order its members are heard in.
|
||||
|
||||
func TestAGroupsBatchWaitsForItsLastMemberHoweverLongItTakes(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
states := map[string]string{"novox/one": "ready", "novox/two": "ready"}
|
||||
groupStates(t, "feat/g", states)
|
||||
first := fromHead(repoMerge("one", "c1", t0), "feat/g")
|
||||
hear(t, open, first, t0)
|
||||
// Past the window and past its maximum, with the second member still to merge: one batch, no walk.
|
||||
for _, after := range []time.Duration{90 * time.Second, 11 * time.Minute, 30 * time.Minute} {
|
||||
cutAt(t, open, t0.Add(after))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Batch.Holds) != 1 {
|
||||
t.Fatalf("%s after the first member: %d walk(s), %d batch(es) %+v", after, len(ws), len(bs), bs)
|
||||
}
|
||||
}
|
||||
// The last member merges: the window runs again from that merge.
|
||||
at := t0.Add(30 * time.Minute)
|
||||
states["novox/two"] = "merged"
|
||||
hear(t, open, fromHead(repoMerge("two", "c2", at), "feat/g"), at.Add(time.Second))
|
||||
cutAt(t, open, at.Add(time.Second+89*time.Second))
|
||||
if ws, _ := walks(t, open); len(ws) != 0 {
|
||||
t.Fatalf("cut before the window ran again: %+v", ws)
|
||||
}
|
||||
cutAt(t, open, at.Add(time.Second+90*time.Second))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 0 || ws[0].CommitOf("novox/one") != "c1" || ws[0].CommitOf("novox/two") != "c2" {
|
||||
t.Fatalf("after the window: %d walk(s) %+v, %d batch(es)", len(ws), ws, len(bs))
|
||||
}
|
||||
}
|
||||
|
||||
// A group with a member of each kind is one batch and one walk, in both orders of hearing, started without
|
||||
// mesh-delivery's word.
|
||||
func TestAGroupWithAMemberOfEachKindIsOneBatchAndOneWalk(t *testing.T) {
|
||||
for name, ownFirst := range map[string]bool{"controller member first": true, "catalogue member first": false} {
|
||||
t.Run(strings.ReplaceAll(name, " ", "_"), func(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
states := map[string]string{"novox/mesh-controller": "ready", "novox/mesh-catalog": "ready"}
|
||||
groupStates(t, "feat/both-kinds", states)
|
||||
own := fromHead(repoMerge("mesh-controller", "k1", t0), "feat/both-kinds")
|
||||
cat := fromHead(catalogueMerge("cat1aaaaaaaa", "app", t0), "feat/both-kinds")
|
||||
first, second := cat, own
|
||||
if ownFirst {
|
||||
first, second = own, cat
|
||||
}
|
||||
second.MergedAt = t0.Add(10 * time.Second).Format(time.RFC3339Nano)
|
||||
hear(t, open, first, t0)
|
||||
states["novox/"+second.Repo] = "merged"
|
||||
hear(t, open, second, t0.Add(10*time.Second))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 {
|
||||
t.Fatalf("both members heard: %d walk(s), %d batch(es) %+v; want one batch of both", len(ws), len(bs), bs)
|
||||
}
|
||||
cutAt(t, open, t0.Add(10*time.Second+90*time.Second))
|
||||
ws, bs = walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 0 || ws[0].CommitOf("novox/mesh-controller") != "k1" ||
|
||||
ws[0].CommitOf("novox/mesh-catalog") != cat.Commit {
|
||||
t.Fatalf("after the window: %d walk(s) %+v, %d batch(es)", len(ws), ws, len(bs))
|
||||
}
|
||||
if ws[0].Delivery != nil && ws[0].Delivery.Awaits != "" {
|
||||
t.Fatalf("a plan carrying the controller waits for %s's word", ws[0].Delivery.Awaits)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A group whose last member has not merged ten minutes after its first raises a condition naming it, and the held
|
||||
// batch is not the batch S18 reads as a cut the controller failed to make.
|
||||
func TestAHeldBatchRaisesTheGroupConditionAndNotTheBatchNotCutOne(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
groupStates(t, "feat/g", map[string]string{"novox/one": "ready", "novox/two": "ready"})
|
||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/g"), t0)
|
||||
now := t0.Add(12 * time.Minute)
|
||||
batches, err := gatherBatches(t.Context(), open.inventory, now)
|
||||
if err != nil || len(batches) != 1 {
|
||||
t.Fatalf("batches %+v: %v", batches, err)
|
||||
}
|
||||
f := &signalFacts{now: now, batches: batches}
|
||||
if got := watchBatchesNotCut(f); len(got) != 0 {
|
||||
t.Fatalf("a held batch is read as not cut: %+v", got)
|
||||
}
|
||||
got := watchGroupsNotMerged(f)
|
||||
if len(got) != 1 || got[0].Kind != kindGroupNotMerged || !strings.Contains(got[0].Summary, "novox/two") ||
|
||||
!strings.Contains(got[0].Summary, "feat/g") {
|
||||
t.Fatalf("the group's condition: %+v", got)
|
||||
}
|
||||
if early := watchGroupsNotMerged(&signalFacts{now: t0.Add(5 * time.Minute), batches: batches}); len(early) != 0 {
|
||||
t.Fatalf("raised before the maximum: %+v", early)
|
||||
}
|
||||
}
|
||||
|
||||
// The batch held for a group survives a restart: the hold is in the record.
|
||||
func TestAHeldBatchSurvivesARestart(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
groupStates(t, "feat/g", map[string]string{"novox/one": "ready", "novox/two": "ready"})
|
||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/g"), t0)
|
||||
reopened, err := open.inventory.Batches(t.Context())
|
||||
if err != nil || len(reopened) != 1 || len(reopened[0].Delivery.Batch.Holds) != 1 {
|
||||
t.Fatalf("the record of the held batch: %+v %v", reopened, err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(time.Hour))
|
||||
if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 1 || bs[0].State != inventory.PlanAssembling {
|
||||
t.Fatalf("after an hour: %d walk(s), batches %+v", len(ws), bs)
|
||||
}
|
||||
}
|
||||
|
||||
// A mesh-delivery that does not yet say the group's gate is composed merges member two only after member one is
|
||||
// delivered, so a batch held for it would wait for ever: the merge waits for the window, as before.
|
||||
func TestAnOlderMeshDeliverysGroupIsNotHeldOpen(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
groupStatesFrom(t, "feat/g", map[string]string{"novox/one": "ready", "novox/two": "ready"}, false)
|
||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/g"), t0)
|
||||
if _, bs := walks(t, open); len(bs) != 1 || len(bs[0].Delivery.Batch.Holds) != 0 {
|
||||
t.Fatalf("held for an older mesh-delivery's group: %+v", bs)
|
||||
}
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 {
|
||||
t.Fatalf("at its window: %d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -34,6 +35,10 @@ var openPullStates = map[string]bool{"proposed": true, "checked": true, "ready":
|
||||
// groupMember is one member of a delivery group as mesh-delivery's `groups` says it: `<owner/repo>@<commit> (state)`.
|
||||
type groupMember struct {
|
||||
Repository, State string
|
||||
// Composed says the group's line says its gate is the composed check's (a mesh-delivery that merges a group in
|
||||
// one pass): only such a group is held open for its last member, since an older mesh-delivery merges member two
|
||||
// only after member one is delivered, and a held batch would wait for ever.
|
||||
Composed bool
|
||||
}
|
||||
|
||||
// groupsLine is one group as mesh-delivery's `groups` says it: its id (the head branch name), its members as text and,
|
||||
@@ -43,6 +48,10 @@ type groupsLine struct {
|
||||
ID string `json:"id"`
|
||||
Members []string `json:"members"`
|
||||
Details []memberLine `json:"member_details,omitempty"`
|
||||
// ComposedGate says the group's gate is the composed check's (novox/hq ADR 0403): a member's own check runs the
|
||||
// repository's check alone. Said by a mesh-delivery that merges a group in one pass; absent from an older one, whose
|
||||
// members keep their own gate.
|
||||
ComposedGate bool `json:"composed_gate,omitempty"`
|
||||
}
|
||||
|
||||
// memberLine is one member of a group, as `groups` says it beside its text.
|
||||
@@ -98,7 +107,9 @@ var deliveryGroupOf = func(ctx context.Context, held bool, head string) ([]group
|
||||
}
|
||||
var members []groupMember
|
||||
for _, s := range l.Members {
|
||||
members = append(members, memberOf(s))
|
||||
m := memberOf(s)
|
||||
m.Composed = l.ComposedGate
|
||||
members = append(members, m)
|
||||
}
|
||||
return members, true, nil
|
||||
}
|
||||
@@ -116,75 +127,102 @@ func memberOf(s string) groupMember {
|
||||
}
|
||||
|
||||
// groupWait says whether a merge heard now waits for the window for its delivery group, and why in words; nothing
|
||||
// when it names no head, its head names no group, or every other member is merged or closed. own is the merge's
|
||||
// kind (ownPath).
|
||||
// when it names no head, its head names no group, or every other member is merged or closed.
|
||||
//
|
||||
// A member of another repository whose merge this controller already keeps in a batch not yet cut of the same
|
||||
// kind is merged, whatever mesh-delivery says yet (it marks a member merged once its walk opened): the merge joins
|
||||
// that batch by their head. One kept in a batch of the other kind keeps the merge waiting, whatever mesh-delivery
|
||||
// says (review of #250): kinds never share a batch, so the merge's batch is a new one, and cut at once it would
|
||||
// walk before that member's batch is cut and break the group's order (ADR 0276 decision 5). Waiting, the two
|
||||
// batches are cut in the order their windows close, an own-path one first at one look, as cutBatchesHeld cuts
|
||||
// every pair of kinds.
|
||||
// A member of another repository whose merge this controller already keeps in a batch not yet cut, of either
|
||||
// kind, is merged, whatever mesh-delivery says yet (it marks a member merged once its walk opened): the merge joins
|
||||
// that batch by their head. **Kinds do not part a group** (novox/hq ADR 0418 clause 6): a controller member and a
|
||||
// catalogue member share one batch.
|
||||
//
|
||||
// Asked before the plans are held: a slow answer holds up no walk.
|
||||
func groupWait(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, held, own bool) (string, error) {
|
||||
why, _, err := groupWaitHolding(ctx, inv, m, held)
|
||||
return why, err
|
||||
}
|
||||
|
||||
// groupWaitHolding is groupWait, and whether the wait is for a member still to merge: such a batch is held open
|
||||
// however long it takes (ADR 0418 clause 3). A wait only because mesh-delivery did not say is no hold.
|
||||
func groupWaitHolding(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, held bool) (string, bool, error) {
|
||||
if m.Head == "" {
|
||||
return "", nil
|
||||
return "", false, nil
|
||||
}
|
||||
same, other, err := keptFromHead(ctx, inv, m.Head, own)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(other) > 0 {
|
||||
return "group " + m.Head + " has a member waiting in a batch of the other kind: " + strings.Join(other, ", "), nil
|
||||
}
|
||||
members, isGroup, err := deliveryGroupOf(ctx, held, m.Head)
|
||||
open, isGroup, composed, err := openMembersOf(ctx, inv, held, m.Head, m.Owner+"/"+m.Repo)
|
||||
if err != nil {
|
||||
return fmt.Sprintf("%s did not say whether group %s has a member still to merge (%v): it waits for the "+
|
||||
"window, the safe side", catalogue.DeliverySeat, m.Head, err), nil
|
||||
"window, the safe side", catalogue.DeliverySeat, m.Head, err), false, nil
|
||||
}
|
||||
if !isGroup {
|
||||
return "", nil
|
||||
if !isGroup || len(open) == 0 {
|
||||
return "", false, nil
|
||||
}
|
||||
return "group " + m.Head + " has a member still to merge: " + strings.Join(open, ", "), composed, nil
|
||||
}
|
||||
|
||||
// openMembersOf are the members of the group a head names that are still to merge, in words: those whose pull
|
||||
// request is open and whose merge no batch not yet cut keeps. self is a repository to leave out, the merge being heard.
|
||||
func openMembersOf(ctx context.Context, inv *inventory.Inventory, held bool, head, self string) ([]string, bool, bool, error) {
|
||||
kept, err := keptFromHead(ctx, inv, head)
|
||||
if err != nil {
|
||||
return nil, false, false, err
|
||||
}
|
||||
members, isGroup, err := deliveryGroupOf(ctx, held, head)
|
||||
if err != nil || !isGroup {
|
||||
return nil, isGroup, false, err
|
||||
}
|
||||
var open []string
|
||||
composed := false
|
||||
for _, g := range members {
|
||||
if strings.EqualFold(g.Repository, m.Owner+"/"+m.Repo) || same[strings.ToLower(g.Repository)] ||
|
||||
!openPullStates[g.State] {
|
||||
composed = composed || g.Composed
|
||||
if strings.EqualFold(g.Repository, self) || kept[strings.ToLower(g.Repository)] || !openPullStates[g.State] {
|
||||
continue
|
||||
}
|
||||
open = append(open, g.Repository+" ("+g.State+")")
|
||||
}
|
||||
if len(open) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
return "group " + m.Head + " has a member still to merge: " + strings.Join(open, ", "), nil
|
||||
sort.Strings(open)
|
||||
return open, true, composed, nil
|
||||
}
|
||||
|
||||
// keptFromHead are the merges from a head that batches not yet cut hold: the repositories (lower case) of those in
|
||||
// a batch of the kind own says, and, in words, each of the other kind with its batch.
|
||||
func keptFromHead(ctx context.Context, inv *inventory.Inventory, head string, own bool) (map[string]bool, []string, error) {
|
||||
// keptFromHead are the repositories (lower case) of the merges from a head that batches not yet cut hold, of
|
||||
// either kind.
|
||||
func keptFromHead(ctx context.Context, inv *inventory.Inventory, head string) (map[string]bool, error) {
|
||||
batches, err := inv.Batches(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, err
|
||||
}
|
||||
same := map[string]bool{}
|
||||
var other []string
|
||||
kept := map[string]bool{}
|
||||
for _, b := range batches {
|
||||
merges, err := inv.MergesOf(ctx, b.ID)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, err
|
||||
}
|
||||
for _, k := range merges {
|
||||
if announcedOf(k).Head != head {
|
||||
continue
|
||||
}
|
||||
if b.OwnPath() == own {
|
||||
same[strings.ToLower(k.Repository)] = true
|
||||
} else {
|
||||
other = append(other, fmt.Sprintf("%s@%s in %s", k.Repository, short(k.Commit), b.ID))
|
||||
if announcedOf(k).Head == head {
|
||||
kept[strings.ToLower(k.Repository)] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return same, other, nil
|
||||
return kept, nil
|
||||
}
|
||||
|
||||
// batchOfGroup is the batch not yet cut that holds a merge of another repository from the same head: the group's
|
||||
// batch, whatever kind it is.
|
||||
func batchOfGroup(ctx context.Context, inv *inventory.Inventory, head, repository string) (inventory.Plan, bool, error) {
|
||||
if head == "" {
|
||||
return inventory.Plan{}, false, nil
|
||||
}
|
||||
batches, err := inv.Batches(ctx)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, false, err
|
||||
}
|
||||
for _, b := range batches {
|
||||
merges, err := inv.MergesOf(ctx, b.ID)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, false, err
|
||||
}
|
||||
for _, k := range merges {
|
||||
if announcedOf(k).Head == head && !strings.EqualFold(k.Repository, repository) {
|
||||
return b, true, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return inventory.Plan{}, false, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,449 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
func TestH1AsksAMachineToReportAndSendsItAgain(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
h, told, clock := healingOn(t, open)
|
||||
record, err := open.inventory.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSent(ctx, record.ID, "d2", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := sentNotReported(t, "laptop")
|
||||
|
||||
// First: the report had not reached the mesh, and asking for it brings it.
|
||||
asked := 0
|
||||
h.askReport = func(ctx context.Context, node string) error {
|
||||
asked++
|
||||
if node != "laptop" {
|
||||
t.Errorf("asked %s", node)
|
||||
}
|
||||
_, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeApplied,
|
||||
At: time.Now().Add(time.Second), Declared: "d2"})
|
||||
return err
|
||||
}
|
||||
h.tick(ctx)
|
||||
c, found, err := conditionsFrom.Get(ctx, key)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("the condition is gone after the act — a healer cleared it, not an observation: %v", err)
|
||||
}
|
||||
if asked != 1 || len(c.Tried) != 1 || c.Tried[0].By != "healer H1" || !strings.Contains(c.Tried[0].Outcome, "acted") ||
|
||||
c.Resolver != "healer:H1" {
|
||||
t.Fatalf("after the first act: asked %d, %+v", asked, c)
|
||||
}
|
||||
if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H1" || acts[0].Outcome != inventory.HealActed ||
|
||||
acts[0].Condition != key || acts[0].By != "healer H1" || acts[0].Epoch != 57 {
|
||||
t.Fatalf("the act was not said as healer-acted: %+v", acts)
|
||||
}
|
||||
|
||||
// Within its settle, nothing more: the observation has its turn.
|
||||
clock.pass(time.Minute)
|
||||
h.tick(ctx)
|
||||
if asked != 1 {
|
||||
t.Fatalf("acted again inside the settle: asked %d", asked)
|
||||
}
|
||||
|
||||
// Second: the machine says nothing, so it is sent again.
|
||||
clock.pass(3 * time.Minute)
|
||||
sent := 0
|
||||
h.askReport = func(context.Context, string) error { asked++; return nil }
|
||||
h.sendAgain = func(_ context.Context, node string) error { sent++; return nil }
|
||||
h.tick(ctx)
|
||||
if asked != 2 || sent != 1 {
|
||||
t.Fatalf("the second act: asked %d, sent %d", asked, sent)
|
||||
}
|
||||
c, _, _ = conditionsFrom.Get(ctx, key)
|
||||
if len(c.Tried) != 2 || !strings.Contains(c.Tried[1].Outcome, "sent again") || !strings.Contains(c.Tried[1].Outcome, "act 2 of 2") {
|
||||
t.Fatalf("tried %+v", c.Tried)
|
||||
}
|
||||
|
||||
// The budget is spent: the operator's, urgent, said; and no healer touches it again.
|
||||
clock.pass(4 * time.Minute)
|
||||
h.tick(ctx)
|
||||
c, _, _ = conditionsFrom.Get(ctx, key)
|
||||
if !c.Escalated() || c.Severity != conditions.Urgent || len(c.Tried) != 3 ||
|
||||
!strings.Contains(c.Tried[2].Outcome, "budget of 2") {
|
||||
t.Fatalf("not handed to the operator: %+v", c)
|
||||
}
|
||||
if acts := told.said(); len(acts) != 3 || acts[2].Outcome != inventory.HealEscalated {
|
||||
t.Fatalf("the escalation was not said: %+v", acts)
|
||||
}
|
||||
clock.pass(time.Hour)
|
||||
h.tick(ctx)
|
||||
if asked != 2 || sent != 1 || len(told.said()) != 3 {
|
||||
t.Fatalf("a healer acted on a condition the operator holds: asked %d sent %d said %d", asked, sent, len(told.said()))
|
||||
}
|
||||
heals, err := open.inventory.HealsSince(ctx, time.Now().Add(-time.Hour))
|
||||
if err != nil || len(heals) != 3 || heals[0].Outcome != inventory.HealActed || heals[1].Outcome != inventory.HealActed ||
|
||||
heals[2].Outcome != inventory.HealEscalated || heals[0].Epoch != 57 {
|
||||
t.Fatalf("the heals kept: %+v %v", heals, err)
|
||||
}
|
||||
// And a heal is not a hand act: what S15 counts never sees it.
|
||||
for _, x := range heals {
|
||||
if x.Healer == "" || strings.HasPrefix(x.Act, "hand-act") {
|
||||
t.Errorf("a heal reads as a hand act: %+v", x)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoHealerActsWithoutTheLease(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
h, told, _ := healingOn(t, open)
|
||||
sentNotReported(t, "laptop")
|
||||
h.epoch = func(context.Context) (uint64, error) { return 0, nil }
|
||||
h.tick(ctx)
|
||||
h.epoch = func(context.Context) (uint64, error) { return 0, errors.New("the lease is not held") }
|
||||
h.tick(ctx)
|
||||
h.epoch = func(context.Context) (uint64, error) { return 57, nil }
|
||||
h.acting = func() bool { return false }
|
||||
h.tick(ctx)
|
||||
if len(told.said()) != 0 {
|
||||
t.Fatalf("a healer acted without the lease: %+v", told.said())
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheBrakeStopsEveryHealerAndSaysSo(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
h, told, clock := healingOn(t, open)
|
||||
for i := 0; i < healBrakeLimit; i++ {
|
||||
if _, err := open.inventory.BeginHeal(ctx, inventory.Heal{Healer: "H3", ConditionKey: "seat.x.anchor.silent",
|
||||
Kind: "holder-silent", Act: "asserted", Outcome: inventory.HealActed,
|
||||
At: clock.now().Add(-time.Duration(healBrakeLimit-i) * time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
sentNotReported(t, "laptop")
|
||||
h.tick(ctx) // the fakes fail the test if anything acts
|
||||
braked, found, err := conditionsFrom.Get(ctx, "mesh.healers.braked")
|
||||
if err != nil || !found || braked.Severity != conditions.Urgent || !strings.Contains(braked.Evidence[0].Said, "H3 on seat.x.anchor.silent ×12") {
|
||||
t.Fatalf("the brake was not said: %+v %v", braked, err)
|
||||
}
|
||||
if len(told.said()) != 0 {
|
||||
t.Fatalf("a healer acted under the brake: %+v", told.said())
|
||||
}
|
||||
// Past the hour of the first act, still held: it lets go an hour after the last.
|
||||
clock.pass(30 * time.Minute)
|
||||
h.tick(ctx)
|
||||
if _, held, _ := conditionsFrom.Get(ctx, "mesh.healers.braked"); !held {
|
||||
t.Fatal("the brake let go before an hour had passed since the last act")
|
||||
}
|
||||
clock.pass(31 * time.Minute)
|
||||
asked := 0
|
||||
h.askReport = func(context.Context, string) error { asked++; return nil }
|
||||
h.sendAgain = func(context.Context, string) error { return nil }
|
||||
h.tick(ctx)
|
||||
if _, held, _ := conditionsFrom.Get(ctx, "mesh.healers.braked"); held {
|
||||
t.Fatal("the brake held an hour after the last act")
|
||||
}
|
||||
if asked != 1 {
|
||||
t.Fatalf("the healers did not act again after the brake let go: asked %d", asked)
|
||||
}
|
||||
}
|
||||
|
||||
func TestH2ClosesAPlanAnotherHasTakenOver(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
h, told, _ := healingOn(t, open)
|
||||
created := time.Now().UTC().Add(-time.Hour)
|
||||
older := inventory.Plan{ID: "plan-old", Repository: "novox/app", Branch: "main", Commit: "0ld0ld0", Created: created,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "asked"}}}
|
||||
waiting := inventory.Plan{ID: "plan-other", Repository: "novox/other", Branch: "main", Commit: "07he707", Created: created,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"b"}}, Modules: map[string]*inventory.PlanModule{"b": {State: "asked"}}}
|
||||
newer := inventory.Plan{ID: "plan-new", Repository: "novox/app", Branch: "main", Commit: "new0new", Created: created.Add(time.Minute),
|
||||
State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "built"}}}
|
||||
for _, p := range []*inventory.Plan{&older, &waiting, &newer} {
|
||||
if err := open.inventory.SavePlan(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, id := range []string{"plan-old", "plan-other"} {
|
||||
if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopePlan, ID: id, Kind: "stalled",
|
||||
Severity: conditions.Warning, Summary: id + " is stalled", Source: "S3"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
h.tick(ctx)
|
||||
closed, err := open.inventory.PlanByID(ctx, "plan-old")
|
||||
if err != nil || closed.State != inventory.PlanSuperseded || !strings.Contains(closed.Note, "closed by healer H2") ||
|
||||
!strings.Contains(closed.Note, "plan-new") {
|
||||
t.Fatalf("the superseded plan: %+v %v", closed, err)
|
||||
}
|
||||
if other, _ := open.inventory.PlanByID(ctx, "plan-other"); other.State != inventory.PlanBuilding {
|
||||
t.Fatalf("a plan still waiting was closed: %+v", other)
|
||||
}
|
||||
if c, _, _ := conditionsFrom.Get(ctx, "plan.plan-other.stalled"); len(c.Tried) != 0 || c.Resolver != conditions.ResolverSelf {
|
||||
t.Fatalf("a plan H2 does not repair was touched: %+v", c)
|
||||
}
|
||||
if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H2" || acts[0].Condition != "plan.plan-old.stalled" {
|
||||
t.Fatalf("said %+v", acts)
|
||||
}
|
||||
// Finished: every module built, none rolled out unsent — closed as done.
|
||||
done := inventory.Plan{ID: "plan-done", Repository: "novox/third", Branch: "main", Commit: "d0ned0n", Created: created,
|
||||
State: inventory.PlanRolling, Tier: 0, Tiers: [][]string{{"c"}}, Modules: map[string]*inventory.PlanModule{"c": {State: "built"}}}
|
||||
if err := open.inventory.SavePlan(ctx, &done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state, why, err := planStale(ctx, open.inventory, done); err != nil || state != inventory.PlanDone || !strings.Contains(why, "finished") {
|
||||
t.Fatalf("a finished plan reads %q %q %v", state, why, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestH4ResetsOnlyWhatTheTableMarksResettable(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
h, _, _ := healingOn(t, open)
|
||||
marked := 0
|
||||
for _, c := range broker.MeshConsumers() {
|
||||
if c.Resettable != "" {
|
||||
marked++
|
||||
if c.Stream != broker.EventsStream || c.Name != broker.ControllerName {
|
||||
t.Errorf("%s on %s is marked resettable: only the controller's own events consumer is", c.Name, c.Stream)
|
||||
}
|
||||
}
|
||||
}
|
||||
if marked != 1 {
|
||||
t.Fatalf("%d consumers are marked resettable, want the controller's events consumer alone", marked)
|
||||
}
|
||||
for _, who := range []string{"EVENTS.anchor_shop", "CONTROL.controller"} {
|
||||
if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "behind",
|
||||
Kind: kindConsumerBehind, Severity: conditions.Warning, Summary: who + " is far behind", Source: "D6"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
h.tick(ctx) // the fake reset fails the test if it is called
|
||||
reset := ""
|
||||
h.resetConsumer = func(stream, name string) (string, error) {
|
||||
reset = stream + "." + name
|
||||
return "it was 1500 behind", nil
|
||||
}
|
||||
if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopeBus, ID: "EVENTS.controller",
|
||||
Token: "behind", Kind: kindConsumerBehind, Severity: conditions.Warning, Summary: "far behind", Source: "D6"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.tick(ctx)
|
||||
if reset != "EVENTS.controller" {
|
||||
t.Fatalf("reset %q", reset)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNatsH3AssertsAMissingConsumerAgainAndBrakesAfterItsBudget(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
|
||||
_ = js.Context().DeleteStream(s)
|
||||
}
|
||||
if _, err := assertBusObjects(ctx, open.inventory, js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h, told, clock := healingOn(t, open)
|
||||
h.js = js
|
||||
h.assertObjects = func(ctx context.Context) error { return assertOnSend(ctx, open.inventory, js, " ") }
|
||||
d := &doctor{open: open, js: js}
|
||||
probe := func() {
|
||||
t.Helper()
|
||||
found, err := probeConsumers(ctx, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := conditionsFrom.Reconcile(ctx, "D6", kindedAs(found, "consumer-wrong")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
const key = "bus.NODES.laptop.missing"
|
||||
|
||||
if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
probe()
|
||||
if c, raised, _ := conditionsFrom.Get(ctx, key); !raised || c.Kind != "consumer-lost" {
|
||||
t.Fatalf("the deleted consumer was not raised: %+v", c)
|
||||
}
|
||||
h.tick(ctx)
|
||||
if _, err := js.Context().ConsumerInfo("NODES", "laptop"); err != nil {
|
||||
t.Fatalf("H3 did not make the consumer again: %v", err)
|
||||
}
|
||||
c, still, _ := conditionsFrom.Get(ctx, key)
|
||||
if !still || len(c.Tried) != 1 || c.Tried[0].By != "healer H3" || c.Resolver != "healer:H3" {
|
||||
t.Fatalf("the act is not in the condition, or the healer cleared it: %+v", c)
|
||||
}
|
||||
probe()
|
||||
if _, still, _ := conditionsFrom.Get(ctx, key); still {
|
||||
t.Fatal("the probe's next run did not clear what H3 repaired")
|
||||
}
|
||||
// Kept in the history as the keeper says it, a moment later.
|
||||
var cleared *conditions.Event
|
||||
for wait := time.Now().Add(5 * time.Second); cleared == nil && time.Now().Before(wait); time.Sleep(20 * time.Millisecond) {
|
||||
history, err := conditionsFrom.HistorySince(ctx, time.Now().Add(-time.Minute))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := range history {
|
||||
if history[i].Key == key && history[i].Change == conditions.ChangeCleared {
|
||||
cleared = &history[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
if cleared == nil || !strings.Contains(cleared.Why, "D6 no longer observes it") || len(cleared.Tried) != 1 {
|
||||
t.Fatalf("the clearing is not the probe's, or forgets what was tried: %+v", cleared)
|
||||
}
|
||||
|
||||
// Again within the hour: the budget is one, so after its settle the operator is told, and H3 stops.
|
||||
clock.pass(10 * time.Minute)
|
||||
if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
probe()
|
||||
h.assertObjects = func(context.Context) error { t.Error("H3 acted past its budget"); return nil }
|
||||
h.tick(ctx)
|
||||
c, _, _ = conditionsFrom.Get(ctx, key)
|
||||
if !c.Escalated() || c.Severity != conditions.Urgent || c.Count != 2 {
|
||||
t.Fatalf("the spent budget was not handed to the operator: %+v", c)
|
||||
}
|
||||
acts := told.said()
|
||||
if len(acts) != 2 || acts[0].Outcome != inventory.HealActed || acts[1].Outcome != inventory.HealEscalated {
|
||||
t.Fatalf("said %+v", acts)
|
||||
}
|
||||
clock.pass(2 * time.Hour)
|
||||
h.tick(ctx)
|
||||
if len(told.said()) != 2 {
|
||||
t.Fatal("a healer acted on what the operator holds")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNatsH4ResetsTheControllersEventsConsumerAndItStillDelivers(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
|
||||
_ = js.Context().DeleteStream(s)
|
||||
}
|
||||
if _, err := assertBusObjects(ctx, open.inventory, js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Bound as the controller binds it (link/receive_nats.go), taking one and acknowledging none.
|
||||
events := make(chan *nats.Msg, 64)
|
||||
sub, err := js.Context().ChanSubscribe("", events, nats.Bind(broker.EventsStream, broker.ControllerName))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
followed := broker.ControllerFollows[0]
|
||||
for i := 0; i < consumerFarBehind+200; i++ {
|
||||
if _, err := js.Context().Publish(followed, []byte(`{"n":`+strconv.Itoa(i)+`}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
d := &doctor{open: open, js: js}
|
||||
probe := func() []conditions.Observation {
|
||||
t.Helper()
|
||||
found, err := probeConsumers(ctx, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := conditionsFrom.Reconcile(ctx, "D6", kindedAs(found, "consumer-wrong")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return found
|
||||
}
|
||||
probe()
|
||||
const key = "bus.EVENTS.controller.behind"
|
||||
if c, raised, _ := conditionsFrom.Get(ctx, key); !raised || c.Kind != kindConsumerBehind {
|
||||
t.Fatalf("a consumer %d behind was not raised: %+v", consumerFarBehind+200, c)
|
||||
}
|
||||
h, told, _ := healingOn(t, open)
|
||||
h.resetConsumer = func(stream, name string) (string, error) {
|
||||
before, after, err := js.ResetConsumer(stream, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "it was " + strconv.FormatUint(before.Pending, 10) + " behind; " + strconv.FormatUint(after.Pending, 10) +
|
||||
" pending now", nil
|
||||
}
|
||||
h.tick(ctx)
|
||||
if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H4" || acts[0].Outcome != inventory.HealActed {
|
||||
t.Fatalf("said %+v", acts)
|
||||
}
|
||||
if found := probe(); len(found) != 0 {
|
||||
t.Fatalf("after the reset the probe still finds %+v", found)
|
||||
}
|
||||
if _, still, _ := conditionsFrom.Get(ctx, key); still {
|
||||
t.Fatal("the probe's next run did not clear what H4 repaired")
|
||||
}
|
||||
// What comes next still reaches the controller.
|
||||
for len(events) > 0 {
|
||||
<-events
|
||||
}
|
||||
if _, err := js.Context().Publish(followed, []byte(`{"after":"the reset"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deadline := time.After(10 * time.Second)
|
||||
for {
|
||||
select {
|
||||
case m := <-events:
|
||||
if strings.Contains(string(m.Data), "after") {
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
case <-deadline:
|
||||
t.Fatal("the controller's subscription heard nothing after its consumer was reset: it would be deaf")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNatsAskToReportReachesTheMachine(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(conn.Close)
|
||||
asked, err := conn.SubscribeSync(broker.AskReportSubject("laptop"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := askToReport(t.Context(), conn, "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
msg, err := asked.NextMsg(5 * time.Second)
|
||||
if err != nil || !strings.Contains(string(msg.Data), "healer H1") {
|
||||
t.Fatalf("the machine heard %v, %v", msg, err)
|
||||
}
|
||||
perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindNode, Node: "laptop", PasswordHash: "x"})
|
||||
if err != nil || !slices.Contains(perms.Subscribe, "mesh.node.laptop.ask.report") ||
|
||||
slices.Contains(perms.Subscribe, "mesh.node.anchor.ask.report") {
|
||||
t.Fatalf("a machine's grant for the question: %v %v", perms.Subscribe, err)
|
||||
}
|
||||
}
|
||||
@@ -6,20 +6,14 @@ import (
|
||||
"errors"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// The test generated from the healer registry (novox/hq to-be 45 §7, ADR 0227 rule 7 "how it is
|
||||
@@ -56,9 +50,14 @@ var inducedFailures = map[string]string{
|
||||
|
||||
func TestEveryHealerAnswersAKindTheMeshRaisesWithABudgetABrakeAndItsEvent(t *testing.T) {
|
||||
kinds := raisedKinds()
|
||||
source, err := os.ReadFile("healers_test.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
// The induced failures are tests of the integration level (a bus), in the file beside this one (issue 895).
|
||||
var source []byte
|
||||
for _, name := range []string{"healers_test.go", "healers_integration_test.go"} {
|
||||
more, err := os.ReadFile(name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
source = append(source, more...)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
answered := map[string]string{}
|
||||
@@ -195,446 +194,24 @@ func sentNotReported(t *testing.T, node string) string {
|
||||
// times): the machine is asked to report; a report that names what it was sent is all it takes, and one
|
||||
// that does not — or none — is a send of its current declaration again. Each act kept in `tried` as
|
||||
// `healer H1`, said as healer-acted, counted; twice, then the operator's, urgent; then nothing more.
|
||||
func TestH1AsksAMachineToReportAndSendsItAgain(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
h, told, clock := healingOn(t, open)
|
||||
record, err := open.inventory.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSent(ctx, record.ID, "d2", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := sentNotReported(t, "laptop")
|
||||
|
||||
// First: the report had not reached the mesh, and asking for it brings it.
|
||||
asked := 0
|
||||
h.askReport = func(ctx context.Context, node string) error {
|
||||
asked++
|
||||
if node != "laptop" {
|
||||
t.Errorf("asked %s", node)
|
||||
}
|
||||
_, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeApplied,
|
||||
At: time.Now().Add(time.Second), Declared: "d2"})
|
||||
return err
|
||||
}
|
||||
h.tick(ctx)
|
||||
c, found, err := conditionsFrom.Get(ctx, key)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("the condition is gone after the act — a healer cleared it, not an observation: %v", err)
|
||||
}
|
||||
if asked != 1 || len(c.Tried) != 1 || c.Tried[0].By != "healer H1" || !strings.Contains(c.Tried[0].Outcome, "acted") ||
|
||||
c.Resolver != "healer:H1" {
|
||||
t.Fatalf("after the first act: asked %d, %+v", asked, c)
|
||||
}
|
||||
if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H1" || acts[0].Outcome != inventory.HealActed ||
|
||||
acts[0].Condition != key || acts[0].By != "healer H1" || acts[0].Epoch != 57 {
|
||||
t.Fatalf("the act was not said as healer-acted: %+v", acts)
|
||||
}
|
||||
|
||||
// Within its settle, nothing more: the observation has its turn.
|
||||
clock.pass(time.Minute)
|
||||
h.tick(ctx)
|
||||
if asked != 1 {
|
||||
t.Fatalf("acted again inside the settle: asked %d", asked)
|
||||
}
|
||||
|
||||
// Second: the machine says nothing, so it is sent again.
|
||||
clock.pass(3 * time.Minute)
|
||||
sent := 0
|
||||
h.askReport = func(context.Context, string) error { asked++; return nil }
|
||||
h.sendAgain = func(_ context.Context, node string) error { sent++; return nil }
|
||||
h.tick(ctx)
|
||||
if asked != 2 || sent != 1 {
|
||||
t.Fatalf("the second act: asked %d, sent %d", asked, sent)
|
||||
}
|
||||
c, _, _ = conditionsFrom.Get(ctx, key)
|
||||
if len(c.Tried) != 2 || !strings.Contains(c.Tried[1].Outcome, "sent again") || !strings.Contains(c.Tried[1].Outcome, "act 2 of 2") {
|
||||
t.Fatalf("tried %+v", c.Tried)
|
||||
}
|
||||
|
||||
// The budget is spent: the operator's, urgent, said; and no healer touches it again.
|
||||
clock.pass(4 * time.Minute)
|
||||
h.tick(ctx)
|
||||
c, _, _ = conditionsFrom.Get(ctx, key)
|
||||
if !c.Escalated() || c.Severity != conditions.Urgent || len(c.Tried) != 3 ||
|
||||
!strings.Contains(c.Tried[2].Outcome, "budget of 2") {
|
||||
t.Fatalf("not handed to the operator: %+v", c)
|
||||
}
|
||||
if acts := told.said(); len(acts) != 3 || acts[2].Outcome != inventory.HealEscalated {
|
||||
t.Fatalf("the escalation was not said: %+v", acts)
|
||||
}
|
||||
clock.pass(time.Hour)
|
||||
h.tick(ctx)
|
||||
if asked != 2 || sent != 1 || len(told.said()) != 3 {
|
||||
t.Fatalf("a healer acted on a condition the operator holds: asked %d sent %d said %d", asked, sent, len(told.said()))
|
||||
}
|
||||
heals, err := open.inventory.HealsSince(ctx, time.Now().Add(-time.Hour))
|
||||
if err != nil || len(heals) != 3 || heals[0].Outcome != inventory.HealActed || heals[1].Outcome != inventory.HealActed ||
|
||||
heals[2].Outcome != inventory.HealEscalated || heals[0].Epoch != 57 {
|
||||
t.Fatalf("the heals kept: %+v %v", heals, err)
|
||||
}
|
||||
// And a heal is not a hand act: what S15 counts never sees it.
|
||||
for _, x := range heals {
|
||||
if x.Healer == "" || strings.HasPrefix(x.Act, "hand-act") {
|
||||
t.Errorf("a heal reads as a hand act: %+v", x)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **Only the controller holding the lease heals** (to-be 45 §6): unleased, or standing by, nothing.
|
||||
func TestNoHealerActsWithoutTheLease(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
h, told, _ := healingOn(t, open)
|
||||
sentNotReported(t, "laptop")
|
||||
h.epoch = func(context.Context) (uint64, error) { return 0, nil }
|
||||
h.tick(ctx)
|
||||
h.epoch = func(context.Context) (uint64, error) { return 0, errors.New("the lease is not held") }
|
||||
h.tick(ctx)
|
||||
h.epoch = func(context.Context) (uint64, error) { return 57, nil }
|
||||
h.acting = func() bool { return false }
|
||||
h.tick(ctx)
|
||||
if len(told.said()) != 0 {
|
||||
t.Fatalf("a healer acted without the lease: %+v", told.said())
|
||||
}
|
||||
}
|
||||
|
||||
// **The mesh-wide brake**: a dozen acts in an hour and every healer stops, said urgently, until an hour
|
||||
// after the last.
|
||||
func TestTheBrakeStopsEveryHealerAndSaysSo(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
h, told, clock := healingOn(t, open)
|
||||
for i := 0; i < healBrakeLimit; i++ {
|
||||
if _, err := open.inventory.BeginHeal(ctx, inventory.Heal{Healer: "H3", ConditionKey: "seat.x.anchor.silent",
|
||||
Kind: "holder-silent", Act: "asserted", Outcome: inventory.HealActed,
|
||||
At: clock.now().Add(-time.Duration(healBrakeLimit-i) * time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
sentNotReported(t, "laptop")
|
||||
h.tick(ctx) // the fakes fail the test if anything acts
|
||||
braked, found, err := conditionsFrom.Get(ctx, "mesh.healers.braked")
|
||||
if err != nil || !found || braked.Severity != conditions.Urgent || !strings.Contains(braked.Evidence[0].Said, "H3 on seat.x.anchor.silent ×12") {
|
||||
t.Fatalf("the brake was not said: %+v %v", braked, err)
|
||||
}
|
||||
if len(told.said()) != 0 {
|
||||
t.Fatalf("a healer acted under the brake: %+v", told.said())
|
||||
}
|
||||
// Past the hour of the first act, still held: it lets go an hour after the last.
|
||||
clock.pass(30 * time.Minute)
|
||||
h.tick(ctx)
|
||||
if _, held, _ := conditionsFrom.Get(ctx, "mesh.healers.braked"); !held {
|
||||
t.Fatal("the brake let go before an hour had passed since the last act")
|
||||
}
|
||||
clock.pass(31 * time.Minute)
|
||||
asked := 0
|
||||
h.askReport = func(context.Context, string) error { asked++; return nil }
|
||||
h.sendAgain = func(context.Context, string) error { return nil }
|
||||
h.tick(ctx)
|
||||
if _, held, _ := conditionsFrom.Get(ctx, "mesh.healers.braked"); held {
|
||||
t.Fatal("the brake held an hour after the last act")
|
||||
}
|
||||
if asked != 1 {
|
||||
t.Fatalf("the healers did not act again after the brake let go: asked %d", asked)
|
||||
}
|
||||
}
|
||||
|
||||
// **H2 closes a plan another has taken over**, with its note — and leaves a plan alone whose wait is
|
||||
// still to come: that one is its own signal's, not a healer's.
|
||||
func TestH2ClosesAPlanAnotherHasTakenOver(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
h, told, _ := healingOn(t, open)
|
||||
created := time.Now().UTC().Add(-time.Hour)
|
||||
older := inventory.Plan{ID: "plan-old", Repository: "novox/app", Branch: "main", Commit: "0ld0ld0", Created: created,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "asked"}}}
|
||||
waiting := inventory.Plan{ID: "plan-other", Repository: "novox/other", Branch: "main", Commit: "07he707", Created: created,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"b"}}, Modules: map[string]*inventory.PlanModule{"b": {State: "asked"}}}
|
||||
newer := inventory.Plan{ID: "plan-new", Repository: "novox/app", Branch: "main", Commit: "new0new", Created: created.Add(time.Minute),
|
||||
State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "built"}}}
|
||||
for _, p := range []*inventory.Plan{&older, &waiting, &newer} {
|
||||
if err := open.inventory.SavePlan(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, id := range []string{"plan-old", "plan-other"} {
|
||||
if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopePlan, ID: id, Kind: "stalled",
|
||||
Severity: conditions.Warning, Summary: id + " is stalled", Source: "S3"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
h.tick(ctx)
|
||||
closed, err := open.inventory.PlanByID(ctx, "plan-old")
|
||||
if err != nil || closed.State != inventory.PlanSuperseded || !strings.Contains(closed.Note, "closed by healer H2") ||
|
||||
!strings.Contains(closed.Note, "plan-new") {
|
||||
t.Fatalf("the superseded plan: %+v %v", closed, err)
|
||||
}
|
||||
if other, _ := open.inventory.PlanByID(ctx, "plan-other"); other.State != inventory.PlanBuilding {
|
||||
t.Fatalf("a plan still waiting was closed: %+v", other)
|
||||
}
|
||||
if c, _, _ := conditionsFrom.Get(ctx, "plan.plan-other.stalled"); len(c.Tried) != 0 || c.Resolver != conditions.ResolverSelf {
|
||||
t.Fatalf("a plan H2 does not repair was touched: %+v", c)
|
||||
}
|
||||
if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H2" || acts[0].Condition != "plan.plan-old.stalled" {
|
||||
t.Fatalf("said %+v", acts)
|
||||
}
|
||||
// Finished: every module built, none rolled out unsent — closed as done.
|
||||
done := inventory.Plan{ID: "plan-done", Repository: "novox/third", Branch: "main", Commit: "d0ned0n", Created: created,
|
||||
State: inventory.PlanRolling, Tier: 0, Tiers: [][]string{{"c"}}, Modules: map[string]*inventory.PlanModule{"c": {State: "built"}}}
|
||||
if err := open.inventory.SavePlan(ctx, &done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state, why, err := planStale(ctx, open.inventory, done); err != nil || state != inventory.PlanDone || !strings.Contains(why, "finished") {
|
||||
t.Fatalf("a finished plan reads %q %q %v", state, why, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **H4 only for a consumer the stream table marks resettable**: a module's consumer far behind is said
|
||||
// and left — what a reset drops, nothing would catch up for it.
|
||||
func TestH4ResetsOnlyWhatTheTableMarksResettable(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
h, _, _ := healingOn(t, open)
|
||||
marked := 0
|
||||
for _, c := range broker.MeshConsumers() {
|
||||
if c.Resettable != "" {
|
||||
marked++
|
||||
if c.Stream != broker.EventsStream || c.Name != broker.ControllerName {
|
||||
t.Errorf("%s on %s is marked resettable: only the controller's own events consumer is", c.Name, c.Stream)
|
||||
}
|
||||
}
|
||||
}
|
||||
if marked != 1 {
|
||||
t.Fatalf("%d consumers are marked resettable, want the controller's events consumer alone", marked)
|
||||
}
|
||||
for _, who := range []string{"EVENTS.anchor_shop", "CONTROL.controller"} {
|
||||
if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "behind",
|
||||
Kind: kindConsumerBehind, Severity: conditions.Warning, Summary: who + " is far behind", Source: "D6"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
h.tick(ctx) // the fake reset fails the test if it is called
|
||||
reset := ""
|
||||
h.resetConsumer = func(stream, name string) (string, error) {
|
||||
reset = stream + "." + name
|
||||
return "it was 1500 behind", nil
|
||||
}
|
||||
if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopeBus, ID: "EVENTS.controller",
|
||||
Token: "behind", Kind: kindConsumerBehind, Severity: conditions.Warning, Summary: "far behind", Source: "D6"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.tick(ctx)
|
||||
if reset != "EVENTS.controller" {
|
||||
t.Fatalf("reset %q", reset)
|
||||
}
|
||||
}
|
||||
|
||||
// **H3 against a real bus** (issue 208's note): a consumer the mesh expects, deleted; D6 says so; H3
|
||||
// asserts the bus's objects the way a send does, and D6's next run clears it — the healer never does.
|
||||
// Deleted again within the hour, the budget is spent: the operator's, urgent, and H3 stops.
|
||||
func TestNatsH3AssertsAMissingConsumerAgainAndBrakesAfterItsBudget(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
|
||||
_ = js.Context().DeleteStream(s)
|
||||
}
|
||||
if _, err := assertBusObjects(ctx, open.inventory, js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h, told, clock := healingOn(t, open)
|
||||
h.js = js
|
||||
h.assertObjects = func(ctx context.Context) error { return assertOnSend(ctx, open.inventory, js, " ") }
|
||||
d := &doctor{open: open, js: js}
|
||||
probe := func() {
|
||||
t.Helper()
|
||||
found, err := probeConsumers(ctx, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := conditionsFrom.Reconcile(ctx, "D6", kindedAs(found, "consumer-wrong")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
const key = "bus.NODES.laptop.missing"
|
||||
|
||||
if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
probe()
|
||||
if c, raised, _ := conditionsFrom.Get(ctx, key); !raised || c.Kind != "consumer-lost" {
|
||||
t.Fatalf("the deleted consumer was not raised: %+v", c)
|
||||
}
|
||||
h.tick(ctx)
|
||||
if _, err := js.Context().ConsumerInfo("NODES", "laptop"); err != nil {
|
||||
t.Fatalf("H3 did not make the consumer again: %v", err)
|
||||
}
|
||||
c, still, _ := conditionsFrom.Get(ctx, key)
|
||||
if !still || len(c.Tried) != 1 || c.Tried[0].By != "healer H3" || c.Resolver != "healer:H3" {
|
||||
t.Fatalf("the act is not in the condition, or the healer cleared it: %+v", c)
|
||||
}
|
||||
probe()
|
||||
if _, still, _ := conditionsFrom.Get(ctx, key); still {
|
||||
t.Fatal("the probe's next run did not clear what H3 repaired")
|
||||
}
|
||||
// Kept in the history as the keeper says it, a moment later.
|
||||
var cleared *conditions.Event
|
||||
for wait := time.Now().Add(5 * time.Second); cleared == nil && time.Now().Before(wait); time.Sleep(20 * time.Millisecond) {
|
||||
history, err := conditionsFrom.HistorySince(ctx, time.Now().Add(-time.Minute))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := range history {
|
||||
if history[i].Key == key && history[i].Change == conditions.ChangeCleared {
|
||||
cleared = &history[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
if cleared == nil || !strings.Contains(cleared.Why, "D6 no longer observes it") || len(cleared.Tried) != 1 {
|
||||
t.Fatalf("the clearing is not the probe's, or forgets what was tried: %+v", cleared)
|
||||
}
|
||||
|
||||
// Again within the hour: the budget is one, so after its settle the operator is told, and H3 stops.
|
||||
clock.pass(10 * time.Minute)
|
||||
if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
probe()
|
||||
h.assertObjects = func(context.Context) error { t.Error("H3 acted past its budget"); return nil }
|
||||
h.tick(ctx)
|
||||
c, _, _ = conditionsFrom.Get(ctx, key)
|
||||
if !c.Escalated() || c.Severity != conditions.Urgent || c.Count != 2 {
|
||||
t.Fatalf("the spent budget was not handed to the operator: %+v", c)
|
||||
}
|
||||
acts := told.said()
|
||||
if len(acts) != 2 || acts[0].Outcome != inventory.HealActed || acts[1].Outcome != inventory.HealEscalated {
|
||||
t.Fatalf("said %+v", acts)
|
||||
}
|
||||
clock.pass(2 * time.Hour)
|
||||
h.tick(ctx)
|
||||
if len(told.said()) != 2 {
|
||||
t.Fatal("a healer acted on what the operator holds")
|
||||
}
|
||||
}
|
||||
|
||||
// **H4 against a real bus** (issue 248): the controller's events consumer a long way behind — the week it
|
||||
// once replayed — is re-made from now by the mesh itself, and the controller's bound subscription still
|
||||
// receives what comes next: a reset that left the controller deaf would be the incident.
|
||||
func TestNatsH4ResetsTheControllersEventsConsumerAndItStillDelivers(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
|
||||
_ = js.Context().DeleteStream(s)
|
||||
}
|
||||
if _, err := assertBusObjects(ctx, open.inventory, js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Bound as the controller binds it (link/receive_nats.go), taking one and acknowledging none.
|
||||
events := make(chan *nats.Msg, 64)
|
||||
sub, err := js.Context().ChanSubscribe("", events, nats.Bind(broker.EventsStream, broker.ControllerName))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
followed := broker.ControllerFollows[0]
|
||||
for i := 0; i < consumerFarBehind+200; i++ {
|
||||
if _, err := js.Context().Publish(followed, []byte(`{"n":`+strconv.Itoa(i)+`}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
d := &doctor{open: open, js: js}
|
||||
probe := func() []conditions.Observation {
|
||||
t.Helper()
|
||||
found, err := probeConsumers(ctx, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := conditionsFrom.Reconcile(ctx, "D6", kindedAs(found, "consumer-wrong")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return found
|
||||
}
|
||||
probe()
|
||||
const key = "bus.EVENTS.controller.behind"
|
||||
if c, raised, _ := conditionsFrom.Get(ctx, key); !raised || c.Kind != kindConsumerBehind {
|
||||
t.Fatalf("a consumer %d behind was not raised: %+v", consumerFarBehind+200, c)
|
||||
}
|
||||
h, told, _ := healingOn(t, open)
|
||||
h.resetConsumer = func(stream, name string) (string, error) {
|
||||
before, after, err := js.ResetConsumer(stream, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "it was " + strconv.FormatUint(before.Pending, 10) + " behind; " + strconv.FormatUint(after.Pending, 10) +
|
||||
" pending now", nil
|
||||
}
|
||||
h.tick(ctx)
|
||||
if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H4" || acts[0].Outcome != inventory.HealActed {
|
||||
t.Fatalf("said %+v", acts)
|
||||
}
|
||||
if found := probe(); len(found) != 0 {
|
||||
t.Fatalf("after the reset the probe still finds %+v", found)
|
||||
}
|
||||
if _, still, _ := conditionsFrom.Get(ctx, key); still {
|
||||
t.Fatal("the probe's next run did not clear what H4 repaired")
|
||||
}
|
||||
// What comes next still reaches the controller.
|
||||
for len(events) > 0 {
|
||||
<-events
|
||||
}
|
||||
if _, err := js.Context().Publish(followed, []byte(`{"after":"the reset"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deadline := time.After(10 * time.Second)
|
||||
for {
|
||||
select {
|
||||
case m := <-events:
|
||||
if strings.Contains(string(m.Data), "after") {
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
case <-deadline:
|
||||
t.Fatal("the controller's subscription heard nothing after its consumer was reset: it would be deaf")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **H1's question reaches the machine**, on the subject its grant lets it hear and nothing else.
|
||||
func TestNatsAskToReportReachesTheMachine(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(conn.Close)
|
||||
asked, err := conn.SubscribeSync(broker.AskReportSubject("laptop"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := askToReport(t.Context(), conn, "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
msg, err := asked.NextMsg(5 * time.Second)
|
||||
if err != nil || !strings.Contains(string(msg.Data), "healer H1") {
|
||||
t.Fatalf("the machine heard %v, %v", msg, err)
|
||||
}
|
||||
perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindNode, Node: "laptop", PasswordHash: "x"})
|
||||
if err != nil || !slices.Contains(perms.Subscribe, "mesh.node.laptop.ask.report") ||
|
||||
slices.Contains(perms.Subscribe, "mesh.node.anchor.ask.report") {
|
||||
t.Fatalf("a machine's grant for the question: %v %v", perms.Subscribe, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asked := time.Now().Add(-time.Hour)
|
||||
aResolver(t, open, "c1c1c1c1c1", asked)
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.Assign(ctx, node, "resolver"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// Recorded by a person's choice: the default rolls out since novox/hq ADR 0236.
|
||||
if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{Why: "each machine checked by hand"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
compose := composeForPush(open, gens)
|
||||
d := &recordedDelivery{inv: inv}
|
||||
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if builds, known, err := inv.SentBuilds(ctx, "laptop"); err != nil || !known || builds["resolver"] != "c1c1c1c1c1" {
|
||||
t.Fatalf("the send did not record the build it carried: %v %v %v", builds, known, err)
|
||||
}
|
||||
digestOfLaptop := func() string {
|
||||
sent, err := inv.Outstanding(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return sent
|
||||
}
|
||||
before := digestOfLaptop()
|
||||
|
||||
// The change merges; the policy is record. `push anchor` sends the anchor...
|
||||
aResolver(t, open, "c2c2c2c2c2", asked.Add(time.Minute))
|
||||
d.declared = nil
|
||||
if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// ...and its cascade leaves the laptop, saying so.
|
||||
var said bytes.Buffer
|
||||
d.declared = nil
|
||||
refused, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true}, compose, d, "", nil, &said)
|
||||
if err != nil || len(refused) != 0 {
|
||||
t.Fatalf("the cascade failed: %v %v", refused, err)
|
||||
}
|
||||
if len(d.declared) != 0 {
|
||||
t.Fatalf("the cascade sent %v a build its policy records", d.declared)
|
||||
}
|
||||
if digestOfLaptop() != before {
|
||||
t.Fatal("the laptop's last send moved: it was sent the held build")
|
||||
}
|
||||
for _, want := range []string{"laptop is behind and was not sent", "resolver would move from c1c1c1c1 to c2c2c2c2",
|
||||
"upgrade policy records", "`push laptop` sends it"} {
|
||||
if !strings.Contains(said.String(), want) {
|
||||
t.Errorf("the push did not say %q:\n%s", want, said.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Held and owed something else at once — a peer joined: still not sent, and both said: why it
|
||||
// is held, and that what else it is owed waits with it.
|
||||
aThirdMachine(t, open)
|
||||
d.declared = nil
|
||||
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.declared = nil
|
||||
said.Reset()
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
|
||||
compose, d, "", nil, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(d.declared) != 0 || digestOfLaptop() != before {
|
||||
t.Fatalf("a held machine owed a consequence was sent: %v", d.declared)
|
||||
}
|
||||
if !strings.Contains(said.String(), "resolver would move") || !strings.Contains(said.String(), "anything else it is owed") {
|
||||
t.Fatalf("the push did not say both:\n%s", said.String())
|
||||
}
|
||||
|
||||
// A policy that rolls out, and a build no gate has seen: still held — a cascade does not judge it
|
||||
// (novox/hq ADR 0236).
|
||||
if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{RollOut: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said.Reset()
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
|
||||
compose, d, "", nil, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(d.declared) != 0 || !strings.Contains(said.String(), "has passed no gate yet") {
|
||||
t.Fatalf("a cascade carried a build no gate has seen: %v\n%s", d.declared, said.String())
|
||||
}
|
||||
// Once it passed a gate on some machine, the laptop is a consequence like any other, and sent.
|
||||
if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: "build-c2", Module: "resolver", Commit: "c2c2c2c2c2",
|
||||
Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said.Reset()
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
|
||||
compose, d, "", nil, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before {
|
||||
t.Fatalf("a rolled-out upgrade's machine was not sent: %v\n%s", d.declared, said.String())
|
||||
}
|
||||
if builds, _, _ := inv.SentBuilds(ctx, "laptop"); builds["resolver"] != "c2c2c2c2c2" {
|
||||
t.Fatalf("the new send did not record the new build: %v", builds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestANamedPushStillSendsAConsequenceNothingHolds(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
aResolver(t, open, "c1c1c1c1c1", time.Now().Add(-time.Hour))
|
||||
if _, err := inv.Assign(ctx, "laptop", "resolver"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
compose := composeForPush(open, gens)
|
||||
d := &recordedDelivery{inv: inv}
|
||||
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// `push spare`, the machine just placed: the others' peers change with it.
|
||||
aThirdMachine(t, open)
|
||||
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.declared = nil
|
||||
var said bytes.Buffer
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(d.declared, []string{"anchor", "laptop"}) {
|
||||
t.Fatalf("a consequence nothing holds was not sent: %v\n%s", d.declared, said.String())
|
||||
}
|
||||
if strings.Contains(said.String(), "was not sent") {
|
||||
t.Fatalf("a machine nothing holds was said to be held:\n%s", said.String())
|
||||
}
|
||||
|
||||
// A machine whose last send was not recorded — a declaration sent by hand — is held until named.
|
||||
record, err := inv.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, record.ID, "sent-by-hand", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.declared = nil
|
||||
said.Reset()
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The anchor, the hub, may still be settling from the machine placed above; the laptop is the
|
||||
// question.
|
||||
if slices.Contains(d.declared, "laptop") || !strings.Contains(said.String(), "laptop is behind and was not sent") {
|
||||
t.Fatalf("a machine whose last send is not known was sent: %v\n%s", d.declared, said.String())
|
||||
}
|
||||
}
|
||||
@@ -1,11 +1,9 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -182,172 +180,6 @@ func aThirdMachine(t *testing.T, open *stores) {
|
||||
|
||||
// The issue as it happened, against the real stores: a change merged with the policy `record`, a push
|
||||
// naming the anchor, and the laptop — running the same module — left with what it had, by name.
|
||||
func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asked := time.Now().Add(-time.Hour)
|
||||
aResolver(t, open, "c1c1c1c1c1", asked)
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.Assign(ctx, node, "resolver"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// Recorded by a person's choice: the default rolls out since novox/hq ADR 0236.
|
||||
if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{Why: "each machine checked by hand"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
compose := composeForPush(open, gens)
|
||||
d := &recordedDelivery{inv: inv}
|
||||
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if builds, known, err := inv.SentBuilds(ctx, "laptop"); err != nil || !known || builds["resolver"] != "c1c1c1c1c1" {
|
||||
t.Fatalf("the send did not record the build it carried: %v %v %v", builds, known, err)
|
||||
}
|
||||
digestOfLaptop := func() string {
|
||||
sent, err := inv.Outstanding(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return sent
|
||||
}
|
||||
before := digestOfLaptop()
|
||||
|
||||
// The change merges; the policy is record. `push anchor` sends the anchor...
|
||||
aResolver(t, open, "c2c2c2c2c2", asked.Add(time.Minute))
|
||||
d.declared = nil
|
||||
if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// ...and its cascade leaves the laptop, saying so.
|
||||
var said bytes.Buffer
|
||||
d.declared = nil
|
||||
refused, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true}, compose, d, "", nil, &said)
|
||||
if err != nil || len(refused) != 0 {
|
||||
t.Fatalf("the cascade failed: %v %v", refused, err)
|
||||
}
|
||||
if len(d.declared) != 0 {
|
||||
t.Fatalf("the cascade sent %v a build its policy records", d.declared)
|
||||
}
|
||||
if digestOfLaptop() != before {
|
||||
t.Fatal("the laptop's last send moved: it was sent the held build")
|
||||
}
|
||||
for _, want := range []string{"laptop is behind and was not sent", "resolver would move from c1c1c1c1 to c2c2c2c2",
|
||||
"upgrade policy records", "`push laptop` sends it"} {
|
||||
if !strings.Contains(said.String(), want) {
|
||||
t.Errorf("the push did not say %q:\n%s", want, said.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Held and owed something else at once — a peer joined: still not sent, and both said: why it
|
||||
// is held, and that what else it is owed waits with it.
|
||||
aThirdMachine(t, open)
|
||||
d.declared = nil
|
||||
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.declared = nil
|
||||
said.Reset()
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
|
||||
compose, d, "", nil, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(d.declared) != 0 || digestOfLaptop() != before {
|
||||
t.Fatalf("a held machine owed a consequence was sent: %v", d.declared)
|
||||
}
|
||||
if !strings.Contains(said.String(), "resolver would move") || !strings.Contains(said.String(), "anything else it is owed") {
|
||||
t.Fatalf("the push did not say both:\n%s", said.String())
|
||||
}
|
||||
|
||||
// A policy that rolls out, and a build no gate has seen: still held — a cascade does not judge it
|
||||
// (novox/hq ADR 0236).
|
||||
if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{RollOut: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said.Reset()
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
|
||||
compose, d, "", nil, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(d.declared) != 0 || !strings.Contains(said.String(), "has passed no gate yet") {
|
||||
t.Fatalf("a cascade carried a build no gate has seen: %v\n%s", d.declared, said.String())
|
||||
}
|
||||
// Once it passed a gate on some machine, the laptop is a consequence like any other, and sent.
|
||||
if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: "build-c2", Module: "resolver", Commit: "c2c2c2c2c2",
|
||||
Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said.Reset()
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
|
||||
compose, d, "", nil, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before {
|
||||
t.Fatalf("a rolled-out upgrade's machine was not sent: %v\n%s", d.declared, said.String())
|
||||
}
|
||||
if builds, _, _ := inv.SentBuilds(ctx, "laptop"); builds["resolver"] != "c2c2c2c2c2" {
|
||||
t.Fatalf("the new send did not record the new build: %v", builds)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 057's case is unchanged: a machine whose builds are all current and whose declaration moved
|
||||
// for another reason is sent by a push that names someone else.
|
||||
func TestANamedPushStillSendsAConsequenceNothingHolds(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
aResolver(t, open, "c1c1c1c1c1", time.Now().Add(-time.Hour))
|
||||
if _, err := inv.Assign(ctx, "laptop", "resolver"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
compose := composeForPush(open, gens)
|
||||
d := &recordedDelivery{inv: inv}
|
||||
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// `push spare`, the machine just placed: the others' peers change with it.
|
||||
aThirdMachine(t, open)
|
||||
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.declared = nil
|
||||
var said bytes.Buffer
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(d.declared, []string{"anchor", "laptop"}) {
|
||||
t.Fatalf("a consequence nothing holds was not sent: %v\n%s", d.declared, said.String())
|
||||
}
|
||||
if strings.Contains(said.String(), "was not sent") {
|
||||
t.Fatalf("a machine nothing holds was said to be held:\n%s", said.String())
|
||||
}
|
||||
|
||||
// A machine whose last send was not recorded — a declaration sent by hand — is held until named.
|
||||
record, err := inv.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, record.ID, "sent-by-hand", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.declared = nil
|
||||
said.Reset()
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The anchor, the hub, may still be settling from the machine placed above; the laptop is the
|
||||
// question.
|
||||
if slices.Contains(d.declared, "laptop") || !strings.Contains(said.String(), "laptop is behind and was not sent") {
|
||||
t.Fatalf("a machine whose last send is not known was sent: %v\n%s", d.declared, said.String())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestAReportedHostVersionIsKeptAndReadBack(t *testing.T) {
|
||||
// The machine has sent this since ADR 0141 and the controller's own copy of the report did not
|
||||
// have the field, so it was unmarshalled into nothing. End to end through the store, because the
|
||||
// fault was a field that existed on one side of the wire only.
|
||||
open := aMesh(t)
|
||||
record, err := open.inventory.NodeByName(t.Context(), "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if record.HostVersion != "" {
|
||||
t.Fatalf("a machine that never reported one has host version %q", record.HostVersion)
|
||||
}
|
||||
if err := open.inventory.RecordHostVersion(t.Context(), record.ID, "ced54d4"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again, err := open.inventory.NodeByName(t.Context(), "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again.HostVersion != "ced54d4" {
|
||||
t.Fatalf("the reported host version read back as %q", again.HostVersion)
|
||||
}
|
||||
// An empty report never clears what a machine last said: a bare word that the node is there says
|
||||
// nothing about its host.
|
||||
if err := open.inventory.RecordHostVersion(t.Context(), record.ID, " "); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kept, err := open.inventory.NodeByName(t.Context(), "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if kept.HostVersion != "ced54d4" {
|
||||
t.Fatalf("a report carrying no host version cleared what the machine had said: %q",
|
||||
kept.HostVersion)
|
||||
}
|
||||
}
|
||||
@@ -72,40 +72,3 @@ func TestAMeshWhereNothingReportedAHostStatesNoDisagreement(t *testing.T) {
|
||||
t.Fatalf("a mesh told no host version reported a split: %v", split)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReportedHostVersionIsKeptAndReadBack(t *testing.T) {
|
||||
// The machine has sent this since ADR 0141 and the controller's own copy of the report did not
|
||||
// have the field, so it was unmarshalled into nothing. End to end through the store, because the
|
||||
// fault was a field that existed on one side of the wire only.
|
||||
open := aMesh(t)
|
||||
record, err := open.inventory.NodeByName(t.Context(), "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if record.HostVersion != "" {
|
||||
t.Fatalf("a machine that never reported one has host version %q", record.HostVersion)
|
||||
}
|
||||
if err := open.inventory.RecordHostVersion(t.Context(), record.ID, "ced54d4"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again, err := open.inventory.NodeByName(t.Context(), "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again.HostVersion != "ced54d4" {
|
||||
t.Fatalf("the reported host version read back as %q", again.HostVersion)
|
||||
}
|
||||
// An empty report never clears what a machine last said: a bare word that the node is there says
|
||||
// nothing about its host.
|
||||
if err := open.inventory.RecordHostVersion(t.Context(), record.ID, " "); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kept, err := open.inventory.NodeByName(t.Context(), "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if kept.HostVersion != "ced54d4" {
|
||||
t.Fatalf("a report carrying no host version cleared what the machine had said: %q",
|
||||
kept.HostVersion)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
func TestAnOverflowingConsumerNeverRefusesItsProvidersMachine(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "objects", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "s3-bucket", Scope: catalogue.ScopeMesh,
|
||||
Identity: &catalogue.OfferIdentity{Max: 20, In: "an S3 access key"}}},
|
||||
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}})
|
||||
register(t, open, catalogue.Manifest{Module: "resolver", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}})
|
||||
register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1",
|
||||
Requires: []string{"wildcard-resolution"}})
|
||||
register(t, open, catalogue.Manifest{Module: "photoalbum", Version: "1", Requires: []string{"s3-bucket"}})
|
||||
register(t, open, catalogue.Manifest{Module: "files", Version: "1", Requires: []string{"s3-bucket"}})
|
||||
for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "resolver"},
|
||||
{"laptop", "networkmanager"}, {"laptop", "photoalbum"}, {"laptop", "files"}} {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
|
||||
// The consumer's machine resolves, and says which of its modules no provider will grant.
|
||||
consumer, _, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
over := consumer.Overflowing()
|
||||
if len(over) != 1 || over[0].Module != "photoalbum" || over[0].Provision != "s3-bucket" {
|
||||
t.Fatalf("the consumer's side does not name exactly photoalbum: %+v", over)
|
||||
}
|
||||
|
||||
// The provider's machine composes. Under ADR 0049's one bound this was a refusal naming
|
||||
// networkmanager, and no push to the provider could go through.
|
||||
plan, settings, err := planFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationFor(ctx, open, "anchor", plan, settings)
|
||||
if err != nil {
|
||||
t.Fatalf("one consumer's identity refused its provider's whole machine: %v", err)
|
||||
}
|
||||
if len(declared.withheld) != 1 || declared.withheld[0].Identity != "mesh_laptop_photoalbum" {
|
||||
t.Fatalf("the overflowing consumer is not the one withheld: %+v", declared.withheld)
|
||||
}
|
||||
grants, _, _, err := grantsFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keyless bool
|
||||
for _, g := range grants {
|
||||
keyless = keyless || g.Provision == "wildcard-resolution" && g.From == "networkmanager"
|
||||
}
|
||||
if !keyless {
|
||||
t.Fatalf("networkmanager, 26 characters, is not granted the keyless resolver provision: %+v", grants)
|
||||
}
|
||||
var granted []string
|
||||
for _, c := range declared.Received["objects"]["s3-bucket"] {
|
||||
granted = append(granted, c.From)
|
||||
}
|
||||
if strings.Join(granted, ",") != "files" {
|
||||
t.Fatalf("the object store grants %v; files and only files fit", granted)
|
||||
}
|
||||
said := printed(t, func() error { reportLeftOut("anchor", declared); return nil })
|
||||
if !strings.Contains(said, `photoalbum on laptop requires s3-bucket from anchor`) ||
|
||||
!strings.Contains(said, "left out of anchor's grants") {
|
||||
t.Fatalf("the push does not say whom it leaves out:\n%s", said)
|
||||
}
|
||||
|
||||
// And `status` names it, and does not call the mesh well while it stands.
|
||||
asked, err := theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(asked.overflowing) != 1 || asked.overflowing[0].Module != "photoalbum" {
|
||||
t.Fatalf("status does not carry the overflow: %+v", asked.overflowing)
|
||||
}
|
||||
if asked.well() {
|
||||
t.Fatal("a mesh with a consumer left out of its grants reads as well")
|
||||
}
|
||||
shown := printed(t, func() error { return printStatus(asked) })
|
||||
if !strings.Contains(shown, "identified too long for a provision they require") ||
|
||||
!strings.Contains(shown, "mesh_laptop_photoalbum") {
|
||||
t.Fatalf("status does not say it:\n%s", shown)
|
||||
}
|
||||
body, err := statusAsJSON(asked)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var doc struct {
|
||||
Overflowing []catalogue.Overflow `json:"overflowing"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &doc); err != nil || len(doc.Overflowing) != 1 ||
|
||||
doc.Overflowing[0].Bound.Max != 20 {
|
||||
t.Fatalf("the document does not carry it: %v\n%s", err, body)
|
||||
}
|
||||
}
|
||||
@@ -2,13 +2,10 @@ package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0225, issue 263: a consumer's identity is bounded by the provision it requires, an
|
||||
@@ -53,98 +50,3 @@ func TestModuleCheckRefusesAnIdentityThatOverflowsWhatItRequires(t *testing.T) {
|
||||
// resolver provision, and a second consumer there overflows an object store's access key. The
|
||||
// provider's machine still composes; the overflowing consumer is left out of its grants and named,
|
||||
// by push and by `status`, and the keyless consumer is granted with its long name.
|
||||
func TestAnOverflowingConsumerNeverRefusesItsProvidersMachine(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "objects", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "s3-bucket", Scope: catalogue.ScopeMesh,
|
||||
Identity: &catalogue.OfferIdentity{Max: 20, In: "an S3 access key"}}},
|
||||
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}})
|
||||
register(t, open, catalogue.Manifest{Module: "resolver", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}})
|
||||
register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1",
|
||||
Requires: []string{"wildcard-resolution"}})
|
||||
register(t, open, catalogue.Manifest{Module: "photoalbum", Version: "1", Requires: []string{"s3-bucket"}})
|
||||
register(t, open, catalogue.Manifest{Module: "files", Version: "1", Requires: []string{"s3-bucket"}})
|
||||
for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "resolver"},
|
||||
{"laptop", "networkmanager"}, {"laptop", "photoalbum"}, {"laptop", "files"}} {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
|
||||
// The consumer's machine resolves, and says which of its modules no provider will grant.
|
||||
consumer, _, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
over := consumer.Overflowing()
|
||||
if len(over) != 1 || over[0].Module != "photoalbum" || over[0].Provision != "s3-bucket" {
|
||||
t.Fatalf("the consumer's side does not name exactly photoalbum: %+v", over)
|
||||
}
|
||||
|
||||
// The provider's machine composes. Under ADR 0049's one bound this was a refusal naming
|
||||
// networkmanager, and no push to the provider could go through.
|
||||
plan, settings, err := planFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationFor(ctx, open, "anchor", plan, settings)
|
||||
if err != nil {
|
||||
t.Fatalf("one consumer's identity refused its provider's whole machine: %v", err)
|
||||
}
|
||||
if len(declared.withheld) != 1 || declared.withheld[0].Identity != "mesh_laptop_photoalbum" {
|
||||
t.Fatalf("the overflowing consumer is not the one withheld: %+v", declared.withheld)
|
||||
}
|
||||
grants, _, _, err := grantsFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keyless bool
|
||||
for _, g := range grants {
|
||||
keyless = keyless || g.Provision == "wildcard-resolution" && g.From == "networkmanager"
|
||||
}
|
||||
if !keyless {
|
||||
t.Fatalf("networkmanager, 26 characters, is not granted the keyless resolver provision: %+v", grants)
|
||||
}
|
||||
var granted []string
|
||||
for _, c := range declared.Received["objects"]["s3-bucket"] {
|
||||
granted = append(granted, c.From)
|
||||
}
|
||||
if strings.Join(granted, ",") != "files" {
|
||||
t.Fatalf("the object store grants %v; files and only files fit", granted)
|
||||
}
|
||||
said := printed(t, func() error { reportLeftOut("anchor", declared); return nil })
|
||||
if !strings.Contains(said, `photoalbum on laptop requires s3-bucket from anchor`) ||
|
||||
!strings.Contains(said, "left out of anchor's grants") {
|
||||
t.Fatalf("the push does not say whom it leaves out:\n%s", said)
|
||||
}
|
||||
|
||||
// And `status` names it, and does not call the mesh well while it stands.
|
||||
asked, err := theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(asked.overflowing) != 1 || asked.overflowing[0].Module != "photoalbum" {
|
||||
t.Fatalf("status does not carry the overflow: %+v", asked.overflowing)
|
||||
}
|
||||
if asked.well() {
|
||||
t.Fatal("a mesh with a consumer left out of its grants reads as well")
|
||||
}
|
||||
shown := printed(t, func() error { return printStatus(asked) })
|
||||
if !strings.Contains(shown, "identified too long for a provision they require") ||
|
||||
!strings.Contains(shown, "mesh_laptop_photoalbum") {
|
||||
t.Fatalf("status does not say it:\n%s", shown)
|
||||
}
|
||||
body, err := statusAsJSON(asked)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var doc struct {
|
||||
Overflowing []catalogue.Overflow `json:"overflowing"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &doc); err != nil || len(doc.Overflowing) != 1 ||
|
||||
doc.Overflowing[0].Bound.Max != 20 {
|
||||
t.Fatalf("the document does not carry it: %v\n%s", err, body)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
)
|
||||
|
||||
func core201(t *testing.T) *stores {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
|
||||
record := func(commit string, asked time.Time) inventory.Build {
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: "1"})
|
||||
b := inventory.Build{ID: "build-mesh-controller-" + commit, Module: "mesh-controller", Commit: commit,
|
||||
Repository: "novox/mesh-controller", Manifest: manifest, Asked: asked, At: asked,
|
||||
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + commit + "0000000000000000"}}}
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
before := record("c1aaaaaaaaaa", old)
|
||||
if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: before.ID, Module: "mesh-controller", Commit: before.Commit,
|
||||
Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record("c2bbbbbbbbbb", now)
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{
|
||||
Repository: "novox/mesh-controller", BuiltFrom: "c2bbbbbbbbbb", Head: "c2bbbbbbbbbb", Asked: now}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "anchor", "mesh-controller"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RestoreModule(ctx, before); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wasMoves := machineMoves
|
||||
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
|
||||
modules, err := open.inventory.Assigned(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sent, known, err := open.inventory.SentBuilds(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.moves(node, modules, sent, known, all), nil
|
||||
}
|
||||
t.Cleanup(func() { machineMoves = wasMoves })
|
||||
return open
|
||||
}
|
||||
|
||||
func TestReplay201(t *testing.T) {
|
||||
open := core201(t)
|
||||
ctx := t.Context()
|
||||
k, _ := withConditionsInMemory(t)
|
||||
|
||||
judged := withScope(ctx, sendScope{judged: map[string]bool{"anchor": true}})
|
||||
if kept, err := ungatedIn(judged, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 {
|
||||
t.Fatalf("a plan's send to the machine is made, the rest of it with it: %v %v", kept, err)
|
||||
}
|
||||
keeps, err := recordedKept(keepingRecorded(judged), open, "anchor")
|
||||
if err != nil || keeps["mesh-controller"] != "c2bbbbbbbbbb" {
|
||||
t.Fatalf("the send keeps the newer controller there: %v %v", keeps, err)
|
||||
}
|
||||
shelf := map[string]catalogue.Manifest{"mesh-controller": {Module: "mesh-controller", Version: "old"}}
|
||||
if _, err := keepRecorded(keepingRecorded(judged), open, "anchor", shelf); err != nil || shelf["mesh-controller"].Version != "1" {
|
||||
t.Fatalf("the send composes the kept build's manifest: %+v %v", shelf, err)
|
||||
}
|
||||
key := coreDowngradeObservation(lease.ComponentController, "anchor", "").Key()
|
||||
if c, found, err := k.Get(ctx, key); err != nil || !found || !strings.Contains(c.Summary, "c2bbbbbb") {
|
||||
t.Fatalf("the condition %s is raised, saying the build kept: %+v %v %v", key, c, found, err)
|
||||
}
|
||||
|
||||
putBack := keepingRecorded(withScope(ctx, sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true}))
|
||||
if keeps, err := recordedKept(putBack, open, "anchor"); err != nil || len(keeps) != 0 {
|
||||
t.Fatalf("the gate's own put-back of the controller carries it back: %v %v", keeps, err)
|
||||
}
|
||||
|
||||
settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"})
|
||||
if _, found, _ := k.Get(ctx, key); !found {
|
||||
t.Fatal("a send that kept the newer controller cleared the condition")
|
||||
}
|
||||
settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c1aaaaaaaaaa"})
|
||||
if _, found, _ := k.Get(ctx, key); found {
|
||||
t.Fatal("a send carrying the mesh's record — a person's push — leaves the condition open")
|
||||
}
|
||||
}
|
||||
|
||||
func TestARollbackWhoseSendWasRefusedIsCompletedByTheNextSend(t *testing.T) {
|
||||
open := core201(t)
|
||||
ctx := t.Context()
|
||||
k, _ := withConditionsInMemory(t)
|
||||
if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: "build-mesh-controller-c2bbbbbbbbbb",
|
||||
Module: "mesh-controller", Commit: "c2bbbbbbbbbb", Previous: "c1aaaaaaaaaa", Machines: []string{"anchor"},
|
||||
Verdict: inventory.GateFailed, Rollback: inventory.NotRolledBack}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
healer := withScope(ctx, sendScope{})
|
||||
if kept, err := ungatedIn(healer, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 {
|
||||
t.Fatalf("the healer's send is made: %v %v", kept, err)
|
||||
}
|
||||
if keeps, err := recordedKept(keepingRecorded(healer), open, "anchor"); err != nil || len(keeps) != 0 {
|
||||
t.Fatalf("the failed controller was kept over the build put back: %v %v", keeps, err)
|
||||
}
|
||||
if open, _ := k.Open(ctx); len(open) != 0 {
|
||||
t.Fatalf("a completed put-back raised a condition: %+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAGateJudgesTheOtherMovesBesideAKeptCore(t *testing.T) {
|
||||
open := core201(t)
|
||||
ctx := t.Context()
|
||||
withConditionsInMemory(t)
|
||||
inv := open.inventory
|
||||
old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
|
||||
for _, b := range []struct {
|
||||
commit string
|
||||
at time.Time
|
||||
}{{"a1aaaaaaaaaa", old}, {"a2bbbbbbbbbb", now}} {
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-" + b.commit, Module: "app", Commit: b.commit,
|
||||
Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: b.at, At: b.at,
|
||||
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-" + b.commit}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{
|
||||
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: b.commit, Head: b.commit,
|
||||
Asked: b.at}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "anchor", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-2",
|
||||
map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a1aaaaaaaaaa"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The send as it is made: the controller kept at c2, the app moved to the record.
|
||||
wasSend := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
return names, open.inventory.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-3",
|
||||
map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a2bbbbbbbbbb"})
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = wasSend })
|
||||
|
||||
moves, sent, _, err := gatedSend(ctx, open, "anchor", nil)
|
||||
if err != nil || len(sent) != 1 {
|
||||
t.Fatalf("the gated send is made: %v %v", sent, err)
|
||||
}
|
||||
if len(moves) != 1 || moves[0].Module != "app" || moves[0].To != "a2bbbbbbbbbb" {
|
||||
t.Fatalf("the gate judges the app's move and not the kept controller: %+v", moves)
|
||||
}
|
||||
builds, _, err := inv.SentBuilds(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g := &inventory.PlanGate{Carried: moves}
|
||||
f := gateFacts{commits: judgedCommits(g, nil), sentBuilds: map[string]map[string]string{"anchor": builds}}
|
||||
for _, m := range []string{"app", "mesh-controller"} {
|
||||
if why, superseded := f.supersededOn(m, "anchor"); superseded {
|
||||
t.Fatalf("the gate is superseded by its own send: %s", why)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,14 +1,10 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
)
|
||||
|
||||
@@ -82,178 +78,16 @@ func TestACoreModuleIsKeptOverAnOlderRecord(t *testing.T) {
|
||||
|
||||
// core201 is a mesh whose control node, anchor, was sent the newer controller c2 while the mesh's record of the
|
||||
// controller is the build before it, c1, which passed its gate when it rolled out.
|
||||
func core201(t *testing.T) *stores {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
|
||||
record := func(commit string, asked time.Time) inventory.Build {
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: "1"})
|
||||
b := inventory.Build{ID: "build-mesh-controller-" + commit, Module: "mesh-controller", Commit: commit,
|
||||
Repository: "novox/mesh-controller", Manifest: manifest, Asked: asked, At: asked,
|
||||
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + commit + "0000000000000000"}}}
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
before := record("c1aaaaaaaaaa", old)
|
||||
if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: before.ID, Module: "mesh-controller", Commit: before.Commit,
|
||||
Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record("c2bbbbbbbbbb", now)
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{
|
||||
Repository: "novox/mesh-controller", BuiltFrom: "c2bbbbbbbbbb", Head: "c2bbbbbbbbbb", Asked: now}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "anchor", "mesh-controller"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RestoreModule(ctx, before); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wasMoves := machineMoves
|
||||
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
|
||||
modules, err := open.inventory.Assigned(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sent, known, err := open.inventory.SentBuilds(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.moves(node, modules, sent, known, all), nil
|
||||
}
|
||||
t.Cleanup(func() { machineMoves = wasMoves })
|
||||
return open
|
||||
}
|
||||
|
||||
// TestReplay201 replays the race on a store: the control node runs the newer controller and the mesh's record is
|
||||
// the build before it. A plan's send there — the node-engine's plan of 2026-10-02 — goes, keeps the newer
|
||||
// controller, and says so as a condition; the gate's own put-back carries it back; a send that carries the record,
|
||||
// as a person's push does, clears the condition.
|
||||
func TestReplay201(t *testing.T) {
|
||||
open := core201(t)
|
||||
ctx := t.Context()
|
||||
k, _ := withConditionsInMemory(t)
|
||||
|
||||
judged := withScope(ctx, sendScope{judged: map[string]bool{"anchor": true}})
|
||||
if kept, err := ungatedIn(judged, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 {
|
||||
t.Fatalf("a plan's send to the machine is made, the rest of it with it: %v %v", kept, err)
|
||||
}
|
||||
keeps, err := recordedKept(keepingRecorded(judged), open, "anchor")
|
||||
if err != nil || keeps["mesh-controller"] != "c2bbbbbbbbbb" {
|
||||
t.Fatalf("the send keeps the newer controller there: %v %v", keeps, err)
|
||||
}
|
||||
shelf := map[string]catalogue.Manifest{"mesh-controller": {Module: "mesh-controller", Version: "old"}}
|
||||
if _, err := keepRecorded(keepingRecorded(judged), open, "anchor", shelf); err != nil || shelf["mesh-controller"].Version != "1" {
|
||||
t.Fatalf("the send composes the kept build's manifest: %+v %v", shelf, err)
|
||||
}
|
||||
key := coreDowngradeObservation(lease.ComponentController, "anchor", "").Key()
|
||||
if c, found, err := k.Get(ctx, key); err != nil || !found || !strings.Contains(c.Summary, "c2bbbbbb") {
|
||||
t.Fatalf("the condition %s is raised, saying the build kept: %+v %v %v", key, c, found, err)
|
||||
}
|
||||
|
||||
putBack := keepingRecorded(withScope(ctx, sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true}))
|
||||
if keeps, err := recordedKept(putBack, open, "anchor"); err != nil || len(keeps) != 0 {
|
||||
t.Fatalf("the gate's own put-back of the controller carries it back: %v %v", keeps, err)
|
||||
}
|
||||
|
||||
settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"})
|
||||
if _, found, _ := k.Get(ctx, key); !found {
|
||||
t.Fatal("a send that kept the newer controller cleared the condition")
|
||||
}
|
||||
settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c1aaaaaaaaaa"})
|
||||
if _, found, _ := k.Get(ctx, key); found {
|
||||
t.Fatal("a send carrying the mesh's record — a person's push — leaves the condition open")
|
||||
}
|
||||
}
|
||||
|
||||
// The review of #273: a gate failed the newer controller, put the record back, and the put-back's own send was
|
||||
// refused. The next automatic send — a healer's, which judges nothing — completes the put-back: a build that
|
||||
// failed its gate is never kept.
|
||||
func TestARollbackWhoseSendWasRefusedIsCompletedByTheNextSend(t *testing.T) {
|
||||
open := core201(t)
|
||||
ctx := t.Context()
|
||||
k, _ := withConditionsInMemory(t)
|
||||
if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: "build-mesh-controller-c2bbbbbbbbbb",
|
||||
Module: "mesh-controller", Commit: "c2bbbbbbbbbb", Previous: "c1aaaaaaaaaa", Machines: []string{"anchor"},
|
||||
Verdict: inventory.GateFailed, Rollback: inventory.NotRolledBack}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
healer := withScope(ctx, sendScope{})
|
||||
if kept, err := ungatedIn(healer, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 {
|
||||
t.Fatalf("the healer's send is made: %v %v", kept, err)
|
||||
}
|
||||
if keeps, err := recordedKept(keepingRecorded(healer), open, "anchor"); err != nil || len(keeps) != 0 {
|
||||
t.Fatalf("the failed controller was kept over the build put back: %v %v", keeps, err)
|
||||
}
|
||||
if open, _ := k.Open(ctx); len(open) != 0 {
|
||||
t.Fatalf("a completed put-back raised a condition: %+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
// Review 91 of #273: a gated send to a machine whose controller is kept over an older record leaves the kept
|
||||
// controller out of what its gate judges. Judged as moving back to the record, the gate would read the send's own
|
||||
// kept controller as another build sent there after it (issue 352) and be superseded, judging nothing.
|
||||
func TestAGateJudgesTheOtherMovesBesideAKeptCore(t *testing.T) {
|
||||
open := core201(t)
|
||||
ctx := t.Context()
|
||||
withConditionsInMemory(t)
|
||||
inv := open.inventory
|
||||
old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
|
||||
for _, b := range []struct {
|
||||
commit string
|
||||
at time.Time
|
||||
}{{"a1aaaaaaaaaa", old}, {"a2bbbbbbbbbb", now}} {
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-" + b.commit, Module: "app", Commit: b.commit,
|
||||
Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: b.at, At: b.at,
|
||||
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-" + b.commit}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{
|
||||
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: b.commit, Head: b.commit,
|
||||
Asked: b.at}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "anchor", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-2",
|
||||
map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a1aaaaaaaaaa"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The send as it is made: the controller kept at c2, the app moved to the record.
|
||||
wasSend := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
return names, open.inventory.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-3",
|
||||
map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a2bbbbbbbbbb"})
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = wasSend })
|
||||
|
||||
moves, sent, _, err := gatedSend(ctx, open, "anchor", nil)
|
||||
if err != nil || len(sent) != 1 {
|
||||
t.Fatalf("the gated send is made: %v %v", sent, err)
|
||||
}
|
||||
if len(moves) != 1 || moves[0].Module != "app" || moves[0].To != "a2bbbbbbbbbb" {
|
||||
t.Fatalf("the gate judges the app's move and not the kept controller: %+v", moves)
|
||||
}
|
||||
builds, _, err := inv.SentBuilds(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g := &inventory.PlanGate{Carried: moves}
|
||||
f := gateFacts{commits: judgedCommits(g, nil), sentBuilds: map[string]map[string]string{"anchor": builds}}
|
||||
for _, m := range []string{"app", "mesh-controller"} {
|
||||
if why, superseded := f.supersededOn(m, "anchor"); superseded {
|
||||
t.Fatalf("the gate is superseded by its own send: %s", why)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func TestReplay348(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv, k := open.inventory, conditionsFrom
|
||||
say := func(at time.Time, n *link.NetworkHealth) {
|
||||
t.Helper()
|
||||
if err := stateHealth(ctx, inv, k, "anchor", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
network := func() (conditions.Condition, bool) {
|
||||
t.Helper()
|
||||
list, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range list {
|
||||
if c.Key == "machine.anchor.network" {
|
||||
return c, true
|
||||
}
|
||||
}
|
||||
return conditions.Condition{}, false
|
||||
}
|
||||
|
||||
// 10:58:45 — the second failing look: raised.
|
||||
say(h0, networkSaying(namesRefused(link.StateUnhealthy, 2)))
|
||||
raised, ok := network()
|
||||
if !ok {
|
||||
t.Fatal("the resolver refusing on the control node raised nothing")
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
sent := time.Now().UTC()
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
|
||||
// 10:59:42 — the restarted engine's first statement: one look failed, a second decides.
|
||||
say(h0.Add(time.Minute), networkSaying(namesRefused(link.StateUnknown, 1)))
|
||||
if _, ok := network(); !ok {
|
||||
t.Fatal("a statement that judged nothing yet cleared the condition: the restarted engine's first look " +
|
||||
"said the fault was gone while it still refused")
|
||||
}
|
||||
// 11:00:23 — its second look: unhealthy again, the same raising.
|
||||
say(h0.Add(2*time.Minute), networkSaying(namesRefused(link.StateUnhealthy, 2)))
|
||||
again, ok := network()
|
||||
if !ok || !again.Raised.Equal(raised.Raised) || again.Count != 1 {
|
||||
t.Fatalf("the same raising was not kept: raised %s (first %s), count %d", again.Raised, raised.Raised, again.Count)
|
||||
}
|
||||
|
||||
// The gate on the control node, for a build sent after the fault began.
|
||||
open2, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := gateFacts{judged: true, open: open2}
|
||||
if w := aboutTheMachine("anchor", []string{"mesh-host"}, sent, f); w.whole != "" || len(w.on) != 0 {
|
||||
t.Fatalf("a fault from before the send held the build: %+v", w)
|
||||
}
|
||||
|
||||
// Decided healthy: cleared.
|
||||
say(h0.Add(3*time.Minute), networkSaying(link.NetworkPart{Part: link.PartNames, State: link.StateHealthy, Since: h0}))
|
||||
if c, ok := network(); ok {
|
||||
t.Fatalf("a statement that decides the names healthy left %s open", c.Key)
|
||||
}
|
||||
}
|
||||
|
||||
// namesRefused is the control node's names part as its node-engine said it in the outage: its own
|
||||
// resolver, at its own address, refusing.
|
||||
func namesRefused(state string, streak int) link.NetworkPart {
|
||||
p := link.NetworkPart{Part: link.PartNames, State: state, Since: h0, Streak: streak}
|
||||
if state == link.StateUnhealthy {
|
||||
p.Reason = "1 of its 2 resolvers do not answer as the mesh's do"
|
||||
p.Said = "10.77.0.1 — anchor.internal (IPv4): read udp 10.77.0.1:35244->10.77.0.1:53: read: connection refused"
|
||||
p.Toward = []string{"10.77.0.1"}
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func networkSaying(parts ...link.NetworkPart) *link.NetworkHealth {
|
||||
state := link.StateHealthy
|
||||
for _, p := range parts {
|
||||
switch {
|
||||
case p.State == link.StateUnhealthy:
|
||||
state = link.StateUnhealthy
|
||||
case p.State == link.StateUnknown && state == link.StateHealthy:
|
||||
state = link.StateUnknown
|
||||
}
|
||||
}
|
||||
return &link.NetworkHealth{State: state, Since: h0, Parts: parts}
|
||||
}
|
||||
|
||||
// A fault there at the send, cleared and reopened after it, is not raised since the send; one that cleared
|
||||
// before the send and came back after it is — a send that breaks a recovered machine fails its gate (review
|
||||
// of mesh-controller PR 179, A2). Read through OpenAt, by both of the gate's readings.
|
||||
func TestAFaultThatFlappedAfterTheSendIsNotTheSendsAndOneThatRecoveredBeforeItIs(t *testing.T) {
|
||||
since := h0
|
||||
network := func(first time.Time, gaps ...conditions.Gap) conditions.Condition {
|
||||
raised := since.Add(time.Minute)
|
||||
if len(gaps) > 0 {
|
||||
raised = gaps[len(gaps)-1].Reopened
|
||||
}
|
||||
return conditions.Condition{Key: "machine.anchor.network", Kind: kindMachineNetwork,
|
||||
Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor"},
|
||||
Summary: "anchor's network is not healthy", Source: sourceNetwork, First: first, Gaps: gaps, Raised: raised}
|
||||
}
|
||||
held := func(c conditions.Condition) bool {
|
||||
return aboutTheMachine("anchor", []string{"mesh-controller"}, since, gateFacts{judged: true,
|
||||
open: []conditions.Condition{c}}).whole != ""
|
||||
}
|
||||
// The day's case: raised before the send, cleared 8 s after it, reopened 49 s after it.
|
||||
flapped := network(since.Add(-49*time.Second),
|
||||
conditions.Gap{Cleared: since.Add(8 * time.Second), Reopened: since.Add(49 * time.Second)})
|
||||
if held(flapped) {
|
||||
t.Fatal("a fault there at the send, flapping after it, held the machine")
|
||||
}
|
||||
// Recovered before the send, broken again after it: the send's.
|
||||
recovered := network(since.Add(-time.Hour),
|
||||
conditions.Gap{Cleared: since.Add(-30 * time.Second), Reopened: since.Add(20 * time.Second)})
|
||||
if !held(recovered) {
|
||||
t.Fatal("a machine recovered at the send and broken after it passed the gate")
|
||||
}
|
||||
// An older gap, before the send, and the fault there at the send: not the send's.
|
||||
twice := network(since.Add(-time.Hour),
|
||||
conditions.Gap{Cleared: since.Add(-50 * time.Minute), Reopened: since.Add(-45 * time.Minute)},
|
||||
conditions.Gap{Cleared: since.Add(10 * time.Second), Reopened: since.Add(30 * time.Second)})
|
||||
if held(twice) {
|
||||
t.Fatal("a fault there at the send, with an older gap, held the machine")
|
||||
}
|
||||
// Raised after the send, never cleared: the send's.
|
||||
if !held(network(time.Time{})) {
|
||||
t.Fatal("a fault raised after the send held nothing")
|
||||
}
|
||||
// And a module's own, through judgeHealth.
|
||||
at := since.Add(2 * time.Minute)
|
||||
g := gateFacts{judged: true, now: at, reports: map[string]inventory.Reported{"anchor": {Node: "anchor",
|
||||
Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, engines: map[string]string{},
|
||||
served: map[string]served{}, rolledBack: map[string][]lease.Rollback{},
|
||||
open: []conditions.Condition{{Key: "provider.app.anchor.x.failing", Subject: conditions.Subject{
|
||||
Scope: conditions.ScopeProvider, ID: "app.anchor.x", Machine: "anchor"}, Summary: "failing",
|
||||
First: since.Add(-time.Hour), Raised: since.Add(time.Minute),
|
||||
Gaps: []conditions.Gap{{Cleared: since.Add(5 * time.Second), Reopened: since.Add(time.Minute)}}}}}
|
||||
if _, why := judgeHealth("app", "", catalogue.Manifest{Module: "app"}, "anchor", since, g); strings.HasPrefix(why, "raised since it was sent") {
|
||||
t.Fatalf("a module's own fault there at the send: %s", why)
|
||||
}
|
||||
g.open[0].Gaps[0].Cleared = since.Add(-5 * time.Second)
|
||||
if _, why := judgeHealth("app", "", catalogue.Manifest{Module: "app"}, "anchor", since, g); !strings.HasPrefix(why, "raised since it was sent") {
|
||||
t.Fatalf("a module's own fault, recovered at the send and back after it, was not counted: %s", why)
|
||||
}
|
||||
}
|
||||
|
||||
// Only an undecided part holds a condition that names it; a condition about another part clears, and a
|
||||
// statement unknown as a whole holds every part (review of PR 179, A4). Pure.
|
||||
func TestAnUndecidedPartHoldsOnlyWhatNamesIt(t *testing.T) {
|
||||
f := netFacts(map[string]*inventory.NetworkHealth{
|
||||
"anchor": aNetwork(link.StateUnknown, inventory.NetworkPart{Part: link.PartNames, State: link.StateUnknown, Streak: 1},
|
||||
inventory.NetworkPart{Part: link.PartRoute, State: link.StateHealthy}),
|
||||
"laptop": aNetwork(link.StateHealthy, inventory.NetworkPart{Part: link.PartNames, State: link.StateHealthy}),
|
||||
"spare": aNetwork(link.StateStarting, inventory.NetworkPart{Part: link.PartTunnel, State: link.StateHealthy}),
|
||||
"other": aNetwork(link.StateStarting, inventory.NetworkPart{Part: link.PartTunnel, State: link.StateStarting}),
|
||||
})
|
||||
u := undecidedParts(f)
|
||||
if !u["anchor"][link.PartNames] || u["anchor"][link.PartRoute] || u["laptop"] != nil || !u["spare"]["*"] ||
|
||||
!u["other"][link.PartTunnel] || u["other"]["*"] {
|
||||
t.Fatalf("undecided: %v", u)
|
||||
}
|
||||
about := func(machine, said string, also ...string) conditions.Condition {
|
||||
return conditions.Condition{Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: machine,
|
||||
Machine: machine, Also: also}, Evidence: []conditions.Evidence{{Said: said}}}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
c conditions.Condition
|
||||
held bool
|
||||
}{
|
||||
{about("anchor", "names since 2026-10-09 10:58:45 UTC: 10.77.0.1 — refused"), true},
|
||||
{about("anchor", "route since 2026-10-09 10:58:45 UTC: no default route"), false},
|
||||
{about("laptop", "names since 2026-10-09 10:58:45 UTC: refused"), false},
|
||||
{about("spare", "route since …: no default route"), true},
|
||||
{about("hub", "anchor: names: refused", "anchor"), true},
|
||||
{about("hub", "anchor: tunnel: no handshake", "anchor"), false},
|
||||
} {
|
||||
if got := heldUndecided(c.c, u); got != c.held {
|
||||
t.Errorf("%s %q held %v, want %v", c.c.Subject.Machine, c.c.Evidence[0].Said, got, c.held)
|
||||
}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user