Compare commits
141
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bf82d30865 | ||
|
|
c432b9b5f6 | ||
|
|
7bd61332cb | ||
|
|
fad655532d | ||
|
|
0c675bcdb5 | ||
|
|
9edc5c758d | ||
|
|
483c387b72 | ||
|
|
fe2e5e91b5 | ||
|
|
5dd0e3c056 | ||
|
|
51db0b5273 | ||
|
|
313efa826c | ||
|
|
72fde0ee1a | ||
|
|
5c4fa43f8b | ||
|
|
7bd04342b6 | ||
|
|
c494ed03a7 | ||
|
|
02c61b983c | ||
|
|
3b6b54a501 | ||
|
|
83ce19b9c0 | ||
|
|
7758301444 | ||
|
|
094d3d5bc6 | ||
|
|
e7bcc107c8 | ||
|
|
ebca7816bf | ||
|
|
9bed7d6398 | ||
|
|
0a2e58c070 | ||
|
|
1747e33923 | ||
|
|
9b6acf0ef5 | ||
|
|
45b4ae92bc | ||
|
|
4f5fab81fe | ||
|
|
4632b6a508 | ||
|
|
9d6a12eb53 | ||
|
|
984d85865d | ||
|
|
ed45cc6415 | ||
|
|
ef551fdfb6 | ||
|
|
7493bd8f18 | ||
|
|
8305e4e3d1 | ||
|
|
fbc7bc976b | ||
|
|
f510b46319 | ||
|
|
926fde2fda | ||
|
|
d5cf3b42fe | ||
|
|
47c7877e8d | ||
|
|
68fbd99e89 | ||
|
|
9a37c143e4 | ||
|
|
eb72075104 | ||
|
|
d6b867a87a | ||
|
|
1dc9961c43 | ||
|
|
11ffab887b | ||
|
|
cfbbad8209 | ||
|
|
d2e9dc6159 | ||
|
|
5557a01f06 | ||
|
|
5bddb16514 | ||
|
|
671e350f56 | ||
|
|
f83fcdc15f | ||
|
|
585caa4371 | ||
|
|
f8d08b0637 | ||
|
|
dc62fd0075 | ||
|
|
525b2c1a11 | ||
|
|
c11222026a | ||
|
|
360c318e85 | ||
|
|
f008fab9d8 | ||
|
|
15a9919df5 | ||
|
|
1390836b73 | ||
|
|
dac7e5f7f6 | ||
|
|
c06a2cd972 | ||
|
|
d5f6b67b94 | ||
|
|
12d4025d30 | ||
|
|
1ca4d8ce60 | ||
|
|
a4f93b52a8 | ||
|
|
a2a671adb7 | ||
|
|
412f11b079 | ||
|
|
5d4eb974ab | ||
|
|
6805e2bcb3 | ||
|
|
c640341c50 | ||
|
|
690b75f659 | ||
|
|
d52de218c8 | ||
|
|
6188e6b1da | ||
|
|
798738cc12 | ||
|
|
babfef4f3a | ||
|
|
96b0966ad8 | ||
|
|
4d30b5de13 | ||
|
|
c97942d591 | ||
|
|
9c69b9da17 | ||
|
|
00065dbdaf | ||
|
|
16362e1bbd | ||
|
|
3496b58f66 | ||
|
|
d1cc9b4e20 | ||
|
|
cc11de2513 | ||
|
|
04fcce2fcc | ||
|
|
b9ceeace41 | ||
|
|
2e0a7d1cbd | ||
|
|
3ced96fc6f | ||
|
|
d3b4549611 | ||
|
|
065cfa0d1d | ||
|
|
0e0e143055 | ||
|
|
2887691414 | ||
|
|
e1499d4196 | ||
|
|
96fc4209d3 | ||
|
|
8a53532d89 | ||
|
|
d0161fac52 | ||
|
|
887de9b5f2 | ||
|
|
1560c498b6 | ||
|
|
df6d72aec2 | ||
|
|
a6bc0936e1 | ||
|
|
513ebd0577 | ||
|
|
88e84a4dd7 | ||
|
|
7b5c063cd3 | ||
|
|
abd3078491 | ||
|
|
0e5aed1253 | ||
|
|
6d3523ff10 | ||
|
|
bd35b1c06c | ||
|
|
150f038ff8 | ||
|
|
6c616838a5 | ||
|
|
de55c63e12 | ||
|
|
abe5f7dc17 | ||
|
|
4d1b81b6cb | ||
|
|
9517f590ac | ||
|
|
9cef820117 | ||
|
|
272ca2a578 | ||
|
|
7160d95323 | ||
|
|
9551bc2380 | ||
|
|
cdaba36eca | ||
|
|
eaf5195998 | ||
|
|
a6f831a633 | ||
|
|
a138c045bb | ||
|
|
988e501ccc | ||
|
|
19eefb66c6 | ||
|
|
081d9244d6 | ||
|
|
b55a38ca9f | ||
|
|
747734687e | ||
|
|
9006c82393 | ||
|
|
0c8c9ffae9 | ||
|
|
1c7c385839 | ||
|
|
65ff6159ad | ||
|
|
e6e1e3bc89 | ||
|
|
07e59c535e | ||
|
|
ba97297f66 | ||
|
|
e6b00e2e51 | ||
|
|
fa19a2d718 | ||
|
|
3e5086a2f6 | ||
|
|
ed331eb972 | ||
|
|
be59f29f46 | ||
|
|
5689553406 |
@@ -99,16 +99,22 @@ proxy-image:
|
|||||||
@echo
|
@echo
|
||||||
@docker image inspect $(PROXY_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
@docker image inspect $(PROXY_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||||
|
|
||||||
# The whole gate. Raises a database, runs everything against it, and takes it down again --
|
# The whole check. Raises a database, runs the repository's own check against it -- merge-check.sh,
|
||||||
# including when the tests fail, which is why the teardown is not conditional.
|
# the very script `mesh/repo-check` runs -- and takes the database down again, including when the
|
||||||
|
# check fails, which is why the teardown is not conditional and the script's exit status is the
|
||||||
|
# target's.
|
||||||
#
|
#
|
||||||
# **Packages in parallel, under the race detector, each test on a bus of its own** (internal/testbus).
|
# **It calls the script rather than restating it** (novox/hq issue 431): this target used to list its
|
||||||
# It was one package at a time against one shared bus, because the live tests assert, read and remove
|
# own steps, its formatting step walked vendor/ where the script's does not, and it failed on
|
||||||
# the mesh's own objects by their fixed names, and two packages at once deleted what the other read; the
|
# third-party code no change could fix -- so a developer's check and the gate disagreed, and the steps
|
||||||
# suite was red run as Go runs it and read as noise. A bus per test, of the release the mesh runs, made
|
# after formatting never ran through it. The script is the one list of steps; this target only gives it
|
||||||
# it the same in any order. The timeout bounds a hang to a failure with a stack, never a stalled gate.
|
# a database (MESH_TEST_POSTGRES, exported above).
|
||||||
check: fmt vet postgres
|
#
|
||||||
@go test -race -timeout 15m ./... ; status=$$? ; $(MAKE) postgres-stop ; exit $$status
|
# **Packages in parallel, under the race detector, each test on a bus of its own** (internal/testbus),
|
||||||
|
# as the script runs them: a bus per test, of the release the mesh runs, makes the suite the same in any
|
||||||
|
# order, and the timeout bounds a hang to a failure with a stack, never a stalled gate.
|
||||||
|
check: postgres
|
||||||
|
@sh merge-check.sh ; status=$$? ; docker rm -f $(PG_CONTAINER) >/dev/null 2>&1 || true ; exit $$status
|
||||||
|
|
||||||
# Without a database the store's tests skip rather than fail, so this is the honest subset and not
|
# Without a database the store's tests skip rather than fail, so this is the honest subset and not
|
||||||
# the gate.
|
# the gate.
|
||||||
@@ -118,8 +124,10 @@ test:
|
|||||||
vet:
|
vet:
|
||||||
go vet ./...
|
go vet ./...
|
||||||
|
|
||||||
|
# Quick steps for a developer, not part of `check`. The directories gofmt reads must be the ones
|
||||||
|
# merge-check.sh lists, never `.`, which walks vendor/ (novox/hq issue 431).
|
||||||
fmt:
|
fmt:
|
||||||
@unformatted=$$(gofmt -l . 2>/dev/null) ; \
|
@unformatted=$$(gofmt -l cmd internal examples) ; \
|
||||||
if [ -n "$$unformatted" ] ; then echo "not gofmt'd:" ; echo "$$unformatted" ; exit 1 ; fi
|
if [ -n "$$unformatted" ] ; then echo "not gofmt'd:" ; echo "$$unformatted" ; exit 1 ; fi
|
||||||
|
|
||||||
postgres:
|
postgres:
|
||||||
|
|||||||
@@ -306,6 +306,9 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
|||||||
for _, r := range built.Read {
|
for _, r := range built.Read {
|
||||||
result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref})
|
result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref})
|
||||||
}
|
}
|
||||||
|
for _, s := range built.Sources {
|
||||||
|
result.Sources = append(result.Sources, link.BuildSource{Repository: s.Repository, Ref: s.Ref, Paths: s.Paths})
|
||||||
|
}
|
||||||
say("built", built.Manifest.Module+" from "+short(built.Commit))
|
say("built", built.Manifest.Module+" from "+short(built.Commit))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -179,7 +180,21 @@ func (a *actor) release() {
|
|||||||
// holderOf is this process as the lease's holder.
|
// holderOf is this process as the lease's holder.
|
||||||
func holderOf(instance string) lease.Holder {
|
func holderOf(instance string) lease.Holder {
|
||||||
host, _ := os.Hostname()
|
host, _ := os.Hostname()
|
||||||
return lease.Holder{Instance: instance, Host: host, Build: version}
|
build := runningBuild()
|
||||||
|
if build == "" {
|
||||||
|
build = version
|
||||||
|
}
|
||||||
|
return lease.Holder{Instance: instance, Host: host, Build: build}
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunningBuildVar is where the declaration tells this process which build it is (module.json, the
|
||||||
|
// controller process's env): the version its bundle is delivered as, `${version}` composed by the
|
||||||
|
// catalogue from the bundle's digest. Empty for a process placed by hand.
|
||||||
|
const RunningBuildVar = "MESH_CONTROLLER_VERSION"
|
||||||
|
|
||||||
|
// runningBuild is the version of the build this process is, or empty when the declaration did not say.
|
||||||
|
func runningBuild() string {
|
||||||
|
return strings.TrimSpace(os.Getenv(RunningBuildVar))
|
||||||
}
|
}
|
||||||
|
|
||||||
// serveUnderTheLease takes the lease for the serving controller, waiting while another holds it, and
|
// serveUnderTheLease takes the lease for the serving controller, waiting while another holds it, and
|
||||||
|
|||||||
@@ -0,0 +1,180 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **A refusal the bus said once is counted once** (novox/hq issue 402). The bus refused the controller
|
||||||
|
// one publish at 18:52:59 UTC on 2026-10-10 and never again, yet S9 looked at the one remembered
|
||||||
|
// advisory every half minute for the hour it is kept, and the condition said "observed 15 time(s), last
|
||||||
|
// 13s ago" with a refusal in its evidence every 30 seconds: two of us read it as a refusal going on and
|
||||||
|
// went looking for a missing grant. The condition counts what the bus said, with when it last said it,
|
||||||
|
// never how often the controller looked.
|
||||||
|
func TestARefusalSaidOnceIsCountedOnceHoweverOftenItIsLookedAt(t *testing.T) {
|
||||||
|
refused := time.Date(2026, 10, 10, 18, 52, 59, 0, time.UTC)
|
||||||
|
now := refused.Add(10 * time.Second)
|
||||||
|
store := conditions.NewInMemory()
|
||||||
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||||
|
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
||||||
|
t.Cleanup(func() { k.Close(t.Context()) })
|
||||||
|
|
||||||
|
// The words the live condition bus.controller.refused carried on 2026-10-10.
|
||||||
|
said := "the bus refused the controller: nats: permissions violation: Permissions Violation for " +
|
||||||
|
`Publish to "mesh.seat.mesh-delivery.tool.times"`
|
||||||
|
advisory := link.Advisory{Kind: link.AdvisoryRefused, ID: "controller", Said: said,
|
||||||
|
First: refused, Last: refused, Count: 1}
|
||||||
|
look := func() conditions.Condition {
|
||||||
|
t.Helper()
|
||||||
|
f := &signalFacts{now: now, host: "novox", advisories: []link.Advisory{advisory}}
|
||||||
|
if err := k.Reconcile(t.Context(), "S9", watchAdvisories(f)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
c, found, err := conditions.ReadOne(t.Context(), store, "bus.controller.refused")
|
||||||
|
if err != nil || !found {
|
||||||
|
t.Fatalf("bus.controller.refused: found %v, %v", found, err)
|
||||||
|
}
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
var c conditions.Condition
|
||||||
|
for range 15 {
|
||||||
|
c = look()
|
||||||
|
now = now.Add(30 * time.Second)
|
||||||
|
}
|
||||||
|
if c.Observations != 1 || len(c.Evidence) != 1 {
|
||||||
|
t.Fatalf("one refusal, looked at 15 times, reads as observed %d time(s) with %d evidence lines: %+v",
|
||||||
|
c.Observations, len(c.Evidence), c.Evidence)
|
||||||
|
}
|
||||||
|
if !c.LastObserved.Equal(refused) || !c.Evidence[0].At.Equal(refused) {
|
||||||
|
t.Fatalf("last observed %s, evidence at %s: the refusal was at %s", c.LastObserved, c.Evidence[0].At, refused)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The bus refuses it three times more between two looks: the condition counts four refusals, says
|
||||||
|
// the newest, and adds one line of evidence for the look that saw them.
|
||||||
|
again := now.Add(-5 * time.Second)
|
||||||
|
advisory.Last, advisory.Count = again, 4
|
||||||
|
c = look()
|
||||||
|
if c.Observations != 4 || len(c.Evidence) != 2 || !c.LastObserved.Equal(again) || !c.Evidence[0].At.Equal(again) {
|
||||||
|
t.Fatalf("four refusals read as observed %d time(s), last %s, evidence %+v", c.Observations, c.LastObserved, c.Evidence)
|
||||||
|
}
|
||||||
|
if !strings.Contains(c.Summary, "(4 times since 18:52 UTC)") {
|
||||||
|
t.Fatalf("summary %q", c.Summary)
|
||||||
|
}
|
||||||
|
now = now.Add(30 * time.Second)
|
||||||
|
if c = look(); c.Observations != 4 || len(c.Evidence) != 2 {
|
||||||
|
t.Fatalf("a look with nothing new counted: observed %d time(s), evidence %+v", c.Observations, c.Evidence)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A condition that looks at the present still counts its looks** (novox/hq issue 402 changes only
|
||||||
|
// what reads a record): a machine silent for three looks was observed three times, and says so.
|
||||||
|
func TestAConditionOfThePresentCountsItsLooks(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 10, 19, 0, 0, 0, time.UTC)
|
||||||
|
store := conditions.NewInMemory()
|
||||||
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||||
|
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
||||||
|
t.Cleanup(func() { k.Close(t.Context()) })
|
||||||
|
var c conditions.Condition
|
||||||
|
for range 3 {
|
||||||
|
var err error
|
||||||
|
if c, err = k.Observe(t.Context(), conditions.Observation{Scope: conditions.ScopeMachine, ID: "ace",
|
||||||
|
Kind: "silent", Machine: "ace", Severity: conditions.Warning, Summary: "ace has not been heard from",
|
||||||
|
Source: "S1"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
now = now.Add(30 * time.Second)
|
||||||
|
}
|
||||||
|
if c.Observations != 3 || len(c.Evidence) != 3 || !c.LastObserved.Equal(now.Add(-30*time.Second)) {
|
||||||
|
t.Fatalf("observed %d time(s), %d evidence lines, last %s", c.Observations, len(c.Evidence), c.LastObserved)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **One key, one watcher** (novox/hq issue 440). A consumer's max-deliveries condition is said from
|
||||||
|
// DEAD_LETTERS while it holds a message the consumer gave up on (issue 330). A max-deliveries advisory
|
||||||
|
// without a token names the same key; read beside it, each look added an observation and a line of
|
||||||
|
// evidence through the dead-letter half, and the two summaries overwrote each other on every look. The
|
||||||
|
// dead-letter row alone says that key, and counts the messages given up on, never the looks.
|
||||||
|
func TestAHeldDeadLetterIsCountedByWhatWasGivenUpNotByTheLooks(t *testing.T) {
|
||||||
|
gaveUp := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
|
||||||
|
now := gaveUp.Add(20 * time.Second)
|
||||||
|
store := conditions.NewInMemory()
|
||||||
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||||
|
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
||||||
|
t.Cleanup(func() { k.Close(t.Context()) })
|
||||||
|
|
||||||
|
const key = "bus.EVENTS.media_sonarr.max-deliveries"
|
||||||
|
held := map[string]int{"EVENTS.media_sonarr": 1}
|
||||||
|
newest := map[string]time.Time{"EVENTS.media_sonarr": gaveUp}
|
||||||
|
advisory := link.Advisory{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.media_sonarr", Stream: "EVENTS",
|
||||||
|
Consumer: "media_sonarr", Said: "sonarr on media handed message 7 over 5 times and gave up on it",
|
||||||
|
First: gaveUp, Last: gaveUp, Count: 1}
|
||||||
|
look := func() conditions.Condition {
|
||||||
|
t.Helper()
|
||||||
|
f := &signalFacts{now: now, host: "novox", advisories: []link.Advisory{advisory},
|
||||||
|
deadLetters: held, deadLettersNewest: newest}
|
||||||
|
if err := k.Reconcile(t.Context(), "S9", watchAdvisories(f)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
c, found, err := conditions.ReadOne(t.Context(), store, key)
|
||||||
|
if err != nil || !found {
|
||||||
|
t.Fatalf("%s: found %v, %v", key, found, err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(c.Summary, "dead-letters verb") {
|
||||||
|
t.Fatalf("the summary is not the dead-letter row's: %q", c.Summary)
|
||||||
|
}
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
var c conditions.Condition
|
||||||
|
for range 5 {
|
||||||
|
c = look()
|
||||||
|
now = now.Add(30 * time.Second)
|
||||||
|
}
|
||||||
|
if c.Observations != 1 || len(c.Evidence) != 1 || !c.LastObserved.Equal(gaveUp) {
|
||||||
|
t.Fatalf("one message given up on, looked at five times, reads as observed %d time(s), last %s, "+
|
||||||
|
"with %d evidence lines: %+v", c.Observations, c.LastObserved, len(c.Evidence), c.Evidence)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The consumer gives up on a second message between two looks: two given up on, one more line.
|
||||||
|
again := now.Add(-10 * time.Second)
|
||||||
|
held["EVENTS.media_sonarr"], newest["EVENTS.media_sonarr"] = 2, again
|
||||||
|
c = look()
|
||||||
|
if c.Observations != 2 || len(c.Evidence) != 2 || !c.LastObserved.Equal(again) {
|
||||||
|
t.Fatalf("two given up on read as observed %d time(s), last %s, evidence %+v", c.Observations,
|
||||||
|
c.LastObserved, c.Evidence)
|
||||||
|
}
|
||||||
|
now = now.Add(30 * time.Second)
|
||||||
|
if c = look(); c.Observations != 2 || len(c.Evidence) != 2 {
|
||||||
|
t.Fatalf("a look with nothing new counted: observed %d time(s), evidence %+v", c.Observations, c.Evidence)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The advisory still says the max-deliveries it alone knows: a message given up on that could not be
|
||||||
|
// kept, under a key of its own (issue 330), which issue 440's change leaves as it was.
|
||||||
|
func TestAMessageThatCouldNotBeKeptIsStillSaidFromTheAdvisory(t *testing.T) {
|
||||||
|
at := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
|
||||||
|
f := &signalFacts{now: at, host: "novox", advisories: []link.Advisory{{Kind: link.AdvisoryMaxDeliveries,
|
||||||
|
ID: "EVENTS.media_sonarr", Stream: "EVENTS", Consumer: "media_sonarr", Token: link.AdvisoryNotKept,
|
||||||
|
Said: "sonarr on media gave up on message 7, and it could not be kept", First: at, Last: at, Count: 1}}}
|
||||||
|
said := watchAdvisories(f)
|
||||||
|
if len(said) != 1 || said[0].Key() != "bus.EVENTS.media_sonarr.not-kept" {
|
||||||
|
t.Fatalf("%+v", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A max-deliveries advisory without a token names a consumer's dead-letter key, which only DEAD_LETTERS
|
||||||
|
// says (novox/hq issues 330 and 440): with nothing held, the advisory alone raises nothing.
|
||||||
|
func TestAMaxDeliveriesAdvisoryAloneRaisesNothing(t *testing.T) {
|
||||||
|
at := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
|
||||||
|
f := &signalFacts{now: at, host: "novox", advisories: []link.Advisory{{Kind: link.AdvisoryMaxDeliveries,
|
||||||
|
ID: "EVENTS.media_sonarr", Stream: "EVENTS", Consumer: "media_sonarr",
|
||||||
|
Said: "sonarr on media handed message 7 over 5 times and gave up on it", First: at, Last: at, Count: 1}}}
|
||||||
|
if said := watchAdvisories(f); len(said) != 0 {
|
||||||
|
t.Fatalf("said %+v", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -119,9 +119,11 @@ func judgedConfined(agent string, h inventory.NodeHealth, had bool, now time.Tim
|
|||||||
}
|
}
|
||||||
|
|
||||||
// searchQuietFor is how long the controller lets an agent account's verdict wait for the node-engine's setuid
|
// searchQuietFor is how long the controller lets an agent account's verdict wait for the node-engine's setuid
|
||||||
// search before that is itself the urgent condition: the engine's bound on one search (link.RootSearchBound, the
|
// search before that is itself the urgent condition: the longest a search is expected to take (link.RootSearchQuiet,
|
||||||
// engine's own value), counted from when this controller first saw it waiting, never from the engine's start.
|
// the engine's own value; novox/hq issue 361 — the search runs to completion, with no bound of its own), counted
|
||||||
const searchQuietFor = link.RootSearchBound
|
// from when this controller first saw it waiting, never from the engine's start. Quiet raises nothing; it never
|
||||||
|
// makes the agent account confined, which only a healthy verdict from a complete, fresh search does.
|
||||||
|
const searchQuietFor = link.RootSearchQuiet
|
||||||
|
|
||||||
// The kinds of an agent account's verdict, for the quiet a search earns.
|
// The kinds of an agent account's verdict, for the quiet a search earns.
|
||||||
const (
|
const (
|
||||||
@@ -207,10 +209,10 @@ func probeAgentAccounts(ctx context.Context, d *doctor) ([]conditions.Observatio
|
|||||||
if confined {
|
if confined {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
// Not judged yet only because the first search since the node-engine started is still running: not the
|
// Not judged yet only because the node-engine's search runs and no complete, fresh one judges: not the
|
||||||
// urgent condition after every restart. The agent is still not confined — ADR 0259's router reads
|
// urgent condition after every restart. The agent is still not confined — ADR 0259's router reads
|
||||||
// agentConfined, not this — and `node show` still says not judged. Loud again once the search fails,
|
// agentConfined, not this — and `node show` still says not judged. Loud again once the search fails,
|
||||||
// runs out its bound, or the statement goes stale.
|
// waits past searchQuietFor, or the statement goes stale.
|
||||||
if quiet {
|
if quiet {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -199,14 +199,20 @@ func TestTheNodeVerbOnlyShows(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// After every node-engine restart its search for setuid programs runs for up to its bound, and the agent account
|
// Until a complete search for setuid programs judges — none since the node-engine's state was kept, or the last
|
||||||
// is not judged until it ends. DA does not raise that as urgent while the search is within its bound, counted from
|
// older than the engine lets one judge — the agent account is not judged, and the search runs to its end with no
|
||||||
// when this controller first saw it waiting — never from the engine's own "since", which a restart resets, so an
|
// bound (novox/hq issue 361). DA does not raise that as urgent within searchQuietFor, counted from when this
|
||||||
// engine restarted in a loop does not keep it quiet. The account is still not confined, and `node show` still
|
// controller first saw it waiting — never from the engine's own "since", which a restart resets, so an engine
|
||||||
// says not judged; a search that failed, or a way to root found, is urgent at once.
|
// restarted in a loop does not keep it quiet. The account is still not confined, and `node show` still says not
|
||||||
|
// judged; a search that failed, or a way to root found, is urgent at once.
|
||||||
func TestASearchStillRunningAfterARestartIsNotUrgent(t *testing.T) {
|
func TestASearchStillRunningAfterARestartIsNotUrgent(t *testing.T) {
|
||||||
if searchQuietFor != rootsearch.Bound {
|
if searchQuietFor != rootsearch.Quiet {
|
||||||
t.Fatalf("the quiet is %s and the node-engine's bound %s: they are one value", searchQuietFor, rootsearch.Bound)
|
t.Fatalf("the quiet is %s and the node-engine's %s: they are one value", searchQuietFor, rootsearch.Quiet)
|
||||||
|
}
|
||||||
|
// A daily search that finishes within the quiet never leaves the account unjudged between two of them.
|
||||||
|
if rootsearch.FreshFor < rootsearch.Every+rootsearch.Quiet {
|
||||||
|
t.Fatalf("a complete search judges for %s, less than a day's search (%s) and the quiet (%s)",
|
||||||
|
rootsearch.FreshFor, rootsearch.Every, rootsearch.Quiet)
|
||||||
}
|
}
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
|
|||||||
+149
-13
@@ -9,7 +9,8 @@ package main
|
|||||||
// actions as options at their levels (Silence acknowledges; Release, Stop, Start and Restart approve),
|
// actions as options at their levels (Silence acknowledges; Release, Stop, Start and Restart approve),
|
||||||
// answered by the operator, expiring after a day (a week when every option only acknowledges). A
|
// answered by the operator, expiring after a day (a week when every option only acknowledges). A
|
||||||
// condition that clears, is silenced, or changes its answers has its ask cancelled; an ask that expired
|
// condition that clears, is silenced, or changes its answers has its ask cancelled; an ask that expired
|
||||||
// unanswered is asked again while the condition lasts. Each ask is kept in the controller's bucket
|
// unanswered is asked again while the condition lasts; one the router refused is asked again after a wait
|
||||||
|
// that grows with each refusal in a row, at most half an hour, so a refusal is never believed longer. Each ask is kept in the controller's bucket
|
||||||
// `asked`, so a restart neither asks twice nor forgets.
|
// `asked`, so a restart neither asks twice nor forgets.
|
||||||
// - **On a warrant**, heard on the seat's event under the controller's own name (which only the router may
|
// - **On a warrant**, heard on the seat's event under the controller's own name (which only the router may
|
||||||
// say), the controller acts once per ask: only for an ask it holds, only for the option it offered at
|
// say), the controller acts once per ask: only for an ask it holds, only for the option it offered at
|
||||||
@@ -58,8 +59,29 @@ const (
|
|||||||
// askMostOpen is how many asks the controller holds open at once (the router refuses a fourth): the
|
// askMostOpen is how many asks the controller holds open at once (the router refuses a fourth): the
|
||||||
// most urgent conditions first, then the oldest.
|
// most urgent conditions first, then the oldest.
|
||||||
askMostOpen = asks.MostOpen
|
askMostOpen = asks.MostOpen
|
||||||
|
// askRefusedRetryMost is the longest a refusal is believed without asking again (novox/hq issue 369): the
|
||||||
|
// router refused while its channels had not yet said they could send, the channels could send twenty
|
||||||
|
// minutes later, and the controller repeated that refusal for eleven hours. A refused ask is asked again
|
||||||
|
// after askEvery, then twice as long after each refusal in a row, never longer than this — and at once when
|
||||||
|
// the channels change.
|
||||||
|
askRefusedRetryMost = 30 * time.Minute
|
||||||
|
// askRefusedCounted is how far back refusals in a row are counted for that wait.
|
||||||
|
askRefusedCounted = 6 * time.Hour
|
||||||
|
// askVerdictWait is how long an ask made again after a refusal waits for the router's word before it is
|
||||||
|
// taken as taken: the router refuses an ask as it reads it, so a refusal comes within seconds. Meanwhile
|
||||||
|
// the condition saying it was not delivered stands as it was, neither cleared nor raised again.
|
||||||
|
askVerdictWait = 2 * time.Minute
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// refusedRetryAfter is how long a part refused n times in a row waits before it is asked again.
|
||||||
|
func refusedRetryAfter(n int) time.Duration {
|
||||||
|
wait := askEvery
|
||||||
|
for i := 1; i < n && wait < askRefusedRetryMost; i++ {
|
||||||
|
wait *= 2
|
||||||
|
}
|
||||||
|
return min(wait, askRefusedRetryMost)
|
||||||
|
}
|
||||||
|
|
||||||
// What became of an ask, as the controller keeps it.
|
// What became of an ask, as the controller keeps it.
|
||||||
const (
|
const (
|
||||||
askOpen = "open"
|
askOpen = "open"
|
||||||
@@ -70,8 +92,9 @@ const (
|
|||||||
type asked struct {
|
type asked struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
Condition string `json:"condition"`
|
Condition string `json:"condition"`
|
||||||
// Channels is what the channels were when it was asked (asker.channels): an ask the router refused is not
|
// Channels is what the channels were when it was asked (asker.channels): an ask the router refused is asked
|
||||||
// asked again until the condition's answers or the channels change.
|
// again at once when the condition's answers or the channels change, and otherwise after a wait that grows
|
||||||
|
// with each refusal in a row (refusedRetryAfter, novox/hq issue 369).
|
||||||
Channels string `json:"channels,omitempty"`
|
Channels string `json:"channels,omitempty"`
|
||||||
Ask asks.Ask `json:"ask"`
|
Ask asks.Ask `json:"ask"`
|
||||||
Actions []conditions.Action `json:"actions"`
|
Actions []conditions.Action `json:"actions"`
|
||||||
@@ -91,8 +114,14 @@ type asked struct {
|
|||||||
// Rehearsal is an ask started at the controller's terminal (rehearse.go): about no condition, its answers
|
// Rehearsal is an ask started at the controller's terminal (rehearse.go): about no condition, its answers
|
||||||
// perform nothing, and the reconciling of conditions leaves it alone.
|
// perform nothing, and the reconciling of conditions leaves it alone.
|
||||||
Rehearsal bool `json:"rehearsal,omitempty"`
|
Rehearsal bool `json:"rehearsal,omitempty"`
|
||||||
|
// Proposal is a settings layer proposed through a verb (proposals.go, novox/hq ADR 0277): about no
|
||||||
|
// condition, set on Approve by the serving controller itself, and left alone by the reconciling of conditions.
|
||||||
|
Proposal *settingsProposal `json:"proposal,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ofACondition says an ask is one of a condition's: not a rehearsal, not a proposal.
|
||||||
|
func (r asked) ofACondition() bool { return !r.Rehearsal && r.Proposal == nil }
|
||||||
|
|
||||||
// partKey is an ask's place among what is asked: its condition and its part.
|
// partKey is an ask's place among what is asked: its condition and its part.
|
||||||
func partKey(condition, part string) string { return condition + "#" + part }
|
func partKey(condition, part string) string { return condition + "#" + part }
|
||||||
|
|
||||||
@@ -161,12 +190,19 @@ type asker struct {
|
|||||||
publish func(ctx context.Context, subject string, body []byte, id string) error
|
publish func(ctx context.Context, subject string, body []byte, id string) error
|
||||||
// call performs an action's verb with its arguments, as the controller.
|
// call performs an action's verb with its arguments, as the controller.
|
||||||
call func(ctx context.Context, a conditions.Action, args map[string]string) error
|
call func(ctx context.Context, a conditions.Action, args map[string]string) error
|
||||||
|
// setLayer sets a proposed settings layer on the operator's warrant (novox/hq ADR 0277); nil cannot.
|
||||||
|
setLayer setOnWarrant
|
||||||
// record writes the hand-act log.
|
// record writes the hand-act log.
|
||||||
record func(ctx context.Context, act link.HandAct) error
|
record func(ctx context.Context, act link.HandAct) error
|
||||||
// routerRecord reads the router's record of an ask for a warrant missed; nil reads nothing.
|
// routerRecord reads the router's record of an ask for a warrant missed; nil reads nothing.
|
||||||
routerRecord func(ctx context.Context, id string) (*asks.Warrant, error)
|
routerRecord func(ctx context.Context, id string) (*asks.Warrant, error)
|
||||||
// routerHere says whether a router holds the seat and takes asks under the asker's name; nil is yes.
|
// routerHere says whether a router holds the seat and takes asks under the asker's name; nil is yes.
|
||||||
routerHere func(ctx context.Context) (bool, error)
|
routerHere func(ctx context.Context) (bool, error)
|
||||||
|
// grantHeld says whether the bus holds the controller's grant to ask: the user list the bus's machine was
|
||||||
|
// last sent is the one the mesh composes now (novox/hq issue 353). The grant is composed from the router's
|
||||||
|
// assignment and reaches the bus only when that machine is next pushed, so between `assign` and `push`
|
||||||
|
// the record says a router is here and the bus refuses every ask. why says what to do; nil is yes.
|
||||||
|
grantHeld func(ctx context.Context) (held bool, why string, err error)
|
||||||
// channels is what the channels are now, as a fingerprint: who holds which kind, promising what.
|
// channels is what the channels are now, as a fingerprint: who holds which kind, promising what.
|
||||||
channels func(ctx context.Context) string
|
channels func(ctx context.Context) string
|
||||||
// raise keeps the asker's own condition (sourceAsker): which conditions needing the operator could not be
|
// raise keeps the asker's own condition (sourceAsker): which conditions needing the operator could not be
|
||||||
@@ -176,6 +212,7 @@ type asker struct {
|
|||||||
logf func(string, ...any)
|
logf func(string, ...any)
|
||||||
|
|
||||||
saidNoRouter bool
|
saidNoRouter bool
|
||||||
|
saidNoGrant bool
|
||||||
|
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
nudged chan struct{}
|
nudged chan struct{}
|
||||||
@@ -258,6 +295,30 @@ func (a *asker) reconcile(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
a.saidNoRouter = false
|
a.saidNoRouter = false
|
||||||
}
|
}
|
||||||
|
if a.grantHeld != nil {
|
||||||
|
held, why, err := a.grantHeld(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !held {
|
||||||
|
if !a.saidNoGrant {
|
||||||
|
a.logf("the bus does not hold the controller's grant to ask yet: %s; the operator is asked nothing until it does", why)
|
||||||
|
a.saidNoGrant = true
|
||||||
|
}
|
||||||
|
open, err := a.open(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var unasked []conditions.Condition
|
||||||
|
for _, c := range open {
|
||||||
|
if wants(c, now) {
|
||||||
|
unasked = append(unasked, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return a.sayUnasked(ctx, unasked, "the bus does not hold the controller's grant to ask yet: "+why)
|
||||||
|
}
|
||||||
|
a.saidNoGrant = false
|
||||||
|
}
|
||||||
channels := ""
|
channels := ""
|
||||||
if a.channels != nil {
|
if a.channels != nil {
|
||||||
channels = a.channels(ctx)
|
channels = a.channels(ctx)
|
||||||
@@ -271,17 +332,45 @@ func (a *asker) reconcile(ctx context.Context) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
byCondition := map[string]asked{} // by partKey
|
byCondition := map[string]asked{} // by partKey
|
||||||
|
// What is open and not a condition's — a rehearsal, a proposal — still counts toward what the router holds
|
||||||
|
// open for the controller (askMostOpen); expired unanswered, it is kept so, as the router says it too.
|
||||||
|
otherOpen := 0
|
||||||
for _, r := range all {
|
for _, r := range all {
|
||||||
if r.State == askOpen && !r.Rehearsal {
|
if r.State == askOpen && !r.ofACondition() && !now.Before(r.Ask.Expires) {
|
||||||
|
if _, err := a.store.Change(ctx, r.ID, func(x *asked) bool {
|
||||||
|
if x.State != askOpen || x.Acted != "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
x.State, x.Ended, x.Acted = string(asks.OutcomeExpired), now, "nothing: the ask expired unanswered"
|
||||||
|
return true
|
||||||
|
}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if r.State == askOpen && !r.ofACondition() {
|
||||||
|
otherOpen++
|
||||||
|
}
|
||||||
|
if r.State == askOpen && r.ofACondition() {
|
||||||
k := partKey(r.Condition, r.Part)
|
k := partKey(r.Condition, r.Part)
|
||||||
if prior, held := byCondition[k]; !held || r.Opened.After(prior.Opened) {
|
if prior, held := byCondition[k]; !held || r.Opened.After(prior.Opened) {
|
||||||
byCondition[k] = r
|
byCondition[k] = r
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// A warrant missed while away, read from the router's record.
|
// A warrant missed while away, read from the router's record: for a condition's ask, and for a proposal's or a
|
||||||
|
// rehearsal's alike.
|
||||||
if a.routerRecord != nil {
|
if a.routerRecord != nil {
|
||||||
|
var lookedUp []asked
|
||||||
for _, r := range byCondition {
|
for _, r := range byCondition {
|
||||||
|
lookedUp = append(lookedUp, r)
|
||||||
|
}
|
||||||
|
for _, r := range all {
|
||||||
|
if r.State == askOpen && !r.ofACondition() {
|
||||||
|
lookedUp = append(lookedUp, r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, r := range lookedUp {
|
||||||
if now.Sub(r.Opened) < askCatchUpAfter {
|
if now.Sub(r.Opened) < askCatchUpAfter {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -297,7 +386,7 @@ func (a *asker) reconcile(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
byCondition = map[string]asked{}
|
byCondition = map[string]asked{}
|
||||||
for _, r := range all {
|
for _, r := range all {
|
||||||
if r.State == askOpen && !r.Rehearsal {
|
if r.State == askOpen && r.ofACondition() {
|
||||||
byCondition[partKey(r.Condition, r.Part)] = r
|
byCondition[partKey(r.Condition, r.Part)] = r
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -316,6 +405,32 @@ func (a *asker) reconcile(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Refusals in a row, by part: those since the last ask that was not refused, within askRefusedCounted.
|
||||||
|
// superseded: a part asked since its last refusal, by an ask the router did not refuse — open, answered,
|
||||||
|
// expired or cancelled. Its refusal is history then, never said again (the review of PR 198: an answered
|
||||||
|
// retry brought the refusal back).
|
||||||
|
inRow, superseded := map[string]int{}, map[string]asked{}
|
||||||
|
for k, last := range refused {
|
||||||
|
for _, r := range all {
|
||||||
|
if partKey(r.Condition, r.Part) == k && r.State != string(asks.OutcomeRefused) && r.State != askUnsent &&
|
||||||
|
r.Opened.After(last.Opened) && r.Opened.After(superseded[k].Opened) {
|
||||||
|
superseded[k] = r
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var since time.Time
|
||||||
|
for _, r := range all {
|
||||||
|
if partKey(r.Condition, r.Part) == k && r.State != string(asks.OutcomeRefused) && r.State != askUnsent &&
|
||||||
|
!r.Opened.After(last.Opened) && r.Opened.After(since) {
|
||||||
|
since = r.Opened
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, r := range all {
|
||||||
|
if partKey(r.Condition, r.Part) == k && r.State == string(asks.OutcomeRefused) && r.Opened.After(since) &&
|
||||||
|
now.Sub(r.Opened) < askRefusedCounted {
|
||||||
|
inRow[k]++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
wanted := map[string]bool{}
|
wanted := map[string]bool{}
|
||||||
var unasked []conditions.Condition // refused by the router, and nothing it was refused for changed
|
var unasked []conditions.Condition // refused by the router, and nothing it was refused for changed
|
||||||
var refusedWords []string
|
var refusedWords []string
|
||||||
@@ -330,7 +445,7 @@ func (a *asker) reconcile(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
return open[i].Key < open[j].Key
|
return open[i].Key < open[j].Key
|
||||||
})
|
})
|
||||||
openNow := 0
|
openNow := otherOpen
|
||||||
for _, c := range open {
|
for _, c := range open {
|
||||||
if !wants(c, now) {
|
if !wants(c, now) {
|
||||||
continue
|
continue
|
||||||
@@ -349,15 +464,28 @@ func (a *asker) reconcile(ctx context.Context) error {
|
|||||||
for _, p := range partsOf(c) {
|
for _, p := range partsOf(c) {
|
||||||
key := partKey(c.Key, p.name)
|
key := partKey(c.Key, p.name)
|
||||||
wanted[key] = true
|
wanted[key] = true
|
||||||
if r, was := refused[key]; was && sameAsked(r.Actions, p.actions) && r.Channels == channels {
|
if r, was := refused[key]; was && sameAsked(r.Actions, p.actions) {
|
||||||
if _, held := byCondition[key]; !held {
|
cur, held := byCondition[key]
|
||||||
|
ended := r.Ended
|
||||||
|
if ended.IsZero() {
|
||||||
|
ended = r.Opened
|
||||||
|
}
|
||||||
|
later, asked := superseded[key]
|
||||||
|
waiting := !held && !asked && r.Channels == channels && now.Sub(ended) < refusedRetryAfter(inRow[key])
|
||||||
|
// Asked again now (the wait over), or lately and the router's word not in yet: said as it was until
|
||||||
|
// that word, so the condition neither clears nor is raised again at each try.
|
||||||
|
retrying := !held && !asked && !waiting
|
||||||
|
verdictDue := held && asked && later.ID == cur.ID && now.Sub(cur.Opened) < askVerdictWait
|
||||||
|
if waiting || retrying || verdictDue {
|
||||||
if !saidUnasked {
|
if !saidUnasked {
|
||||||
unasked, saidUnasked = append(unasked, c), true
|
unasked, saidUnasked = append(unasked, c), true
|
||||||
}
|
}
|
||||||
if r.Warrant != nil && r.Warrant.Words != "" {
|
if r.Warrant != nil && r.Warrant.Words != "" {
|
||||||
refusedWords = append(refusedWords, r.Warrant.Words)
|
refusedWords = append(refusedWords, r.Warrant.Words)
|
||||||
}
|
}
|
||||||
continue // refused, and nothing it was refused for has changed
|
}
|
||||||
|
if waiting {
|
||||||
|
continue // refused lately, and nothing it was refused for has changed: asked again after the wait
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if r, done := answered[key]; done && sameAsked(r.Actions, p.actions) {
|
if r, done := answered[key]; done && sameAsked(r.Actions, p.actions) {
|
||||||
@@ -459,7 +587,7 @@ func (a *asker) sayUnasked(ctx context.Context, unasked []conditions.Condition,
|
|||||||
Summary: fmt.Sprintf("%d condition(s) that need the operator could not be asked on any channel: %s; %s",
|
Summary: fmt.Sprintf("%d condition(s) that need the operator could not be asked on any channel: %s; %s",
|
||||||
len(keys), strings.Join(keys, ", "), why),
|
len(keys), strings.Join(keys, ", "), why),
|
||||||
Headline: "Questions for you not delivered",
|
Headline: "Questions for you not delivered",
|
||||||
Explanation: "Needs you: answer them from the mesh MCP server. The mesh could not send you its questions on any channel.",
|
Explanation: "The mesh could not send you its questions on any channel.",
|
||||||
Needs: "answer them from the mesh MCP server, and check why no channel carries them.",
|
Needs: "answer them from the mesh MCP server, and check why no channel carries them.",
|
||||||
Resolved: "The mesh can ask you again"})
|
Resolved: "The mesh can ask you again"})
|
||||||
}
|
}
|
||||||
@@ -681,7 +809,7 @@ func (a *asker) Decided(ctx context.Context, body []byte) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
stillOpen := r.Rehearsal // a rehearsal is about no condition
|
stillOpen := r.Rehearsal || r.Proposal != nil // a rehearsal and a proposal are about no condition
|
||||||
for _, c := range open {
|
for _, c := range open {
|
||||||
stillOpen = stillOpen || c.Key == r.Condition
|
stillOpen = stillOpen || c.Key == r.Condition
|
||||||
}
|
}
|
||||||
@@ -728,15 +856,23 @@ func (a *asker) Decided(ctx context.Context, body []byte) error {
|
|||||||
args["why"] = why
|
args["why"] = why
|
||||||
}
|
}
|
||||||
var acted error
|
var acted error
|
||||||
|
outcome := "done"
|
||||||
switch {
|
switch {
|
||||||
case r.Rehearsal && act.Verb == rehearsalVerb:
|
case r.Rehearsal && act.Verb == rehearsalVerb:
|
||||||
// A rehearsal's answer performs nothing: it is recorded below as the operator's decision.
|
// A rehearsal's answer performs nothing: it is recorded below as the operator's decision.
|
||||||
|
case r.Proposal != nil && act.Verb == proposalVerb:
|
||||||
|
// A proposed settings layer, set by this controller itself on Approve (novox/hq ADR 0277): the act's
|
||||||
|
// digest of the values is held to the record's own values before anything is set.
|
||||||
|
outcome, acted = a.decideProposal(ctx, *r, act, w)
|
||||||
|
if acted == nil && outcome != "" && !strings.HasPrefix(outcome, "nothing") {
|
||||||
|
outcome = "done: " + outcome
|
||||||
|
}
|
||||||
case act.Arguments["silence"] != "":
|
case act.Arguments["silence"] != "":
|
||||||
acted = a.silence(ctx, act.Arguments["silence"], conditions.MaxSilence, byWords(w), why)
|
acted = a.silence(ctx, act.Arguments["silence"], conditions.MaxSilence, byWords(w), why)
|
||||||
default:
|
default:
|
||||||
acted = a.call(ctx, act, args)
|
acted = a.call(ctx, act, args)
|
||||||
}
|
}
|
||||||
ended, outcome := a.now(), "done"
|
ended := a.now()
|
||||||
if acted != nil {
|
if acted != nil {
|
||||||
outcome = "failed: " + acted.Error()
|
outcome = "failed: " + acted.Error()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -377,33 +378,50 @@ func TestAWarrantMissedWhileAwayIsReadFromTheRoutersRecord(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// After review (2026-10-08): a refused ask is not asked again until its answers or the channels change.
|
// After review (2026-10-08): a refused ask is not asked again at once; issue 369: nor is the refusal believed
|
||||||
func TestAnAskTheRouterRefusedWaitsUntilSomethingChanges(t *testing.T) {
|
// for ever — it is asked again after a wait that doubles with each refusal in a row, and at once when the
|
||||||
|
// channels change.
|
||||||
|
func TestAnAskTheRouterRefusedIsAskedAgainAfterAGrowingWait(t *testing.T) {
|
||||||
r := newAskerRig(t)
|
r := newAskerRig(t)
|
||||||
channels := "channel/telegram=telegram@anchor[choice]own:true"
|
channels := "channel/telegram=telegram@anchor[choice]own:true"
|
||||||
r.a.channels = func(context.Context) string { return channels }
|
r.a.channels = func(context.Context) string { return channels }
|
||||||
r.open = []conditions.Condition{heldCondition()}
|
r.open = []conditions.Condition{heldCondition()}
|
||||||
_ = r.a.reconcile(context.Background())
|
refuse := func() {
|
||||||
first := r.asksSent(t)[0]
|
t.Helper()
|
||||||
refusal, _ := json.Marshal(asks.Warrant{Ask: first.ID, Asker: "mesh-controller", Outcome: asks.OutcomeRefused,
|
sent := r.asksSent(t)
|
||||||
Words: "no channel can carry any of its answers now", At: r.now})
|
refusal, _ := json.Marshal(asks.Warrant{Ask: sent[len(sent)-1].ID, Asker: "mesh-controller",
|
||||||
|
Outcome: asks.OutcomeRefused, Words: "no channel can carry any of its answers now", At: r.now})
|
||||||
if err := r.a.Decided(context.Background(), refusal); err != nil {
|
if err := r.a.Decided(context.Background(), refusal); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
_ = r.a.reconcile(context.Background())
|
||||||
|
first := r.asksSent(t)[0]
|
||||||
|
refuse()
|
||||||
if got := r.store[first.ID]; got.State != string(asks.OutcomeRefused) || !strings.Contains(got.Acted, "nothing") {
|
if got := r.store[first.ID]; got.State != string(asks.OutcomeRefused) || !strings.Contains(got.Acted, "nothing") {
|
||||||
t.Fatalf("the refusal was kept as %+v", got)
|
t.Fatalf("the refusal was kept as %+v", got)
|
||||||
}
|
}
|
||||||
for i := 0; i < 3; i++ {
|
// Each wait: nothing asked before it ends, asked once when it has.
|
||||||
r.now = r.now.Add(askEvery)
|
for i, wait := range []time.Duration{time.Minute, 2 * time.Minute, 4 * time.Minute, 8 * time.Minute,
|
||||||
|
16 * time.Minute, 30 * time.Minute, 30 * time.Minute} {
|
||||||
|
before := len(r.asksSent(t))
|
||||||
|
r.now = r.now.Add(wait - 10*time.Second)
|
||||||
_ = r.a.reconcile(context.Background())
|
_ = r.a.reconcile(context.Background())
|
||||||
|
if n := len(r.asksSent(t)); n != before {
|
||||||
|
t.Fatalf("refusal %d: asked again before %s", i+1, wait)
|
||||||
}
|
}
|
||||||
if n := len(r.asksSent(t)); n != 1 {
|
r.now = r.now.Add(10 * time.Second)
|
||||||
t.Fatalf("asked again %d time(s) though nothing changed", n-1)
|
_ = r.a.reconcile(context.Background())
|
||||||
|
if n := len(r.asksSent(t)); n != before+1 {
|
||||||
|
t.Fatalf("refusal %d: not asked again after %s (%d asks)", i+1, wait, n)
|
||||||
}
|
}
|
||||||
|
refuse()
|
||||||
|
}
|
||||||
|
before := len(r.asksSent(t))
|
||||||
channels = "channel/telegram=telegram@anchor[choice,verified-sender]own:true"
|
channels = "channel/telegram=telegram@anchor[choice,verified-sender]own:true"
|
||||||
_ = r.a.reconcile(context.Background())
|
_ = r.a.reconcile(context.Background())
|
||||||
if n := len(r.asksSent(t)); n != 2 {
|
if n := len(r.asksSent(t)); n != before+1 {
|
||||||
t.Errorf("not asked again once the channels changed: %d", n)
|
t.Errorf("not asked again once the channels changed: %d", n-before)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -654,3 +672,139 @@ func TestAnAcknowledgementNeverSharesAnAskWithAnApproval(t *testing.T) {
|
|||||||
t.Errorf("the approval kept through a silence was not performed: %v", r.called)
|
t.Errorf("the approval kept through a silence was not performed: %v", r.called)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// novox/hq issue 353: the controller's grant to ask is composed from the router's assignment and reaches the
|
||||||
|
// bus only when its machine is pushed. Between the two, the bus refuses every ask (measured 2026-10-09, 17:54 to
|
||||||
|
// 17:56 local: seven refusals of mesh.seat.operator-channel.accept.ask.mesh-controller). So nothing is asked
|
||||||
|
// while the bus's user list is behind, it is said once, the conditions that need the operator are raised as
|
||||||
|
// undelivered with what to do, and the asks go out once the bus holds the grant.
|
||||||
|
func TestNothingIsAskedWhileTheBusLacksTheControllersGrant(t *testing.T) {
|
||||||
|
r := newAskerRig(t)
|
||||||
|
var said []string
|
||||||
|
r.a.logf = func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }
|
||||||
|
var raised [][]conditions.Observation
|
||||||
|
r.a.raise = func(_ context.Context, obs []conditions.Observation) error {
|
||||||
|
raised = append(raised, obs)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
held := false
|
||||||
|
r.a.grantHeld = func(context.Context) (bool, string, error) {
|
||||||
|
return held, "the bus's user list on anchor is behind what the mesh composes; `push anchor` carries it", nil
|
||||||
|
}
|
||||||
|
r.open = []conditions.Condition{heldCondition()}
|
||||||
|
_ = r.a.reconcile(context.Background())
|
||||||
|
_ = r.a.reconcile(context.Background())
|
||||||
|
if len(r.asksSent(t)) != 0 {
|
||||||
|
t.Error("asked while the bus lacks the grant")
|
||||||
|
}
|
||||||
|
n := 0
|
||||||
|
for _, s := range said {
|
||||||
|
if strings.Contains(s, "does not hold the controller's grant") {
|
||||||
|
n++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if n != 1 {
|
||||||
|
t.Errorf("said %d times: %q", n, said)
|
||||||
|
}
|
||||||
|
if len(raised) == 0 || len(raised[len(raised)-1]) != 1 ||
|
||||||
|
!strings.Contains(raised[len(raised)-1][0].Summary, "`push anchor` carries it") ||
|
||||||
|
raised[len(raised)-1][0].Kind != "asks-undelivered" {
|
||||||
|
t.Fatalf("not said as a condition with what to do: %+v", raised)
|
||||||
|
}
|
||||||
|
held = true
|
||||||
|
_ = r.a.reconcile(context.Background())
|
||||||
|
if sent := r.asksSent(t); len(sent) != 1 || sent[0].About != heldCondition().Key {
|
||||||
|
t.Errorf("not asked once the bus holds the grant: %+v", sent)
|
||||||
|
}
|
||||||
|
if last := raised[len(raised)-1]; len(last) != 0 {
|
||||||
|
t.Errorf("the undelivered condition was not cleared: %+v", last)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq issue 369: the router refused while its channels had not yet said they could send; they could twenty
|
||||||
|
// minutes later, and "Questions for you not delivered" repeated that refusal for eleven hours, escalating on it.
|
||||||
|
// The ask is made again; while the router's word on it is awaited the condition stands unchanged (neither
|
||||||
|
// cleared nor raised again); once the router took it, the condition clears; a new refusal is said in its own
|
||||||
|
// words.
|
||||||
|
func TestARefusalIsAskedAgainAndTheUndeliveredConditionClearsOnceTaken(t *testing.T) {
|
||||||
|
r := newAskerRig(t)
|
||||||
|
var raised [][]conditions.Observation
|
||||||
|
r.a.raise = func(_ context.Context, obs []conditions.Observation) error {
|
||||||
|
raised = append(raised, obs)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
last := func() []conditions.Observation { return raised[len(raised)-1] }
|
||||||
|
r.a.channels = func(context.Context) string { return "channel/telegram=telegram@anchor[choice]own:true" }
|
||||||
|
r.open = []conditions.Condition{unitsCondition()}
|
||||||
|
refuse := func(words string) {
|
||||||
|
t.Helper()
|
||||||
|
sent := r.asksSent(t)
|
||||||
|
refusal, _ := json.Marshal(asks.Warrant{Ask: sent[len(sent)-1].ID, Asker: "mesh-controller",
|
||||||
|
Outcome: asks.OutcomeRefused, Words: words, At: r.now})
|
||||||
|
if err := r.a.Decided(context.Background(), refusal); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ = r.a.reconcile(context.Background())
|
||||||
|
refuse("no channel can carry any of its answers now: telegram: telegram has not said whether it can send")
|
||||||
|
_ = r.a.reconcile(context.Background())
|
||||||
|
if got := last(); len(got) != 1 || !strings.Contains(got[0].Summary, "telegram has not said") {
|
||||||
|
t.Fatalf("the refusal, said as %+v", got)
|
||||||
|
}
|
||||||
|
// The wait ends: asked again; until the router's word on it is in, the condition stands as it was.
|
||||||
|
r.now = r.now.Add(askEvery)
|
||||||
|
_ = r.a.reconcile(context.Background())
|
||||||
|
if n := len(r.asksSent(t)); n != 2 {
|
||||||
|
t.Fatalf("not asked again: %d asks", n)
|
||||||
|
}
|
||||||
|
if got := last(); len(got) != 1 {
|
||||||
|
t.Fatalf("cleared while the router's word on the new ask was awaited: the condition would flap")
|
||||||
|
}
|
||||||
|
r.now = r.now.Add(askVerdictWait / 2)
|
||||||
|
_ = r.a.reconcile(context.Background())
|
||||||
|
if got := last(); len(got) != 1 {
|
||||||
|
t.Fatalf("cleared inside the verdict wait: the condition would flap")
|
||||||
|
}
|
||||||
|
// Refused again, now for a reason of today: said in those words.
|
||||||
|
refuse("no channel can carry any of its answers now: telegram: no account is linked")
|
||||||
|
_ = r.a.reconcile(context.Background())
|
||||||
|
if got := last(); len(got) != 1 || !strings.Contains(got[0].Summary, "no account is linked") ||
|
||||||
|
strings.Contains(got[0].Summary, "has not said") {
|
||||||
|
t.Fatalf("an old refusal's words repeated: %+v", got)
|
||||||
|
}
|
||||||
|
// Asked again after the doubled wait, and taken: no refusal comes, and the condition clears.
|
||||||
|
r.now = r.now.Add(2 * askEvery)
|
||||||
|
_ = r.a.reconcile(context.Background())
|
||||||
|
if n := len(r.asksSent(t)); n != 3 {
|
||||||
|
t.Fatalf("not asked again after the doubled wait: %d asks", n)
|
||||||
|
}
|
||||||
|
r.now = r.now.Add(askVerdictWait)
|
||||||
|
_ = r.a.reconcile(context.Background())
|
||||||
|
if got := last(); len(got) != 0 {
|
||||||
|
t.Fatalf("taken, and still said undelivered: %+v", got)
|
||||||
|
}
|
||||||
|
if n := len(r.asksSent(t)); n != 3 {
|
||||||
|
t.Fatalf("an ask taken was asked again: %d asks", n)
|
||||||
|
}
|
||||||
|
// The operator answers it; the condition stays open (its fix takes a while): the old refusal is not said again.
|
||||||
|
taken := r.asksSent(t)[2]
|
||||||
|
r.store.Change(context.Background(), taken.ID, func(x *asked) bool {
|
||||||
|
x.State, x.Ended = string(asks.OutcomeChosen), r.now
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
for i := 0; i < 5; i++ {
|
||||||
|
r.now = r.now.Add(askEvery)
|
||||||
|
_ = r.a.reconcile(context.Background())
|
||||||
|
if got := last(); len(got) != 0 {
|
||||||
|
t.Fatalf("an answered ask brought its old refusal back: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Its explanation opens with one verdict, not two.
|
||||||
|
r.open = append(r.open, heldCondition())
|
||||||
|
_ = r.a.reconcile(context.Background())
|
||||||
|
refuse("no channel can carry any of its answers now")
|
||||||
|
_ = r.a.reconcile(context.Background())
|
||||||
|
if e := conditions.Verdict(last()[0].Needs, last()[0].Explanation); strings.Count(e, "Needs you") != 1 {
|
||||||
|
t.Errorf("the explanation says its verdict twice: %q", e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
@@ -234,6 +235,38 @@ func routerHereIn(inv *inventory.Inventory) func(ctx context.Context) (bool, err
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// grantHeldIn says whether the bus holds the controller's grant to ask (novox/hq issue 353): the user list the
|
||||||
|
// machine holding the bus was last sent is the one the mesh composes now (brokerBehind, the same judgement a
|
||||||
|
// push makes to send that machine first). While it is behind, the controller's ask is refused by the bus,
|
||||||
|
// whatever the record says of the router, so nothing is asked and the operator is told to push that machine.
|
||||||
|
// Judged at most every grantLookEvery: composing the list resolves the bus's machine whole.
|
||||||
|
func grantHeldIn(open *stores) func(ctx context.Context) (bool, string, error) {
|
||||||
|
var mu sync.Mutex
|
||||||
|
var at time.Time
|
||||||
|
var held bool
|
||||||
|
var why string
|
||||||
|
return func(ctx context.Context) (bool, string, error) {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
if !at.IsZero() && time.Since(at) < grantLookEvery {
|
||||||
|
return held, why, nil
|
||||||
|
}
|
||||||
|
machine, behind, err := brokerBehind(ctx, open, nil)
|
||||||
|
if err != nil {
|
||||||
|
return false, "", err
|
||||||
|
}
|
||||||
|
at, held, why = time.Now(), !behind, ""
|
||||||
|
if behind {
|
||||||
|
why = fmt.Sprintf("the bus's user list on %s is behind what the mesh composes, so the bus has not been "+
|
||||||
|
"given the controller's grant to ask; `push %s` carries it", machine, machine)
|
||||||
|
}
|
||||||
|
return held, why, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// grantLookEvery is how often the bus's user list is judged against the one its machine was last sent.
|
||||||
|
const grantLookEvery = 30 * time.Second
|
||||||
|
|
||||||
// channelsIn is what the channels are now, as a fingerprint: each module claiming a kind of the channel
|
// channelsIn is what the channels are now, as a fingerprint: each module claiming a kind of the channel
|
||||||
// bench, where, promising what, and whether of its own account. An ask the router refused is asked again
|
// bench, where, promising what, and whether of its own account. An ask the router refused is asked again
|
||||||
// once this changes.
|
// once this changes.
|
||||||
@@ -281,12 +314,15 @@ func startAsking(ctx context.Context, open *stores, server *link.Server, conn *n
|
|||||||
return err
|
return err
|
||||||
},
|
},
|
||||||
call: callAction(conn),
|
call: callAction(conn),
|
||||||
|
// A proposed settings layer is set by this controller itself on the warrant (novox/hq ADR 0277).
|
||||||
|
setLayer: setLayerIn(open),
|
||||||
record: func(ctx context.Context, act link.HandAct) error {
|
record: func(ctx context.Context, act link.HandAct) error {
|
||||||
_, err := link.RecordHandAct(ctx, conn, act)
|
_, err := link.RecordHandAct(ctx, conn, act)
|
||||||
return err
|
return err
|
||||||
},
|
},
|
||||||
routerRecord: routerRecordOf(conn, open.inventory),
|
routerRecord: routerRecordOf(conn, open.inventory),
|
||||||
routerHere: routerHereIn(open.inventory),
|
routerHere: routerHereIn(open.inventory),
|
||||||
|
grantHeld: grantHeldIn(open),
|
||||||
channels: channelsIn(open.inventory),
|
channels: channelsIn(open.inventory),
|
||||||
raise: func(ctx context.Context, obs []conditions.Observation) error {
|
raise: func(ctx context.Context, obs []conditions.Observation) error {
|
||||||
return keeper.Reconcile(ctx, sourceAsker, obs)
|
return keeper.Reconcile(ctx, sourceAsker, obs)
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,902 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The suite's tests of a merge, but for the merge window's own (this file's), plan the merge as it is heard: a
|
||||||
|
// window of nothing closes at once, so a merge with no walk open is cut into its walk at once, as every merge
|
||||||
|
// was planned before novox/hq ADR 0276. The window's tests set it through the controller's settings.
|
||||||
|
func init() { mergeWindowDefault = 0 }
|
||||||
|
|
||||||
|
// windowed is a mesh whose controller's settings give a merge window of 90 seconds and at most 10 minutes, as its
|
||||||
|
// settings file says them, and
|
||||||
|
// modules built from the catalogue (app, notes) and from three repositories of their own.
|
||||||
|
func windowed(t *testing.T) *stores {
|
||||||
|
t.Helper()
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
settings := t.TempDir() + "/merge-window.json"
|
||||||
|
if err := os.WriteFile(settings, []byte(`{"merge-window": "90s", "merge-window-at-most": "10m"}`), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Setenv(mergeWindowFileVar, settings)
|
||||||
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"},
|
||||||
|
inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Ref: "main", BuiltFrom: "c0",
|
||||||
|
Head: "c0"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, m := range []struct{ module, repository, path string }{
|
||||||
|
{"app", "novox/mesh-catalog", "modules/app"},
|
||||||
|
{"notes", "novox/mesh-catalog", "modules/notes"},
|
||||||
|
{"one", "novox/one", ""},
|
||||||
|
{"two", "novox/two", ""},
|
||||||
|
{"three", "novox/three", ""},
|
||||||
|
} {
|
||||||
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1"},
|
||||||
|
inventory.Source{Repository: m.repository, Seat: "git", Path: m.path, Ref: "main", BuiltFrom: "c0",
|
||||||
|
Head: "c0"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return open
|
||||||
|
}
|
||||||
|
|
||||||
|
// t0 is the moment a test's clock starts: merges are dated by it, and heard after it.
|
||||||
|
var t0 = time.Now().UTC().Truncate(time.Second)
|
||||||
|
|
||||||
|
// catalogueMerge is a merge of the catalogue changing one module's directory, made at a moment.
|
||||||
|
func catalogueMerge(commit, module string, made time.Time) link.SourceMoved {
|
||||||
|
return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit,
|
||||||
|
MergedAt: made.Format(time.RFC3339Nano), Paths: []string{"modules/" + module + "/module.json"},
|
||||||
|
ModuleDirs: []string{"modules/" + module}, ModuleDirsSaid: true}
|
||||||
|
}
|
||||||
|
|
||||||
|
// repoMerge is a merge of a repository of one module's own.
|
||||||
|
func repoMerge(repo, commit string, made time.Time) link.SourceMoved {
|
||||||
|
return link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Commit: commit,
|
||||||
|
MergedAt: made.Format(time.RFC3339Nano), Paths: []string{"main.go"}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// hear is a merge heard at a moment.
|
||||||
|
func hear(t *testing.T, open *stores, m link.SourceMoved, now time.Time) {
|
||||||
|
t.Helper()
|
||||||
|
if err := (following{open: open}).hearMerge(t.Context(), m, now); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// cutAt is the cutter looking at a moment.
|
||||||
|
func cutAt(t *testing.T, open *stores, now time.Time) {
|
||||||
|
t.Helper()
|
||||||
|
if err := cutBatchesHeld(t.Context(), open, now); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// walks is every plan record that is a walk, newest first, and the batches not yet cut.
|
||||||
|
func walks(t *testing.T, open *stores) (walks, batches []inventory.Plan) {
|
||||||
|
t.Helper()
|
||||||
|
recent, err := open.inventory.RecentPlans(t.Context(), 50)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, p := range recent {
|
||||||
|
if p.Batch() {
|
||||||
|
batches = append(batches, p)
|
||||||
|
} else {
|
||||||
|
walks = append(walks, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return walks, batches
|
||||||
|
}
|
||||||
|
|
||||||
|
// The replay of issue 362: two catalogue merges 18 seconds apart are one batch, cut once into one walk at the
|
||||||
|
// later commit, which names both merges — the earlier carried by the later.
|
||||||
|
func TestTwoMergesSecondsApartAreOneWalkAtTheLaterCommit(t *testing.T) {
|
||||||
|
open := windowed(t)
|
||||||
|
asked := asksWithPaths(t)
|
||||||
|
claude := catalogueMerge("553b7191claude", "app", t0)
|
||||||
|
dunst := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second))
|
||||||
|
hear(t, open, claude, t0.Add(time.Second))
|
||||||
|
hear(t, open, dunst, t0.Add(18*time.Second))
|
||||||
|
if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 1 {
|
||||||
|
t.Fatalf("within the window: %d walk(s), %d batch(es)", len(ws), len(bs))
|
||||||
|
}
|
||||||
|
cutAt(t, open, t0.Add(18*time.Second+89*time.Second))
|
||||||
|
if ws, _ := walks(t, open); len(ws) != 0 {
|
||||||
|
t.Fatalf("cut before the window closed: %+v", ws)
|
||||||
|
}
|
||||||
|
cutAt(t, open, t0.Add(18*time.Second+90*time.Second))
|
||||||
|
ws, bs := walks(t, open)
|
||||||
|
if len(ws) != 1 || len(bs) != 0 {
|
||||||
|
t.Fatalf("after the window: %d walk(s), %d batch(es)", len(ws), len(bs))
|
||||||
|
}
|
||||||
|
w := ws[0]
|
||||||
|
if w.Commit != dunst.Commit || len(w.Commits) != 1 || w.Commits[0].Commit != dunst.Commit {
|
||||||
|
t.Fatalf("the walk is at %s %+v, not the later commit", w.Commit, w.Commits)
|
||||||
|
}
|
||||||
|
for _, m := range []string{"app", "notes"} {
|
||||||
|
if _, in := w.Modules[m]; !in {
|
||||||
|
t.Fatalf("the walk does not build %s, which one of its merges moved: %v", m, w.Modules)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
want := []inventory.PlanMerge{{Repository: "novox/mesh-catalog", Commit: claude.Commit, Carried: dunst.Commit},
|
||||||
|
{Repository: "novox/mesh-catalog", Commit: dunst.Commit}}
|
||||||
|
if w.Delivery == nil || !slices.EqualFunc(w.Delivery.Merges, want, sameMerge) {
|
||||||
|
t.Fatalf("the walk answers %+v, want %+v", w.Delivery, want)
|
||||||
|
}
|
||||||
|
if len(*asked) != 2 || (*asked)[0][2] != "main" || (*asked)[1][2] != "main" {
|
||||||
|
t.Fatalf("the walk did not ask its modules at the branch, which holds the commit it carries: %v", *asked)
|
||||||
|
}
|
||||||
|
// Heard again, as the bus may hand it over twice: never a second merge, nor a second walk.
|
||||||
|
hear(t, open, claude, t0.Add(5*time.Minute))
|
||||||
|
if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 {
|
||||||
|
t.Fatalf("a merge heard twice made %d walk(s) and %d batch(es)", len(ws), len(bs))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Merges of three repositories in one window are one walk, one commit for each.
|
||||||
|
func TestThreeRepositoriesInOneWindowAreOneWalk(t *testing.T) {
|
||||||
|
open := windowed(t)
|
||||||
|
asksRecorded(t)
|
||||||
|
for i, r := range []string{"one", "two", "three"} {
|
||||||
|
hear(t, open, repoMerge(r, "c-"+r, t0.Add(time.Duration(i)*20*time.Second)),
|
||||||
|
t0.Add(time.Duration(i)*20*time.Second+time.Second))
|
||||||
|
}
|
||||||
|
cutAt(t, open, t0.Add(41*time.Second+90*time.Second))
|
||||||
|
ws, bs := walks(t, open)
|
||||||
|
if len(ws) != 1 || len(bs) != 0 {
|
||||||
|
t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs))
|
||||||
|
}
|
||||||
|
w := ws[0]
|
||||||
|
if len(w.Commits) != 3 || len(w.Delivery.Merges) != 3 {
|
||||||
|
t.Fatalf("the walk carries %+v and answers %+v", w.Commits, w.Delivery.Merges)
|
||||||
|
}
|
||||||
|
for _, r := range []string{"one", "two", "three"} {
|
||||||
|
if w.CommitOf("novox/"+r) != "c-"+r {
|
||||||
|
t.Fatalf("the walk carries %s at %q", r, w.CommitOf("novox/"+r))
|
||||||
|
}
|
||||||
|
if _, in := w.Modules[r]; !in {
|
||||||
|
t.Fatalf("the walk does not build %s: %v", r, w.Modules)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Each module's ask is recorded at its own repository's commit.
|
||||||
|
for _, r := range []string{"one", "two", "three"} {
|
||||||
|
q, found, err := open.inventory.BuildRequestByID(t.Context(), w.Modules[r].Build)
|
||||||
|
if err != nil || !found || q.Commit != "c-"+r {
|
||||||
|
t.Fatalf("%s's build was not recorded at its own commit: %+v %v %v", r, q, found, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Each merge restarts the window: a merge at second 80 keeps the batch open until second 170; the window
|
||||||
|
// closes at most ten minutes after its first merge however busy.
|
||||||
|
func TestAMergeRestartsTheWindowAndTheMaximumClosesIt(t *testing.T) {
|
||||||
|
open := windowed(t)
|
||||||
|
asksWithPaths(t)
|
||||||
|
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||||
|
hear(t, open, repoMerge("two", "c2", t0.Add(80*time.Second)), t0.Add(80*time.Second))
|
||||||
|
cutAt(t, open, t0.Add(100*time.Second))
|
||||||
|
if ws, _ := walks(t, open); len(ws) != 0 {
|
||||||
|
t.Fatal("the window closed 90 seconds after its first merge, not after its last")
|
||||||
|
}
|
||||||
|
cutAt(t, open, t0.Add(170*time.Second))
|
||||||
|
if ws, _ := walks(t, open); len(ws) != 1 || len(ws[0].Commits) != 2 {
|
||||||
|
t.Fatalf("the window did not close 90 seconds after its last merge: %+v", ws)
|
||||||
|
}
|
||||||
|
|
||||||
|
busy := windowed(t)
|
||||||
|
asksWithPaths(t)
|
||||||
|
var last time.Time
|
||||||
|
for i := 0; i < 12; i++ {
|
||||||
|
last = t0.Add(time.Duration(i) * time.Minute)
|
||||||
|
repo := []string{"one", "two", "three"}[i%3]
|
||||||
|
hear(t, busy, repoMerge(repo, "c"+string(rune('a'+i)), last), last)
|
||||||
|
if ws, _ := walks(t, busy); len(ws) > 0 {
|
||||||
|
if i != 10 {
|
||||||
|
t.Fatalf("a busy window closed at merge %d (minute %d), not at its maximum", i, i)
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ws, _ := walks(t, busy)
|
||||||
|
if len(ws) != 1 {
|
||||||
|
t.Fatalf("ten minutes of merges a minute apart were never cut: %d walk(s)", len(ws))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// `plans` lists the batch being assembled first, in the operator's words: how long is left, when at the latest,
|
||||||
|
// what is grouped, and that the plan is not yet calculated. It says so on the bus as plan-moved, too.
|
||||||
|
func TestTheAssemblingBatchIsShown(t *testing.T) {
|
||||||
|
open := windowed(t)
|
||||||
|
asksWithPaths(t)
|
||||||
|
var said []inventory.Plan
|
||||||
|
was := inventory.PlanSaved
|
||||||
|
inventory.PlanSaved = func(p inventory.Plan) { said = append(said, p) }
|
||||||
|
t.Cleanup(func() { inventory.PlanSaved = was })
|
||||||
|
now := time.Now().UTC()
|
||||||
|
claude := catalogueMerge("553b7191claude", "app", now.Add(-20*time.Second))
|
||||||
|
claude.Number, claude.Title = 174, "claude-code: an agent proposes a section"
|
||||||
|
dunst := catalogueMerge("48bda475dunst", "notes", now.Add(-2*time.Second))
|
||||||
|
dunst.Number, dunst.Title = 175, "dunst: the font the operator chose"
|
||||||
|
hear(t, open, claude, now.Add(-19*time.Second))
|
||||||
|
hear(t, open, dunst, now.Add(-time.Second))
|
||||||
|
hear(t, open, repoMerge("one", "a6bc0931one", now), now)
|
||||||
|
out := captured(t, func() error { return plansCommand(t.Context(), nil) })
|
||||||
|
lines := strings.Split(out, "\n")
|
||||||
|
first := lines[0]
|
||||||
|
for _, want := range []string{"assembling: ", " s left (at the latest ", "grouped: novox/mesh-catalog@48bda475 " +
|
||||||
|
"(answers 553b7191), novox/one@a6bc0931; plan not yet calculated"} {
|
||||||
|
if !strings.Contains(first, want) {
|
||||||
|
t.Fatalf("plans' first line %q does not say %q", first, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Under it, one line per repository: the pull request, what it answers, what it moves.
|
||||||
|
if len(lines) < 3 || strings.TrimSpace(lines[1]) != "mesh-catalog #175 dunst: the font the operator chose (answers "+
|
||||||
|
"#174 claude-code: an agent proposes a section) · app, notes" ||
|
||||||
|
strings.TrimSpace(lines[2]) != "one a6bc0931 · one" {
|
||||||
|
t.Fatalf("the grouped list reads %q", lines[1:4])
|
||||||
|
}
|
||||||
|
if len(said) == 0 || said[len(said)-1].State != inventory.PlanAssembling || said[len(said)-1].Delivery.Batch == nil ||
|
||||||
|
len(said[len(said)-1].Delivery.Merges) != 3 {
|
||||||
|
t.Fatalf("the batch was not said as assembling with its merges: %+v", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// captured is what a command printed.
|
||||||
|
func captured(t *testing.T, run func() error) string {
|
||||||
|
t.Helper()
|
||||||
|
r, w, err := os.Pipe()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
was := os.Stdout
|
||||||
|
os.Stdout = w
|
||||||
|
runErr := run()
|
||||||
|
os.Stdout = was
|
||||||
|
_ = w.Close()
|
||||||
|
var b bytes.Buffer
|
||||||
|
_, _ = io.Copy(&b, r)
|
||||||
|
if runErr != nil {
|
||||||
|
t.Fatal(runErr)
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// One walk at a time: a merge heard while a walk runs joins the next batch, which is cut when the walk ends;
|
||||||
|
// the walk that started is never superseded.
|
||||||
|
func TestAMergeDuringAWalkJoinsTheNextBatch(t *testing.T) {
|
||||||
|
open := windowed(t)
|
||||||
|
asksWithPaths(t)
|
||||||
|
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||||
|
cutAt(t, open, t0.Add(90*time.Second))
|
||||||
|
ws, _ := walks(t, open)
|
||||||
|
if len(ws) != 1 || !ws[0].Open() {
|
||||||
|
t.Fatalf("the first batch was not cut: %+v", ws)
|
||||||
|
}
|
||||||
|
first := ws[0]
|
||||||
|
hear(t, open, repoMerge("two", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
|
||||||
|
cutAt(t, open, t0.Add(5*time.Minute))
|
||||||
|
ws, bs := walks(t, open)
|
||||||
|
if len(ws) != 1 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != first.ID {
|
||||||
|
t.Fatalf("the merge during the walk: %d walk(s), batches %+v", len(ws), bs)
|
||||||
|
}
|
||||||
|
if got, _ := open.inventory.PlanByID(t.Context(), first.ID); !got.Open() {
|
||||||
|
t.Fatalf("the started walk was %s by the next merge", got.State)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(batchWords(bs[0], t0.Add(5*time.Minute)), "queued behind "+first.ID) {
|
||||||
|
t.Fatalf("a queued batch reads %q", batchWords(bs[0], t0.Add(5*time.Minute)))
|
||||||
|
}
|
||||||
|
first.State = inventory.PlanDone
|
||||||
|
if err := open.inventory.SavePlan(t.Context(), &first); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cutAt(t, open, t0.Add(6*time.Minute))
|
||||||
|
ws, bs = walks(t, open)
|
||||||
|
if len(ws) != 2 || len(bs) != 0 || ws[0].Commit != "c2" {
|
||||||
|
t.Fatalf("the queued batch was not cut when the walk ended: %+v %+v", ws, bs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A walk waiting for its delivery's word is folded into the next batch's walk, which answers its merges; it
|
||||||
|
// names the walk that took it over.
|
||||||
|
func TestAWaitingWalkIsFoldedIntoTheNextBatch(t *testing.T) {
|
||||||
|
open := windowed(t)
|
||||||
|
asksWithPaths(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1",
|
||||||
|
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
||||||
|
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main",
|
||||||
|
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||||
|
cutAt(t, open, t0.Add(90*time.Second))
|
||||||
|
ws, _ := walks(t, open)
|
||||||
|
if len(ws) != 1 || !ws[0].Waiting() {
|
||||||
|
t.Fatalf("the walk does not wait for its word: %+v", ws)
|
||||||
|
}
|
||||||
|
waiting := ws[0]
|
||||||
|
hear(t, open, repoMerge("two", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
|
||||||
|
cutAt(t, open, t0.Add(2*time.Minute+90*time.Second))
|
||||||
|
ws, bs := walks(t, open)
|
||||||
|
if len(bs) != 0 || len(ws) != 2 {
|
||||||
|
t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs))
|
||||||
|
}
|
||||||
|
folded, err := open.inventory.PlanByID(ctx, waiting.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
newer := ws[0]
|
||||||
|
if folded.State != inventory.PlanSuperseded || folded.Delivery.TakenOverBy != newer.ID {
|
||||||
|
t.Fatalf("the waiting walk is %s, taken over by %q", folded.State, folded.Delivery.TakenOverBy)
|
||||||
|
}
|
||||||
|
if len(newer.Delivery.Merges) != 2 || newer.CommitOf("novox/one") != "c1" || newer.CommitOf("novox/two") != "c2" {
|
||||||
|
t.Fatalf("the newer walk answers %+v", newer.Delivery.Merges)
|
||||||
|
}
|
||||||
|
if _, in := newer.Modules["one"]; !in {
|
||||||
|
t.Fatalf("what the folded walk was to build is not built: %v", newer.Modules)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A merge heard after a later merge of its branch was walked is answered by that walk when it builds all it
|
||||||
|
// moves — named at once on a walk done — and joins the next batch when it does not.
|
||||||
|
func TestALateMergeIsAnsweredByTheWalkOfTheLaterOne(t *testing.T) {
|
||||||
|
open := windowed(t)
|
||||||
|
asksWithPaths(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
later := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second))
|
||||||
|
hear(t, open, later, t0.Add(18*time.Second))
|
||||||
|
cutAt(t, open, t0.Add(2*time.Minute))
|
||||||
|
ws, _ := walks(t, open)
|
||||||
|
w := ws[0]
|
||||||
|
w.State = inventory.PlanDone
|
||||||
|
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Moves only notes, which the done walk built from the branch after it: answered there, at once.
|
||||||
|
hear(t, open, catalogueMerge("553b7191early", "notes", t0), t0.Add(15*time.Minute))
|
||||||
|
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
||||||
|
if got.State != inventory.PlanDone || len(got.Delivery.Merges) != 2 ||
|
||||||
|
!sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: "553b7191early",
|
||||||
|
Carried: later.Commit}) {
|
||||||
|
t.Fatalf("the late merge is not named by the walk that carried it: %+v", got.Delivery.Merges)
|
||||||
|
}
|
||||||
|
// Moves app, which that walk never built: the next batch walks it.
|
||||||
|
hear(t, open, catalogueMerge("1111aaaaapp", "app", t0.Add(time.Second)), t0.Add(16*time.Minute))
|
||||||
|
_, bs := walks(t, open)
|
||||||
|
if len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "1111aaaaapp" {
|
||||||
|
t.Fatalf("a late merge moving what the walk never built did not join the next batch: %+v", bs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A failed walk marks nothing delivered: its earlier merges are walked alone, on their own commit, the newest
|
||||||
|
// first; the first delivered answers the older ones, and the search stops.
|
||||||
|
func TestAFailedWalkWalksItsEarlierMergesAlone(t *testing.T) {
|
||||||
|
open := windowed(t)
|
||||||
|
asked := asksWithPaths(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
// Made before the cut marks their modules seen, as merges are: walked again, they are news all the same.
|
||||||
|
oldest := catalogueMerge("aaaa0001", "app", t0.Add(-3*time.Minute))
|
||||||
|
middle := catalogueMerge("bbbb0002", "app", t0.Add(-3*time.Minute+10*time.Second))
|
||||||
|
newest := catalogueMerge("cccc0003", "notes", t0.Add(-3*time.Minute+20*time.Second))
|
||||||
|
for i, m := range []link.SourceMoved{oldest, middle, newest} {
|
||||||
|
hear(t, open, m, t0.Add(time.Duration(i)*10*time.Second+time.Second))
|
||||||
|
}
|
||||||
|
cutAt(t, open, t0.Add(2*time.Minute))
|
||||||
|
ws, _ := walks(t, open)
|
||||||
|
failed := ws[0]
|
||||||
|
failed.State, failed.Note = inventory.PlanFailed, "notes failed to build in tier 0"
|
||||||
|
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cutAt(t, open, t0.Add(3*time.Minute))
|
||||||
|
got, _ := open.inventory.PlanByID(ctx, failed.ID)
|
||||||
|
if got.State != inventory.PlanFailed || len(got.Delivery.Merges) != 3 {
|
||||||
|
t.Fatalf("the failed walk's record changed: %s %+v", got.State, got.Delivery.Merges)
|
||||||
|
}
|
||||||
|
ws, _ = walks(t, open)
|
||||||
|
alone := ws[0]
|
||||||
|
if alone.ID == failed.ID || alone.Commit != middle.Commit || len(alone.Delivery.Merges) != 1 ||
|
||||||
|
!sameMerge(alone.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: middle.Commit}) {
|
||||||
|
t.Fatalf("the newest earlier merge was not walked alone on its own commit: %+v", alone)
|
||||||
|
}
|
||||||
|
if _, in := alone.Modules["app"]; !in || (*asked)[len(*asked)-1] != [3]string{"novox/mesh-catalog", "modules/app",
|
||||||
|
middle.Commit} {
|
||||||
|
t.Fatalf("the merge walked alone does not build app at its own commit: %v, asked %v", alone.Modules, *asked)
|
||||||
|
}
|
||||||
|
// Retried, the failed walk would name merges the search answers now.
|
||||||
|
if _, err := retryPlan(ctx, open, failed.ID); err == nil || !strings.Contains(err.Error(), "walked alone") {
|
||||||
|
t.Fatalf("a failed walk whose merges are searched was retried: %v", err)
|
||||||
|
}
|
||||||
|
alone.State = inventory.PlanDone
|
||||||
|
if err := open.inventory.SavePlan(ctx, &alone); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cutAt(t, open, t0.Add(4*time.Minute))
|
||||||
|
ws, bs := walks(t, open)
|
||||||
|
if ws[0].ID != alone.ID || len(bs) != 0 {
|
||||||
|
t.Fatalf("the search went on after a merge was delivered: %+v", ws[0])
|
||||||
|
}
|
||||||
|
done, _ := open.inventory.PlanByID(ctx, alone.ID)
|
||||||
|
if len(done.Delivery.Merges) != 2 || !sameMerge(done.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog",
|
||||||
|
Commit: oldest.Commit, Carried: middle.Commit}) {
|
||||||
|
t.Fatalf("the oldest merge is not answered by the walk that delivered the one after it: %+v", done.Delivery.Merges)
|
||||||
|
}
|
||||||
|
// A stopped walk starts no search: a person ended it.
|
||||||
|
hear(t, open, catalogueMerge("dddd0004", "app", t0.Add(5*time.Minute)), t0.Add(5*time.Minute))
|
||||||
|
hear(t, open, catalogueMerge("eeee0005", "notes", t0.Add(5*time.Minute+time.Second)), t0.Add(5*time.Minute+time.Second))
|
||||||
|
cutAt(t, open, t0.Add(8*time.Minute))
|
||||||
|
ws, _ = walks(t, open)
|
||||||
|
if _, err := stopWalk(ctx, open.inventory, ws[0].ID, "mesh-delivery for jochen", "not now"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cutAt(t, open, t0.Add(9*time.Minute))
|
||||||
|
if again, _ := walks(t, open); again[0].ID != ws[0].ID {
|
||||||
|
t.Fatalf("a stopped walk's earlier merge was walked alone: %+v", again[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A delivery group's order holds inside the walk (ADR 0249 unchanged): merges of two repositories from one head
|
||||||
|
// branch name, the node-engine's before the controller's, are tiered so; without the group, one tier.
|
||||||
|
func TestAGroupsOrderBecomesTiersInsideTheWalk(t *testing.T) {
|
||||||
|
for _, grouped := range []bool{true, false} {
|
||||||
|
open := windowed(t)
|
||||||
|
asksWithPaths(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
for _, m := range []string{"mesh-host"} {
|
||||||
|
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"},
|
||||||
|
inventory.Source{Repository: "novox/" + m, Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
host := repoMerge("mesh-host", "h1", t0)
|
||||||
|
ctl := repoMerge("mesh-controller", "k1", t0.Add(time.Second))
|
||||||
|
host.Head, ctl.Head = "feat/together", "feat/together"
|
||||||
|
if !grouped {
|
||||||
|
ctl.Head = "feat/alone"
|
||||||
|
}
|
||||||
|
hear(t, open, ctl, t0.Add(2*time.Second))
|
||||||
|
hear(t, open, host, t0.Add(3*time.Second))
|
||||||
|
cutAt(t, open, t0.Add(2*time.Minute))
|
||||||
|
ws, _ := walks(t, open)
|
||||||
|
if len(ws) != 1 {
|
||||||
|
t.Fatalf("%d walks", len(ws))
|
||||||
|
}
|
||||||
|
tiers := ws[0].Tiers
|
||||||
|
if grouped {
|
||||||
|
if len(tiers) != 2 || !slices.Equal(tiers[0], []string{"mesh-host"}) ||
|
||||||
|
!slices.Equal(tiers[1], []string{"mesh-controller"}) {
|
||||||
|
t.Fatalf("the group's order is not the walk's tiers: %v", tiers)
|
||||||
|
}
|
||||||
|
} else if len(tiers) != 1 {
|
||||||
|
t.Fatalf("two merges of no group were ordered: %v", tiers)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A restarted controller resumes the window where it stood: the batch, its merges and their times are in the
|
||||||
|
// store, a merge handed over again is not doubled, and the batch is cut when its window closes.
|
||||||
|
func TestABatchSurvivesARestart(t *testing.T) {
|
||||||
|
open := windowed(t)
|
||||||
|
asksWithPaths(t)
|
||||||
|
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||||
|
hear(t, open, repoMerge("two", "c2", t0.Add(30*time.Second)), t0.Add(30*time.Second))
|
||||||
|
|
||||||
|
// A new controller: nothing of the first one's but the store.
|
||||||
|
again, err := openStores(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(again.Close)
|
||||||
|
hear(t, again, repoMerge("one", "c1", t0), t0.Add(60*time.Second)) // the bus hands it over again
|
||||||
|
cutAt(t, again, t0.Add(119*time.Second))
|
||||||
|
if ws, bs := walks(t, again); len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 {
|
||||||
|
t.Fatalf("the restarted controller's batch: %d walk(s), %+v", len(ws), bs)
|
||||||
|
}
|
||||||
|
cutAt(t, again, t0.Add(120*time.Second))
|
||||||
|
ws, bs := walks(t, again)
|
||||||
|
if len(ws) != 1 || len(bs) != 0 || len(ws[0].Delivery.Merges) != 2 {
|
||||||
|
t.Fatalf("the restarted controller did not cut the batch at its window: %+v %+v", ws, bs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The window's settings reach the controller in its settings file, read at every look; one that says no
|
||||||
|
// duration, or no file, is the default.
|
||||||
|
func TestTheWindowIsTheControllersSetting(t *testing.T) {
|
||||||
|
file := t.TempDir() + "/merge-window.json"
|
||||||
|
t.Setenv(mergeWindowFileVar, file)
|
||||||
|
for _, c := range []struct {
|
||||||
|
content string
|
||||||
|
window, atMost time.Duration
|
||||||
|
}{
|
||||||
|
{`{"merge-window": "60s", "merge-window-at-most": "5m"}`, time.Minute, 5 * time.Minute},
|
||||||
|
{`{"merge-window": 30, "merge-window-at-most": ""}`, 30 * time.Second, mergeWindowAtMostDefault},
|
||||||
|
{`{"merge-window": "soon"}`, mergeWindowDefault, mergeWindowAtMostDefault},
|
||||||
|
} {
|
||||||
|
if err := os.WriteFile(file, []byte(c.content), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if w, m := mergeWindowOf(); w != c.window || m != c.atMost {
|
||||||
|
t.Errorf("%s reads as %s and %s, want %s and %s", c.content, w, m, c.window, c.atMost)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Setenv(mergeWindowFileVar, file+".gone")
|
||||||
|
if w, m := mergeWindowOf(); w != mergeWindowDefault || m != mergeWindowAtMostDefault {
|
||||||
|
t.Errorf("no file reads as %s and %s", w, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// S16 names every merge a waiting walk answers, never one commit a supersession may have ended (issue 362).
|
||||||
|
func TestAWaitingWalkNamesTheMergesItAnswers(t *testing.T) {
|
||||||
|
f := calm(t0)
|
||||||
|
f.waits = []waitFacts{{id: "plan-2", repository: "novox/mesh-catalog", commit: "48bda475dunst", awaits: "mesh-delivery",
|
||||||
|
since: t0.Add(-31 * time.Minute), merges: []inventory.PlanMerge{
|
||||||
|
{Repository: "novox/mesh-catalog", Commit: "553b7191claude", Carried: "48bda475dunst"},
|
||||||
|
{Repository: "novox/mesh-catalog", Commit: "48bda475dunst"}}}}
|
||||||
|
got := watchWaits(f)
|
||||||
|
if len(got) != 1 || !strings.Contains(got[0].Summary, "novox/mesh-catalog@553b7191") ||
|
||||||
|
!strings.Contains(got[0].Summary, "novox/mesh-catalog@48bda475") {
|
||||||
|
t.Fatalf("the waiting walk does not name both merges: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A merge heard after a later merge of its branch was cut is named by that walk however its modules read since
|
||||||
|
// the cut, in a repository of one module (review of this change); a walk that never built what it moves names
|
||||||
|
// nothing, and the merge joins the next batch.
|
||||||
|
func TestALateMergeOfAOneModuleRepositoryIsNamed(t *testing.T) {
|
||||||
|
open := windowed(t)
|
||||||
|
asksWithPaths(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
c2 := repoMerge("one", "c2", t0.Add(-50*time.Second))
|
||||||
|
hear(t, open, c2, t0)
|
||||||
|
cutAt(t, open, t0.Add(2*time.Minute))
|
||||||
|
ws, _ := walks(t, open)
|
||||||
|
w := ws[0]
|
||||||
|
w.State = inventory.PlanDone
|
||||||
|
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
hear(t, open, repoMerge("one", "c1", t0.Add(-60*time.Second)), t0.Add(15*time.Minute))
|
||||||
|
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
||||||
|
if len(got.Delivery.Merges) != 2 || !sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/one",
|
||||||
|
Commit: "c1", Carried: "c2"}) {
|
||||||
|
t.Fatalf("the late merge was not named by the walk that carried it: %+v", got.Delivery.Merges)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A walk of two that built only two: a late merge of one is not its to name.
|
||||||
|
hear(t, open, repoMerge("two", "d2", t0.Add(16*time.Minute)), t0.Add(16*time.Minute))
|
||||||
|
cutAt(t, open, t0.Add(18*time.Minute))
|
||||||
|
ws, _ = walks(t, open)
|
||||||
|
w = ws[0]
|
||||||
|
delete(w.Modules, "two")
|
||||||
|
w.State = inventory.PlanDone
|
||||||
|
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
hear(t, open, repoMerge("two", "d1", t0.Add(15*time.Minute)), t0.Add(20*time.Minute))
|
||||||
|
if got, _ := open.inventory.PlanByID(ctx, w.ID); len(got.Delivery.Merges) != 1 {
|
||||||
|
t.Fatalf("a walk that never built what the late merge moves named it: %+v", got.Delivery.Merges)
|
||||||
|
}
|
||||||
|
if _, bs := walks(t, open); len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "d1" {
|
||||||
|
t.Fatalf("the late merge did not join the next batch: %+v", bs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// One walk at a time holds against the delivery's word too: a waiting walk let go beside a started one takes the
|
||||||
|
// word and starts once that one ended, asking nothing before.
|
||||||
|
func TestAWalkLetGoBesideAStartedOneStartsOnceItEnded(t *testing.T) {
|
||||||
|
open := windowed(t)
|
||||||
|
asked := asksWithPaths(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||||
|
cutAt(t, open, t0.Add(2*time.Minute))
|
||||||
|
started, _ := walks(t, open)
|
||||||
|
waiting := inventory.Plan{ID: "plan-waiting", Repository: "novox/two", Branch: "main", Commit: "d1", Created: t0,
|
||||||
|
State: inventory.PlanBuilding, Tiers: [][]string{{"two"}}, Modules: map[string]*inventory.PlanModule{"two": {}},
|
||||||
|
Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}}
|
||||||
|
if err := open.inventory.SavePlan(ctx, &waiting); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := letGo(ctx, open.inventory, waiting.ID, catalogue.DeliverySeat, "its turn"); err != nil {
|
||||||
|
t.Fatalf("the word was refused beside %s: %v", started[0].ID, err)
|
||||||
|
}
|
||||||
|
before := len(*asked)
|
||||||
|
advanceHeld(ctx, open)
|
||||||
|
got, _ := open.inventory.PlanByID(ctx, waiting.ID)
|
||||||
|
if len(*asked) != before || !strings.Contains(got.Note, "starts once "+started[0].ID) {
|
||||||
|
t.Fatalf("a walk let go beside a started one asked (%d → %d) or does not say it waits: %q", before, len(*asked), got.Note)
|
||||||
|
}
|
||||||
|
// Read as a wait, an hour on: its note on the line, never LATE, and no tier of it late for S3.
|
||||||
|
later := time.Now().Add(time.Hour)
|
||||||
|
if line := planLine(got, later); !strings.Contains(line, "starts once "+started[0].ID) || strings.Contains(line, "LATE") {
|
||||||
|
t.Fatalf("a deferred walk reads %q", line)
|
||||||
|
}
|
||||||
|
facts, _, err := gatherPlans(ctx, open.inventory, later, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, f := range facts {
|
||||||
|
if f.id == got.ID {
|
||||||
|
t.Fatalf("a deferred walk is watched as a tier running late: %+v", f)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
done := started[0]
|
||||||
|
done.State = inventory.PlanDone
|
||||||
|
if err := open.inventory.SavePlan(ctx, &done); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
advanceHeld(ctx, open)
|
||||||
|
if len(*asked) != before+1 {
|
||||||
|
t.Fatalf("the walk did not start once the started one ended: asked %v", *asked)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A file a merge of the batch removed and a later one brought back is not removed; one removed last is.
|
||||||
|
func TestARemovedFileIsTheLastMergesWord(t *testing.T) {
|
||||||
|
ev := func(commit string, paths, removed []string) inventory.BatchedMerge {
|
||||||
|
m := link.SourceMoved{Owner: "novox", Repo: "one", Base: "main", Commit: commit, Paths: paths, Removed: removed}
|
||||||
|
b, _ := json.Marshal(m)
|
||||||
|
return inventory.BatchedMerge{Repository: "novox/one", Branch: "main", Commit: commit, Event: b,
|
||||||
|
Merged: t0.Add(time.Duration(len(commit)) * time.Second)}
|
||||||
|
}
|
||||||
|
got := combinedMerges([]inventory.BatchedMerge{
|
||||||
|
ev("a", []string{"x/module.json", "y/module.json"}, []string{"x/module.json", "y/module.json"}),
|
||||||
|
ev("bb", []string{"x/module.json"}, nil),
|
||||||
|
ev("ccc", []string{"z.go"}, nil)})
|
||||||
|
if len(got) != 1 || got[0].Commit != "ccc" || !slices.Equal(got[0].Removed, []string{"y/module.json"}) ||
|
||||||
|
len(got[0].Paths) != 3 {
|
||||||
|
t.Fatalf("combined as %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The catch-up hands over what the bus lost after its branch's later merges were cut, and the record keeps it:
|
||||||
|
// an earlier merge is named by the walk that carried it; a merge made before a cut and heard after it, which
|
||||||
|
// no later merge carries, joins the next batch — neither reads as history and is dropped (review of this change).
|
||||||
|
func TestTheCatchUpKeepsWhatACutMadeHistory(t *testing.T) {
|
||||||
|
open := windowed(t)
|
||||||
|
asksWithPaths(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
base := time.Now().UTC().Add(-time.Hour).Truncate(time.Second)
|
||||||
|
hear(t, open, repoMerge("one", "c2", base.Add(10*time.Second)), base.Add(11*time.Second))
|
||||||
|
cutAt(t, open, base.Add(2*time.Minute))
|
||||||
|
ws, _ := walks(t, open)
|
||||||
|
w := ws[0]
|
||||||
|
w.State = inventory.PlanDone
|
||||||
|
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
lost := []link.AnnouncedMerge{
|
||||||
|
{SourceMoved: repoMerge("one", "c1", base), At: base.Add(time.Second)},
|
||||||
|
{SourceMoved: repoMerge("one", "c3lost", base.Add(20*time.Second)), At: base.Add(21 * time.Second)},
|
||||||
|
}
|
||||||
|
catalogued := func(ctx context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
|
||||||
|
entries, err := open.inventory.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
read, err := readForPlanning(ctx, open.inventory)
|
||||||
|
return entries, read, err
|
||||||
|
}
|
||||||
|
if err := catchUpOnMerges(ctx, time.Now(), unheardMerges{announcedList(lost), open.inventory}, catalogued,
|
||||||
|
(following{open}).SourceMoved, t.Logf); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
||||||
|
if len(got.Delivery.Merges) != 2 || !sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/one",
|
||||||
|
Commit: "c1", Carried: "c2"}) {
|
||||||
|
t.Fatalf("the earlier merge the catch-up handed over is not named by the walk that carried it: %+v",
|
||||||
|
got.Delivery.Merges)
|
||||||
|
}
|
||||||
|
if _, kept, err := open.inventory.MergeOf(ctx, "novox/one", "c3lost"); err != nil || !kept {
|
||||||
|
t.Fatalf("the merge made before the cut and heard after it was dropped: %v %v", kept, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A merge on the controller's own path never shares a batch with one that waits for mesh-delivery's word
|
||||||
|
// (decided during the build, 2026-10-10): the two are batches of their own, cut one after the other; a walk on
|
||||||
|
// the own path folds no waiting catalogue walk but batches its merges again behind it; the catalogue's walk
|
||||||
|
// still waits for the word, so no catalogue delivery skips its turn.
|
||||||
|
func TestAMergeOnTheControllersPathNeverSharesABatch(t *testing.T) {
|
||||||
|
open := windowed(t)
|
||||||
|
asked := asksWithPaths(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1",
|
||||||
|
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
||||||
|
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main",
|
||||||
|
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
hear(t, open, catalogueMerge("aaaa0001", "app", t0), t0)
|
||||||
|
hear(t, open, repoMerge("mesh-controller", "k1", t0.Add(10*time.Second)), t0.Add(10*time.Second))
|
||||||
|
hear(t, open, catalogueMerge("bbbb0002", "notes", t0.Add(20*time.Second)), t0.Add(20*time.Second))
|
||||||
|
_, bs := walks(t, open)
|
||||||
|
var ownBatch, catalogueBatch inventory.Plan
|
||||||
|
for _, b := range bs {
|
||||||
|
if b.OwnPath() {
|
||||||
|
ownBatch = b
|
||||||
|
} else {
|
||||||
|
catalogueBatch = b
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(bs) != 2 || ownBatch.ID == "" || catalogueBatch.ID == "" || len(catalogueBatch.Delivery.Merges) != 2 ||
|
||||||
|
len(ownBatch.Delivery.Merges) != 1 {
|
||||||
|
t.Fatalf("a controller merge and two catalogue merges in one window: %+v", bs)
|
||||||
|
}
|
||||||
|
if !strings.Contains(batchWords(ownBatch, t0.Add(30*time.Second)), "own path") {
|
||||||
|
t.Fatalf("the own-path batch does not say so: %q", batchWords(ownBatch, t0.Add(30*time.Second)))
|
||||||
|
}
|
||||||
|
// Both windows closed, the controller's batch is cut first and starts; the catalogue batch queues behind it,
|
||||||
|
// is cut when the controller's walk ended, and waits for its word with both merges.
|
||||||
|
cutAt(t, open, t0.Add(2*time.Minute))
|
||||||
|
ws, bs := walks(t, open)
|
||||||
|
if len(ws) != 1 || ws[0].Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k1" {
|
||||||
|
t.Fatalf("the controller's batch was not cut first into a started walk: %+v", ws)
|
||||||
|
}
|
||||||
|
ownWalk := ws[0]
|
||||||
|
for _, m := range []string{"app", "notes"} {
|
||||||
|
if _, in := ownWalk.Modules[m]; in {
|
||||||
|
t.Fatalf("the controller's walk builds %s, a catalogue module: it skipped its turn", m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != ownWalk.ID || bs[0].OwnPath() {
|
||||||
|
t.Fatalf("the catalogue batch does not queue behind the controller's walk: %+v", bs)
|
||||||
|
}
|
||||||
|
ownWalk.State = inventory.PlanDone
|
||||||
|
if err := open.inventory.SavePlan(ctx, &ownWalk); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cutAt(t, open, t0.Add(3*time.Minute))
|
||||||
|
ws, bs = walks(t, open)
|
||||||
|
if len(bs) != 0 || ws[0].ID != catalogueBatch.ID || !ws[0].Waiting() || len(ws[0].Delivery.Merges) != 2 {
|
||||||
|
t.Fatalf("the catalogue batch was not cut into a waiting walk once the controller's ended: %+v %+v", ws, bs)
|
||||||
|
}
|
||||||
|
for _, m := range []string{"app", "notes"} {
|
||||||
|
if _, in := ws[0].Modules[m]; !in {
|
||||||
|
t.Fatalf("the catalogue walk does not build %s: %v", m, ws[0].Modules)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, a := range *asked {
|
||||||
|
if a[1] == "modules/app" || a[1] == "modules/notes" {
|
||||||
|
t.Fatalf("a catalogue module was asked without the word: %v", *asked)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A catalogue walk waiting for its word when an own-path batch is cut keeps waiting beside the own-path
|
||||||
|
// walk, is not folded into it, and its word is taken meanwhile: it starts once the own-path walk ended.
|
||||||
|
catalogueWalk := ws[0]
|
||||||
|
hear(t, open, repoMerge("mesh-controller", "k2", t0.Add(4*time.Minute)), t0.Add(4*time.Minute))
|
||||||
|
cutAt(t, open, t0.Add(6*time.Minute))
|
||||||
|
ws, _ = walks(t, open)
|
||||||
|
kept, _ := open.inventory.PlanByID(ctx, catalogueWalk.ID)
|
||||||
|
if !kept.Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k2" || ws[0].Waiting() {
|
||||||
|
t.Fatalf("the waiting catalogue walk was not kept waiting beside the controller's walk: %s %+v", kept.State, ws)
|
||||||
|
}
|
||||||
|
if _, in := ws[0].Modules["app"]; in {
|
||||||
|
t.Fatalf("the controller's walk folded the waiting catalogue walk in: %v", ws[0].Modules)
|
||||||
|
}
|
||||||
|
if _, err := letGo(ctx, open.inventory, catalogueWalk.ID, catalogue.DeliverySeat, "its turn"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
before := len(*asked)
|
||||||
|
advanceHeld(ctx, open)
|
||||||
|
if len(*asked) != before {
|
||||||
|
t.Fatalf("the catalogue walk started beside the controller's: asked %v", (*asked)[before:])
|
||||||
|
}
|
||||||
|
own2 := ws[0]
|
||||||
|
own2.State = inventory.PlanDone
|
||||||
|
if err := open.inventory.SavePlan(ctx, &own2); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
advanceHeld(ctx, open)
|
||||||
|
if len(*asked) < before+1 || (*asked)[before][1] != "modules/app" {
|
||||||
|
t.Fatalf("the catalogue walk did not start once the controller's ended: %v", (*asked)[before:])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// `plans` names a walk by what it moves (asked by the operator, 2026-10-10): the repository's pull request and
|
||||||
|
// title, the modules it moves and the machines running them, then the state words; a record naming no pull
|
||||||
|
// request is named by its commit, and a title is cut at fifty runes.
|
||||||
|
func TestAPlanLineNamesWhatItMoves(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)
|
||||||
|
p := inventory.Plan{ID: "plan-1", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1c1c1c1c1",
|
||||||
|
State: inventory.PlanBuilding, Tiers: [][]string{{"messenger", "telegram"}}, TierEntered: now.Add(-2 * time.Minute),
|
||||||
|
Modules: map[string]*inventory.PlanModule{"messenger": {State: "asked"}, "telegram": {State: "asked"}},
|
||||||
|
Commits: []inventory.PlanCommit{{Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1c1c1c1c1"}},
|
||||||
|
Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{{Repository: "novox/mesh-catalog",
|
||||||
|
Commit: "c1c1c1c1c1", Number: 175, Title: "a tap shows its outcome", Moves: []string{"telegram", "messenger"}}}}}
|
||||||
|
running := func(module string) []string {
|
||||||
|
if module == "messenger" || module == "telegram" {
|
||||||
|
return []string{"novox"}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if got, want := planLineOn(p, now, pauseView{}, tierAtLeast, running),
|
||||||
|
"mesh-catalog #175 a tap shows its outcome · messenger, telegram → novox · tier 1 of 1, building for 2m0s"; got != want {
|
||||||
|
t.Fatalf("the line reads %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
if got := planLine(p, now); !strings.HasPrefix(got, "mesh-catalog #175 a tap shows its outcome · messenger, telegram · tier") {
|
||||||
|
t.Fatalf("without the machines the line reads %q", got)
|
||||||
|
}
|
||||||
|
old := p
|
||||||
|
old.Commits, old.Delivery = nil, nil
|
||||||
|
if got := planLine(old, now); !strings.HasPrefix(got, "mesh-catalog c1c1c1c1 · tier 1 of 1") {
|
||||||
|
t.Fatalf("a record naming no pull request reads %q", got)
|
||||||
|
}
|
||||||
|
long := p
|
||||||
|
long.Delivery.Merges[0].Title = strings.Repeat("abcdefghij", 6)
|
||||||
|
if got := planHeadline(long, nil); !strings.Contains(got, "#175 "+strings.Repeat("abcdefghij", 4)+"abcdefghi…") {
|
||||||
|
t.Fatalf("a long title is not cut at fifty runes: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// sameMerge compares what a walk names of a merge: its repository, commit and the commit carrying it.
|
||||||
|
func sameMerge(a, b inventory.PlanMerge) bool {
|
||||||
|
return a.Repository == b.Repository && a.Commit == b.Commit && a.Carried == b.Carried
|
||||||
|
}
|
||||||
|
|
||||||
|
// A catalogue merge heard after a later merge of its branch was walked without the word (a merge that touched
|
||||||
|
// the bus too, on the controller's own path) is not answered by that walk while the delivery seat has a holder:
|
||||||
|
// its delivery would skip its turn. It joins the next catalogue batch.
|
||||||
|
func TestALateCatalogueMergeIsNotAnsweredByAnOwnPathWalk(t *testing.T) {
|
||||||
|
open := windowed(t)
|
||||||
|
asksWithPaths(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
for _, m := range []struct {
|
||||||
|
module string
|
||||||
|
claims []catalogue.Claim
|
||||||
|
}{
|
||||||
|
{"nats", nil},
|
||||||
|
{"mesh-delivery", []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
||||||
|
} {
|
||||||
|
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1", Claims: m.claims},
|
||||||
|
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + m.module, Ref: "main",
|
||||||
|
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
mixed := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "m1xed", MergedAt: t0.Format(time.RFC3339Nano),
|
||||||
|
Paths: []string{"modules/nats/module.json", "modules/app/module.json"}, ModuleDirs: []string{"modules/nats", "modules/app"},
|
||||||
|
ModuleDirsSaid: true}
|
||||||
|
hear(t, open, mixed, t0.Add(time.Second))
|
||||||
|
cutAt(t, open, t0.Add(2*time.Minute))
|
||||||
|
ws, _ := walks(t, open)
|
||||||
|
if len(ws) != 1 || ws[0].Waiting() {
|
||||||
|
t.Fatalf("the mixed merge's walk waited, or was not cut: %+v", ws)
|
||||||
|
}
|
||||||
|
done := ws[0]
|
||||||
|
done.State = inventory.PlanDone
|
||||||
|
if err := open.inventory.SavePlan(ctx, &done); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
hear(t, open, catalogueMerge("ear1ier", "app", t0.Add(-30*time.Second)), t0.Add(3*time.Minute))
|
||||||
|
got, _ := open.inventory.PlanByID(ctx, done.ID)
|
||||||
|
_, bs := walks(t, open)
|
||||||
|
if len(got.Delivery.Merges) != 1 || len(bs) != 1 || bs[0].OwnPath() || bs[0].Delivery.Merges[0].Commit != "ear1ier" {
|
||||||
|
t.Fatalf("the late catalogue merge was answered by the own-path walk (%+v) rather than the next catalogue batch (%+v)",
|
||||||
|
got.Delivery.Merges, bs)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,103 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **Every wait of a batch has a bound and a condition** (novox/hq ADR 0276 decision 9): a batch still
|
||||||
|
// assembling a minute past its maximum while no walk is open is a cut the controller failed to make (S18),
|
||||||
|
// and a batch waiting behind an open walk longer than that walk's bound waits on a walk gone wrong (S19).
|
||||||
|
|
||||||
|
// The kinds S18 and S19 raise.
|
||||||
|
const (
|
||||||
|
kindBatchNotCut = "batch-not-cut"
|
||||||
|
kindBatchBehindWalk = "batch-behind-walk"
|
||||||
|
)
|
||||||
|
|
||||||
|
// batchFacts is one batch not yet cut, as the watchdogs read it.
|
||||||
|
type batchFacts struct {
|
||||||
|
id, state, grouped string
|
||||||
|
// atMost is when its window closes at the latest; closed when it closed.
|
||||||
|
atMost, closed time.Time
|
||||||
|
// behind is the open walk it waits behind, and walkBound that walk's bound: a tier's bound for each of its
|
||||||
|
// tiers.
|
||||||
|
behind string
|
||||||
|
walkBound time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// gatherBatches is every batch not yet cut, with the bound of the walk it waits behind.
|
||||||
|
func gatherBatches(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]batchFacts, error) {
|
||||||
|
batches, err := inv.Batches(ctx)
|
||||||
|
if err != nil || len(batches) == 0 {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
bounds, err := measuredTierBounds(ctx, inv, now)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var out []batchFacts
|
||||||
|
for _, b := range batches {
|
||||||
|
f := batchFacts{id: b.ID, state: b.State, grouped: groupedWords(b)}
|
||||||
|
if w := b.Delivery; w != nil && w.Batch != nil {
|
||||||
|
f.atMost, f.closed, f.behind = w.Batch.AtMost, w.Batch.ClosesAt, w.Batch.Behind
|
||||||
|
if f.atMost.Before(f.closed) {
|
||||||
|
f.closed = f.atMost
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if f.behind != "" {
|
||||||
|
if walk, err := inv.PlanByID(ctx, f.behind); err == nil {
|
||||||
|
f.walkBound = bounds.of(walk.Repository) * time.Duration(max(1, len(walk.Tiers)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out = append(out, f)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// watchBatchesNotCut is S18: a batch still assembling a minute past its maximum, while no walk is open.
|
||||||
|
func watchBatchesNotCut(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, b := range f.batches {
|
||||||
|
if b.state != inventory.PlanAssembling || b.atMost.IsZero() || f.now.Sub(b.atMost) <= batchLateAfter {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
late := f.now.Sub(b.atMost)
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id, Kind: kindBatchNotCut,
|
||||||
|
Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("the batch %s is still assembling %s past its latest close (%s), with no walk open: "+
|
||||||
|
"the controller did not cut it; grouped: %s", b.id, ago(late), b.atMost.UTC().Format(time.RFC3339), b.grouped),
|
||||||
|
Said: fmt.Sprintf("assembling since %s past its maximum", ago(late)),
|
||||||
|
Headline: "Merged changes are not being delivered",
|
||||||
|
Explanation: fmt.Sprintf("Merges collected for one delivery should have been planned %s ago and were not. "+
|
||||||
|
"Nothing is lost; the mesh keeps them until it plans them.", humanDuration(late)),
|
||||||
|
Resolved: "Merged changes are being delivered again"})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// watchBatchesBehind is S19: a batch waiting behind an open walk longer than that walk's bound, naming it.
|
||||||
|
func watchBatchesBehind(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, b := range f.batches {
|
||||||
|
if b.state != inventory.PlanQueued || b.behind == "" || b.walkBound <= 0 || f.now.Sub(b.closed) <= b.walkBound {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
in := f.now.Sub(b.closed)
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id, Kind: kindBatchBehindWalk,
|
||||||
|
Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("the batch %s has waited %s behind the walk %s, longer than that walk's bound (%s); "+
|
||||||
|
"grouped: %s; `plans %s` says where that walk stands", b.id, ago(in), b.behind, ago(b.walkBound),
|
||||||
|
b.grouped, b.behind),
|
||||||
|
Said: fmt.Sprintf("queued behind %s for %s", b.behind, ago(in)),
|
||||||
|
Headline: "Merged changes wait behind a slow delivery",
|
||||||
|
Explanation: fmt.Sprintf("Merges collected for the next delivery have waited %s for the delivery before "+
|
||||||
|
"them, which is taking longer than it should. Nothing is lost.", humanDuration(in)),
|
||||||
|
Resolved: "Merged changes no longer wait behind a slow delivery"})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,207 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **A filling bucket or log is said before it is full** (novox/hq ADR 0297 §6, issue 501).
|
||||||
|
//
|
||||||
|
// A module's bucket and its log each have a cap on the bus, and when either is full the bus refuses
|
||||||
|
// the next write: the module stops doing what it writes for, and nothing said so before. On
|
||||||
|
// 2026-10-11 the issue tracker's bucket held a sixth of its cap after two days, growing toward a stop
|
||||||
|
// nobody would have heard coming. So the self-check reads every module's bucket and log, and one at
|
||||||
|
// fillRaiseAt of its cap or more raises a condition naming the module, the bucket or log, and how full
|
||||||
|
// it is.
|
||||||
|
//
|
||||||
|
// **Cleared by observation below fillClearBelow, not below fillRaiseAt**: a bucket or log hovering at
|
||||||
|
// its threshold would otherwise be raised and cleared on every run. Between the two, an open condition
|
||||||
|
// is kept and none is raised.
|
||||||
|
//
|
||||||
|
// **One that cannot be read is said, and the rest are still judged.** An unanswered question about one
|
||||||
|
// stream is no reason to know nothing of the others: it is a finding of its own, naming the bucket or
|
||||||
|
// log and why, and a fill condition already open for it is kept, because not knowing is not a pass.
|
||||||
|
|
||||||
|
// The fill thresholds, in percent of a bucket's or log's cap.
|
||||||
|
const (
|
||||||
|
fillRaiseAt = 75
|
||||||
|
fillClearBelow = 70
|
||||||
|
)
|
||||||
|
|
||||||
|
// The conditions the fill probe raises: one filling, and one that could not be read.
|
||||||
|
const (
|
||||||
|
kindBucketOrLogFilling = "bucket-or-log-filling"
|
||||||
|
kindBucketOrLogUnread = "bucket-or-log-unread"
|
||||||
|
)
|
||||||
|
|
||||||
|
// probeFillID is the probe's id in the registry.
|
||||||
|
const probeFillID = "D-fill"
|
||||||
|
|
||||||
|
// bucketOrLog is one module's bucket or log as the fill probe reads it.
|
||||||
|
type bucketOrLog struct {
|
||||||
|
Module string
|
||||||
|
// Kind is `bucket` or `log`; Name its local name; Stream the stream it is on the bus.
|
||||||
|
Kind, Name, Stream string
|
||||||
|
}
|
||||||
|
|
||||||
|
// filled is how full one bucket or log is, read from the bus.
|
||||||
|
type filled struct {
|
||||||
|
Bytes, Max uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
// percent is how full, in whole percent, rounded down.
|
||||||
|
func (f filled) percent() uint64 {
|
||||||
|
if f.Max == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return f.Bytes * 100 / f.Max
|
||||||
|
}
|
||||||
|
|
||||||
|
// fillKey is where a bucket's or log's fill condition is kept.
|
||||||
|
func fillKey(s bucketOrLog) string { return conditions.Key(conditions.ScopeBus, s.Stream, "filling") }
|
||||||
|
|
||||||
|
// fillObservation is what one bucket's or log's fill says: a finding at fillRaiseAt or above, and,
|
||||||
|
// while its condition is open, at fillClearBelow or above too; nothing otherwise, which is what clears
|
||||||
|
// it. The operator's words are the kind's (plain_words.go); the summary and the evidence name the
|
||||||
|
// module, the bucket or log, and the fill.
|
||||||
|
func fillObservation(s bucketOrLog, f filled, open bool) (conditions.Observation, bool) {
|
||||||
|
if f.Max == 0 {
|
||||||
|
return conditions.Observation{}, false // one with no cap cannot fill
|
||||||
|
}
|
||||||
|
// Compared in bytes, not rounded percent: 74.9% is not 75%.
|
||||||
|
atRaise := f.Bytes*100 >= f.Max*fillRaiseAt
|
||||||
|
aboveClear := f.Bytes*100 >= f.Max*fillClearBelow
|
||||||
|
if !atRaise && !(open && aboveClear) {
|
||||||
|
return conditions.Observation{}, false
|
||||||
|
}
|
||||||
|
pct := f.percent()
|
||||||
|
return conditions.Observation{
|
||||||
|
Scope: conditions.ScopeBus, ID: s.Stream, Token: "filling", Kind: kindBucketOrLogFilling,
|
||||||
|
Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s's %s %s holds %s of %s, %d%%: at its cap the bus refuses the module's next write",
|
||||||
|
s.Module, s.Kind, s.Name, mibWords(f.Bytes), mibWords(f.Max), pct),
|
||||||
|
Said: fmt.Sprintf("module %s, %s %s (stream %s): %d of %d bytes, %d%%", s.Module, s.Kind, s.Name,
|
||||||
|
s.Stream, f.Bytes, f.Max, pct),
|
||||||
|
}, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// unreadObservation says one bucket or log could not be read, and why. Asked over the network, so one
|
||||||
|
// look can be wrong: raised on the second look in a row (confirm.go).
|
||||||
|
func unreadObservation(s bucketOrLog, why error) conditions.Observation {
|
||||||
|
return conditions.Observation{
|
||||||
|
Scope: conditions.ScopeBus, ID: s.Stream, Token: "unread", Kind: kindBucketOrLogUnread,
|
||||||
|
Severity: conditions.Warning, Confirm: true,
|
||||||
|
Summary: fmt.Sprintf("%s's %s %s could not be read, so how full it is is not known", s.Module, s.Kind, s.Name),
|
||||||
|
Said: fmt.Sprintf("module %s, %s %s (stream %s): %v", s.Module, s.Kind, s.Name, s.Stream, why),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// keptFilling is an open fill condition said again for a bucket or log that could not be read this
|
||||||
|
// time: not knowing how full it is now is no reason to say it has room. Only ever made from a condition
|
||||||
|
// read back as open — its own summary and severity, never words made up for it.
|
||||||
|
func keptFilling(s bucketOrLog, open conditions.Condition, why error) conditions.Observation {
|
||||||
|
return conditions.Observation{
|
||||||
|
Scope: conditions.ScopeBus, ID: s.Stream, Token: "filling", Kind: kindBucketOrLogFilling,
|
||||||
|
Severity: open.Severity, Summary: open.Summary,
|
||||||
|
Said: fmt.Sprintf("module %s, %s %s (stream %s): not read this time (%v); kept open as it was",
|
||||||
|
s.Module, s.Kind, s.Name, s.Stream, why),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// mibWords is a size as a person reads it, in MiB with one decimal.
|
||||||
|
func mibWords(b uint64) string {
|
||||||
|
return fmt.Sprintf("%.1f MiB", float64(b)/(1024*1024))
|
||||||
|
}
|
||||||
|
|
||||||
|
// readFill is how full one bucket or log is on the bus; found false when it is not on the bus.
|
||||||
|
type readFill func(ctx context.Context, s bucketOrLog) (f filled, found bool, err error)
|
||||||
|
|
||||||
|
// openFill is the fill condition open under a key, if one is, or why it could not be read.
|
||||||
|
type openFill func(ctx context.Context, key string) (conditions.Condition, bool, error)
|
||||||
|
|
||||||
|
// judgeFills judges every bucket and log on its own: one that cannot be read is said as unread, with
|
||||||
|
// its fill condition kept only when that condition was read back and is open — when it cannot be read
|
||||||
|
// either, nothing is said of its fill, which the unread finding already covers. Every other is judged
|
||||||
|
// by its fill; for one of those, a condition that cannot be read is taken as open, so an unknown never
|
||||||
|
// clears a fill measured between fillClearBelow and fillRaiseAt.
|
||||||
|
func judgeFills(ctx context.Context, all []bucketOrLog, read readFill, open openFill) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, s := range all {
|
||||||
|
f, found, err := read(ctx, s)
|
||||||
|
if err != nil {
|
||||||
|
out = append(out, unreadObservation(s, err))
|
||||||
|
if c, isOpen, cerr := open(ctx, fillKey(s)); cerr == nil && isOpen {
|
||||||
|
out = append(out, keptFilling(s, c, err))
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
continue // not on the bus: created on the controller's next raise, and nothing there can fill
|
||||||
|
}
|
||||||
|
_, isOpen, cerr := open(ctx, fillKey(s))
|
||||||
|
isOpen = isOpen || cerr != nil
|
||||||
|
if o, said := fillObservation(s, f, isOpen); said {
|
||||||
|
out = append(out, o)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.SliceStable(out, func(i, j int) bool { return out[i].Key() < out[j].Key() })
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// declaredBucketsAndLogs is every module's bucket and log the catalogue declares, by its stream.
|
||||||
|
func declaredBucketsAndLogs(ctx context.Context, d *doctor) ([]bucketOrLog, error) {
|
||||||
|
buckets, err := d.open.inventory.DeclaredBuckets(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
logs, err := d.open.inventory.DeclaredLogs(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var out []bucketOrLog
|
||||||
|
for _, b := range buckets {
|
||||||
|
out = append(out, bucketOrLog{Module: b.Module, Kind: "bucket", Name: b.Name, Stream: "KV_" + b.Bucket()})
|
||||||
|
}
|
||||||
|
for _, l := range logs {
|
||||||
|
out = append(out, bucketOrLog{Module: l.Module, Kind: "log", Name: l.Name, Stream: l.Stream()})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeFill reads every module's bucket and log on the bus and says each one filling toward its cap,
|
||||||
|
// and each one that could not be read.
|
||||||
|
func probeFill(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
all, err := declaredBucketsAndLogs(ctx, d)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
js := d.js.Context()
|
||||||
|
read := func(ctx context.Context, s bucketOrLog) (filled, bool, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return filled{}, false, err
|
||||||
|
}
|
||||||
|
info, err := js.StreamInfo(s.Stream, nats.Context(ctx))
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, nats.ErrStreamNotFound):
|
||||||
|
return filled{}, false, nil
|
||||||
|
case err != nil:
|
||||||
|
return filled{}, false, err
|
||||||
|
case info.Config.MaxBytes <= 0:
|
||||||
|
return filled{}, true, nil
|
||||||
|
}
|
||||||
|
return filled{Bytes: info.State.Bytes, Max: uint64(info.Config.MaxBytes)}, true, nil
|
||||||
|
}
|
||||||
|
open := func(ctx context.Context, key string) (conditions.Condition, bool, error) {
|
||||||
|
if d.keeper == nil {
|
||||||
|
return conditions.Condition{}, false, nil
|
||||||
|
}
|
||||||
|
return d.keeper.Get(ctx, key)
|
||||||
|
}
|
||||||
|
return judgeFills(ctx, all, read, open), nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,165 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **A filling log or bucket is said at three quarters of its cap and cleared below seven tenths**
|
||||||
|
// (novox/hq ADR 0297 §6): raised at 75% naming the module, the bucket or log and its fill; not at 74.9%;
|
||||||
|
// kept between 70% and 75% while it is open, and not raised there when it is not; and cleared — said no
|
||||||
|
// more — once it reads below 70%, open or not.
|
||||||
|
func TestAFillingBucketOrLogIsRaisedAtThreeQuartersAndClearedBelowSevenTenths(t *testing.T) {
|
||||||
|
const mib = 1024 * 1024
|
||||||
|
log := bucketOrLog{Module: "mesh-issues", Kind: "log", Name: "changes", Stream: "LOG_mesh-issues_changes"}
|
||||||
|
bucket := bucketOrLog{Module: "mesh-issues", Kind: "bucket", Name: "issues", Stream: "KV_mesh-issues_issues"}
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
of bucketOrLog
|
||||||
|
bytes uint64
|
||||||
|
max uint64
|
||||||
|
open bool
|
||||||
|
said bool
|
||||||
|
}{
|
||||||
|
{"a log at exactly 75%", log, 768 * mib, 1024 * mib, false, true},
|
||||||
|
{"a bucket at exactly 75%", bucket, 48 * mib, 64 * mib, false, true},
|
||||||
|
{"a bucket full", bucket, 64 * mib, 64 * mib, false, true},
|
||||||
|
{"a log just under 75%", log, 768*mib - 1, 1024 * mib, false, false},
|
||||||
|
{"a bucket at 72%, not open", bucket, 64 * mib * 72 / 100, 64 * mib, false, false},
|
||||||
|
{"a bucket at 72%, open: kept", bucket, 64 * mib * 72 / 100, 64 * mib, true, true},
|
||||||
|
{"a log at exactly 70%, open: kept", log, 700 * mib, 1000 * mib, true, true},
|
||||||
|
{"a log just under 70%, open: cleared", log, 700*mib - 1, 1000 * mib, true, false},
|
||||||
|
{"a bucket at 10%, open: cleared", bucket, 64 * mib / 10, 64 * mib, true, false},
|
||||||
|
{"a bucket with no cap", bucket, 100, 0, true, false},
|
||||||
|
} {
|
||||||
|
o, said := fillObservation(c.of, filled{Bytes: c.bytes, Max: c.max}, c.open)
|
||||||
|
if said != c.said {
|
||||||
|
t.Errorf("%s: said %v, want %v", c.name, said, c.said)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !said {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if o.Key() != fillKey(c.of) || o.Kind != kindBucketOrLogFilling || o.Severity != conditions.Warning {
|
||||||
|
t.Errorf("%s: raised as %s (%s, %s)", c.name, o.Key(), o.Kind, o.Severity)
|
||||||
|
}
|
||||||
|
for _, part := range []string{c.of.Module, c.of.Kind + " " + c.of.Name, "%", " of "} {
|
||||||
|
if !strings.Contains(o.Summary, part) {
|
||||||
|
t.Errorf("%s: does not name %q: %q", c.name, part, o.Summary)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !strings.Contains(o.Said, "bytes") {
|
||||||
|
t.Errorf("%s: the evidence does not say the fill in bytes: %q", c.name, o.Said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
o, _ := fillObservation(log, filled{Bytes: 800 * mib, Max: 1024 * mib}, false)
|
||||||
|
if !strings.Contains(o.Summary, "800.0 MiB of 1024.0 MiB, 78%") {
|
||||||
|
t.Errorf("the fill is said as %q", o.Summary)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **One bucket or log that cannot be read does not stop the others being judged** (review of #231): it is
|
||||||
|
// said as unread with its reason, a fill condition open for it is kept, and every other bucket and log
|
||||||
|
// is judged by its own fill.
|
||||||
|
func TestAnUnreadableBucketOrLogIsSaidAndTheRestAreStillJudged(t *testing.T) {
|
||||||
|
const mib = 1024 * 1024
|
||||||
|
broken := bucketOrLog{Module: "a", Kind: "bucket", Name: "broken", Stream: "KV_a_broken"}
|
||||||
|
brokenOpen := bucketOrLog{Module: "a", Kind: "log", Name: "was-filling", Stream: "LOG_a_was-filling"}
|
||||||
|
full := bucketOrLog{Module: "b", Kind: "log", Name: "changes", Stream: "LOG_b_changes"}
|
||||||
|
empty := bucketOrLog{Module: "c", Kind: "bucket", Name: "quiet", Stream: "KV_c_quiet"}
|
||||||
|
absent := bucketOrLog{Module: "d", Kind: "bucket", Name: "not-yet", Stream: "KV_d_not-yet"}
|
||||||
|
refusal := errors.New("the bus did not answer")
|
||||||
|
read := func(_ context.Context, s bucketOrLog) (filled, bool, error) {
|
||||||
|
switch s {
|
||||||
|
case broken, brokenOpen:
|
||||||
|
return filled{}, false, refusal
|
||||||
|
case full:
|
||||||
|
return filled{Bytes: 900 * mib, Max: 1000 * mib}, true, nil
|
||||||
|
case empty:
|
||||||
|
return filled{Bytes: 1, Max: 64 * mib}, true, nil
|
||||||
|
}
|
||||||
|
return filled{}, false, nil
|
||||||
|
}
|
||||||
|
open := func(_ context.Context, key string) (conditions.Condition, bool, error) {
|
||||||
|
if key == fillKey(brokenOpen) {
|
||||||
|
return conditions.Condition{Key: key, Severity: conditions.Warning,
|
||||||
|
Summary: "a's log was-filling holds 800.0 MiB of 1024.0 MiB, 78%"}, true, nil
|
||||||
|
}
|
||||||
|
return conditions.Condition{}, false, nil
|
||||||
|
}
|
||||||
|
got := map[string]conditions.Observation{}
|
||||||
|
for _, o := range judgeFills(context.Background(), []bucketOrLog{broken, brokenOpen, full, empty, absent}, read, open) {
|
||||||
|
got[o.Key()] = o
|
||||||
|
}
|
||||||
|
for _, s := range []bucketOrLog{broken, brokenOpen} {
|
||||||
|
o, said := got[conditions.Key(conditions.ScopeBus, s.Stream, "unread")]
|
||||||
|
if !said || o.Kind != kindBucketOrLogUnread || !o.Confirm || !strings.Contains(o.Said, refusal.Error()) {
|
||||||
|
t.Errorf("%s could not be read and was said as %+v", s.Stream, o)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if o, kept := got[fillKey(brokenOpen)]; !kept || !strings.Contains(o.Said, "kept open") {
|
||||||
|
t.Errorf("an open fill condition of a log not read this time was not kept: %+v", o)
|
||||||
|
}
|
||||||
|
if _, said := got[fillKey(broken)]; said {
|
||||||
|
t.Error("a bucket not read and not filling was said filling")
|
||||||
|
}
|
||||||
|
if o, said := got[fillKey(full)]; !said || o.Kind != kindBucketOrLogFilling {
|
||||||
|
t.Errorf("a log at 90%% beside an unreadable one was not judged: %+v", got)
|
||||||
|
}
|
||||||
|
if len(got) != 4 {
|
||||||
|
t.Errorf("said %d findings, want 4 (two unread, one kept, one filling): %v", len(got), got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Both kinds have plain words of their own, which hold to the plain rule (novox/hq ADR 0253).
|
||||||
|
func TestAFillingOrUnreadBucketOrLogIsSaidInPlainWords(t *testing.T) {
|
||||||
|
for _, kind := range []string{kindBucketOrLogFilling, kindBucketOrLogUnread} {
|
||||||
|
wording, has := plainWordings[kind]
|
||||||
|
if !has {
|
||||||
|
t.Errorf("%s has no plain words", kind)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if why, ok := conditions.PlainWords(wording(conditions.Observation{Kind: kind})); !ok {
|
||||||
|
t.Errorf("%s: %s", kind, why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **When neither the bucket or log nor its condition can be read, nothing is said of its fill** (second
|
||||||
|
// review of #231): the unread finding covers it, and no fill is invented for it. For one that is read and
|
||||||
|
// measured between seven tenths and three quarters, a condition that cannot be read is taken as open, so
|
||||||
|
// an unknown never clears it.
|
||||||
|
func TestNothingIsSaidOfAFillWhenNeitherItNorItsConditionCanBeRead(t *testing.T) {
|
||||||
|
const mib = 1024 * 1024
|
||||||
|
broken := bucketOrLog{Module: "a", Kind: "log", Name: "changes", Stream: "LOG_a_changes"}
|
||||||
|
between := bucketOrLog{Module: "b", Kind: "bucket", Name: "issues", Stream: "KV_b_issues"}
|
||||||
|
read := func(_ context.Context, s bucketOrLog) (filled, bool, error) {
|
||||||
|
if s == broken {
|
||||||
|
return filled{}, false, errors.New("the bus did not answer")
|
||||||
|
}
|
||||||
|
return filled{Bytes: 72 * mib, Max: 100 * mib}, true, nil
|
||||||
|
}
|
||||||
|
open := func(context.Context, string) (conditions.Condition, bool, error) {
|
||||||
|
return conditions.Condition{}, false, errors.New("the conditions bucket did not answer")
|
||||||
|
}
|
||||||
|
got := map[string]conditions.Observation{}
|
||||||
|
for _, o := range judgeFills(context.Background(), []bucketOrLog{broken, between}, read, open) {
|
||||||
|
got[o.Key()] = o
|
||||||
|
}
|
||||||
|
if o, said := got[fillKey(broken)]; said {
|
||||||
|
t.Fatalf("a fill was said for a log whose fill and condition could not be read: %+v", o)
|
||||||
|
}
|
||||||
|
if _, said := got[conditions.Key(conditions.ScopeBus, broken.Stream, "unread")]; !said {
|
||||||
|
t.Error("the log that could not be read was not said as unread")
|
||||||
|
}
|
||||||
|
if _, kept := got[fillKey(between)]; !kept {
|
||||||
|
t.Error("a bucket at 72% whose condition could not be read was cleared")
|
||||||
|
}
|
||||||
|
if len(got) != 2 {
|
||||||
|
t.Errorf("said %d findings, want 2: %v", len(got), got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -166,6 +166,10 @@ func buildFrom(result link.BuildResult) inventory.Build {
|
|||||||
for _, r := range result.Read {
|
for _, r := range result.Read {
|
||||||
kept.Read = append(kept.Read, inventory.ReadRepository{Repository: r.Repository, Ref: r.Ref})
|
kept.Read = append(kept.Read, inventory.ReadRepository{Repository: r.Repository, Ref: r.Ref})
|
||||||
}
|
}
|
||||||
|
// What it was made from, as files (novox/hq ADR 0267): the planner maps the next merge onto it.
|
||||||
|
for _, s := range result.Sources {
|
||||||
|
kept.Sources = append(kept.Sources, inventory.BuildSource{Repository: s.Repository, Ref: s.Ref, Paths: s.Paths})
|
||||||
|
}
|
||||||
var announced []inventory.Artifact
|
var announced []inventory.Artifact
|
||||||
for _, made := range result.Made {
|
for _, made := range result.Made {
|
||||||
announced = append(announced, inventory.Artifact{
|
announced = append(announced, inventory.Artifact{
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"hash/fnv"
|
"hash/fnv"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
@@ -11,6 +12,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
@@ -208,7 +210,9 @@ func TestARollbackNeverPutsBackABuildFromAnotherRepository(t *testing.T) {
|
|||||||
if _, _, err := takeIn(ctx, inv, fork); !errors.Is(err, errNotItsSource) {
|
if _, _, err := takeIn(ctx, inv, fork); !errors.Is(err, errNotItsSource) {
|
||||||
t.Fatalf("the fork's build was taken in: %v", err)
|
t.Fatalf("the fork's build was taken in: %v", err)
|
||||||
}
|
}
|
||||||
failed := onTrunk("build-1791600000000000000", "novox/mesh-catalog", "git", "modules/sudo",
|
// Asked after the registered build, whenever the test runs: an id naming a fixed moment read as older
|
||||||
|
// than the registered build once the clock passed it (2026-10-10 02:40 UTC), and the test failed on main.
|
||||||
|
failed := onTrunk(fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano()), "novox/mesh-catalog", "git", "modules/sudo",
|
||||||
map[string]any{"module": "sudo", "version": "2"})
|
map[string]any{"module": "sudo", "version": "2"})
|
||||||
failed.Commit = "badbadbad0123456"
|
failed.Commit = "badbadbad0123456"
|
||||||
if _, _, err := takeIn(ctx, inv, failed); err != nil {
|
if _, _, err := takeIn(ctx, inv, failed); err != nil {
|
||||||
|
|||||||
@@ -32,6 +32,9 @@ import (
|
|||||||
// provision it wants that a manifest given here offers. An overflow is refused in the pull request
|
// provision it wants that a manifest given here offers. An overflow is refused in the pull request
|
||||||
// that introduces it — a new requirement, a lowered bound, a longer slug — instead of on the
|
// that introduces it — a new requirement, a lowered bound, a longer slug — instead of on the
|
||||||
// provider's machine when a real machine's name first meets the module's.
|
// provider's machine when a real machine's name first meets the module's.
|
||||||
|
// SomeManifestsVar, set by the merge gate, says the manifests given are only some of their repository's.
|
||||||
|
const SomeManifestsVar = "MESH_MODULE_CHECK_SOME"
|
||||||
|
|
||||||
func moduleCheck(paths []string, out io.Writer) error {
|
func moduleCheck(paths []string, out io.Writer) error {
|
||||||
return moduleCheckFor(paths, catalogue.DefaultLongestMachine, out)
|
return moduleCheckFor(paths, catalogue.DefaultLongestMachine, out)
|
||||||
}
|
}
|
||||||
@@ -85,12 +88,24 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
|||||||
// Between the manifests: a seat declared twice, a use of a seat nothing declares, a claim on
|
// Between the manifests: a seat declared twice, a use of a seat nothing declares, a claim on
|
||||||
// a seat that does not exist. Run only over what parsed, because a problem inside one manifest
|
// a seat that does not exist. Run only over what parsed, because a problem inside one manifest
|
||||||
// has already been said and would be said again here in a worse form.
|
// has already been said and would be said again here in a worse form.
|
||||||
|
//
|
||||||
|
// **Over some of a repository's manifests, a seat none of them declares is a note** (novox/hq issue 364), as
|
||||||
|
// this command's own word says above: the merge gate passes only the manifests a change touches, and says so
|
||||||
|
// with SomeManifestsVar, so a module that uses or claims a seat another module declares (the operator
|
||||||
|
// channel's, a channel bench) was refused there for a manifest it was not given. Given every manifest — the
|
||||||
|
// catalogue's own check, and registration — it stays a refusal.
|
||||||
|
some := os.Getenv(SomeManifestsVar) != ""
|
||||||
problems := catalogue.CatalogueProblems(shelf)
|
problems := catalogue.CatalogueProblems(shelf)
|
||||||
sort.Strings(problems)
|
sort.Strings(problems)
|
||||||
for _, p := range problems {
|
for _, p := range problems {
|
||||||
|
if some && catalogue.IsUndeclaredSeat(p) {
|
||||||
|
fmt.Fprintf(out, "note: %s among the manifests given; registration judges it against the whole catalogue, "+
|
||||||
|
"and passing the declaring module's manifest too judges it here\n", p)
|
||||||
|
continue
|
||||||
|
}
|
||||||
fmt.Fprintln(out, p)
|
fmt.Fprintln(out, p)
|
||||||
|
failed++
|
||||||
}
|
}
|
||||||
failed += len(problems)
|
|
||||||
|
|
||||||
// Between the manifests too: an identity against the bounds of the provisions it wants, which
|
// Between the manifests too: an identity against the bounds of the provisions it wants, which
|
||||||
// only the provider's manifest states.
|
// only the provider's manifest states.
|
||||||
@@ -164,6 +179,14 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
|||||||
}
|
}
|
||||||
fmt.Fprintf(out, ", keeps state %s", strings.Join(kept, ", "))
|
fmt.Fprintf(out, ", keeps state %s", strings.Join(kept, ", "))
|
||||||
}
|
}
|
||||||
|
// And the logs it keeps, with their caps (novox/hq ADR 0297).
|
||||||
|
if len(m.Logs) > 0 {
|
||||||
|
kept := make([]string, 0, len(m.Logs))
|
||||||
|
for _, l := range m.Logs {
|
||||||
|
kept = append(kept, fmt.Sprintf("%s (%d MiB)", l.Name, l.Cap()))
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, ", keeps log %s", strings.Join(kept, ", "))
|
||||||
|
}
|
||||||
if len(m.Reads) > 0 {
|
if len(m.Reads) > 0 {
|
||||||
fmt.Fprintf(out, ", reads %s", strings.Join(m.Reads, ", "))
|
fmt.Fprintf(out, ", reads %s", strings.Join(m.Reads, ", "))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -82,7 +82,9 @@ func checkHereCommand(ctx context.Context, args []string) error {
|
|||||||
if _, err := git("fetch", "--quiet", "origin", *base); err != nil {
|
if _, err := git("fetch", "--quiet", "origin", *base); err != nil {
|
||||||
return fmt.Errorf("cannot fetch %s to say what the change touches: %w", *base, err)
|
return fmt.Errorf("cannot fetch %s to say what the change touches: %w", *base, err)
|
||||||
}
|
}
|
||||||
changedText, err := git("diff", "--name-only", "origin/"+*base+"...HEAD")
|
// Without rename detection, so a file moved out of a build source is said under its old name as well:
|
||||||
|
// its going is a change to the build that held it (novox/hq ADR 0267).
|
||||||
|
changedText, err := git("diff", "--name-only", "--no-renames", "origin/"+*base+"...HEAD")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,8 +6,10 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/beside"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The check anybody can run is the check registration runs (novox/hq issue 148, ADR 0037): a manifest
|
// The check anybody can run is the check registration runs (novox/hq issue 148, ADR 0037): a manifest
|
||||||
@@ -56,10 +58,7 @@ func TestModuleCheckJudgesBetweenTheManifestsGiven(t *testing.T) {
|
|||||||
|
|
||||||
// The real catalogue passes the command, the way it passes the test that used to be the only check.
|
// The real catalogue passes the command, the way it passes the test that used to be the only check.
|
||||||
func TestModuleCheckPassesTheCatalogue(t *testing.T) {
|
func TestModuleCheckPassesTheCatalogue(t *testing.T) {
|
||||||
root := filepath.Join("..", "..", "..", "mesh-catalog", "modules")
|
root := beside.Catalogue(t)
|
||||||
if _, err := os.Stat(root); err != nil {
|
|
||||||
t.Skipf("catalogue sibling not present: %v", err)
|
|
||||||
}
|
|
||||||
paths, err := manifestsUnder(root)
|
paths, err := manifestsUnder(root)
|
||||||
if err != nil || len(paths) == 0 {
|
if err != nil || len(paths) == 0 {
|
||||||
t.Fatalf("no manifests under %s: %v", root, err)
|
t.Fatalf("no manifests under %s: %v", root, err)
|
||||||
@@ -103,3 +102,23 @@ func TestTheControllersManifestServesEveryVerbOfItsSeat(t *testing.T) {
|
|||||||
t.Fatalf("the controller's own module.json fails module check: %v\n%s", err, out.String())
|
t.Fatalf("the controller's own module.json fails module check: %v\n%s", err, out.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A module that uses a seat another module declares (novox/hq issue 364): refused over the whole catalogue when the
|
||||||
|
// declarer is missing, a note when the gate says it gives only the manifests a change touches.
|
||||||
|
func TestASeatAnotherModuleDeclaresIsANoteOverSomeManifests(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
user := filepath.Join(dir, "user.json")
|
||||||
|
os.WriteFile(user, []byte(`{"module":"asker","version":"1","uses":["operator-channel"]}`), 0o600)
|
||||||
|
var out bytes.Buffer
|
||||||
|
if err := moduleCheck([]string{user}, &out); err == nil {
|
||||||
|
t.Fatalf("a use of a seat nothing given declares passed the whole-catalogue check:\n%s", out.String())
|
||||||
|
}
|
||||||
|
t.Setenv(SomeManifestsVar, "1")
|
||||||
|
out.Reset()
|
||||||
|
if err := moduleCheck([]string{user}, &out); err != nil {
|
||||||
|
t.Fatalf("over some manifests the use was refused:\n%s", out.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(out.String(), "note: asker uses the seat \"operator-channel\"") {
|
||||||
|
t.Fatalf("the note was not said:\n%s", out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -5,6 +5,8 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"path"
|
"path"
|
||||||
|
"regexp"
|
||||||
|
"runtime/debug"
|
||||||
"slices"
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -143,7 +145,7 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
read, err := inv.ReadRepositories(ctx)
|
read, err := readForPlanning(ctx, inv)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -246,9 +248,11 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
|
|||||||
if dir == "mesh-controller" {
|
if dir == "mesh-controller" {
|
||||||
beside["mesh-controller-main"] = link.CheckedOut{Repository: url, Ref: refs["mesh-controller-main"]}
|
beside["mesh-controller-main"] = link.CheckedOut{Repository: url, Ref: refs["mesh-controller-main"]}
|
||||||
if e.Source.Seat != "" {
|
if e.Source.Seat != "" {
|
||||||
if lab, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository,
|
for _, sibling := range []string{"mesh-lab", "mesh-sdk"} {
|
||||||
"mesh-lab")}); err == nil {
|
if url, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository,
|
||||||
beside["mesh-lab"] = link.CheckedOut{Repository: lab, Ref: refs["mesh-lab"]}
|
sibling)}); err == nil {
|
||||||
|
beside[sibling] = link.CheckedOut{Repository: url, Ref: refs[sibling]}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -271,17 +275,54 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
|
|||||||
// catalogue, whose checkout beside is what tests read its files from, at its main, what the next merge
|
// catalogue, whose checkout beside is what tests read its files from, at its main, what the next merge
|
||||||
// builds from; and beside the controller its main, for a judge the running controller predates, and the
|
// builds from; and beside the controller its main, for a judge the running controller predates, and the
|
||||||
// lab's main, whose replays every check runs. **One rule, read by the check the controller asks for and by
|
// lab's main, whose replays every check runs. **One rule, read by the check the controller asks for and by
|
||||||
// the facts snapshot** (Facts.Beside), so a check run by hand clones what the build seat clones.
|
// the facts snapshot** (Facts.Beside), so a check run by hand clones what the build seat clones. Beside the
|
||||||
|
// controller also the SDK, checked out at the commit the running controller's go.mod pins (sdkPinned), not
|
||||||
|
// one a desktop holds (novox/hq issue 449). The checkout only places the clone: the conformance test reads the
|
||||||
|
// fixtures at the pin of the tree under check, from the clone's history, so a pull request moving the SDK is
|
||||||
|
// judged against the SDK it moves to (internal/link/conformance_test.go).
|
||||||
func besideRefs(dir, running string) map[string]string {
|
func besideRefs(dir, running string) map[string]string {
|
||||||
switch dir {
|
switch dir {
|
||||||
case "mesh-catalog":
|
case "mesh-catalog":
|
||||||
return map[string]string{dir: "main"}
|
return map[string]string{dir: "main"}
|
||||||
case "mesh-controller":
|
case "mesh-controller":
|
||||||
return map[string]string{dir: running, "mesh-controller-main": "main", "mesh-lab": "main"}
|
return map[string]string{dir: running, "mesh-controller-main": "main", "mesh-lab": "main",
|
||||||
|
"mesh-sdk": sdkPinned()}
|
||||||
}
|
}
|
||||||
return map[string]string{dir: running}
|
return map[string]string{dir: running}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// sdkModule is the Go module of the SDK the controller is built against.
|
||||||
|
const sdkModule = "git.novox.be/novox/mesh-sdk/go"
|
||||||
|
|
||||||
|
// pseudoCommit is the commit a Go pseudo-version names: v0.1.11-0.20261009143344-f047d0a4a970 → f047d0a4a970.
|
||||||
|
var pseudoCommit = regexp.MustCompile(`-([0-9a-f]{12})$`)
|
||||||
|
|
||||||
|
// sdkPinned is the ref of the SDK repository this controller was built from, as its go.mod pins it and its
|
||||||
|
// build records it: a pseudo-version's commit, or a release's tag (the SDK tags its Go module under go/).
|
||||||
|
// "main" only for a binary that records no SDK version — a local replace — which a running controller is
|
||||||
|
// not (novox/hq issue 449).
|
||||||
|
func sdkPinned() string {
|
||||||
|
info, ok := debug.ReadBuildInfo()
|
||||||
|
if !ok {
|
||||||
|
return "main"
|
||||||
|
}
|
||||||
|
for _, dep := range info.Deps {
|
||||||
|
if dep.Path != sdkModule {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if dep.Replace != nil {
|
||||||
|
dep = dep.Replace
|
||||||
|
}
|
||||||
|
if m := pseudoCommit.FindStringSubmatch(dep.Version); m != nil {
|
||||||
|
return m[1]
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(dep.Version, "v") {
|
||||||
|
return "go/" + dep.Version
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "main"
|
||||||
|
}
|
||||||
|
|
||||||
// siblingOf is another repository of the same owner: novox/mesh-controller → novox/mesh-lab.
|
// siblingOf is another repository of the same owner: novox/mesh-controller → novox/mesh-lab.
|
||||||
func siblingOf(repository, name string) string {
|
func siblingOf(repository, name string) string {
|
||||||
if cut := strings.LastIndex(repository, "/"); cut >= 0 {
|
if cut := strings.LastIndex(repository, "/"); cut >= 0 {
|
||||||
|
|||||||
@@ -169,6 +169,44 @@ func TestAShrinkOfMoreThanHalfIsUrgent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// THE FALSE ALARM (issue 368), replayed through the store D13 reads: an agent's home moved from the
|
||||||
|
// operator's own home (94.7 MB) to the agent account's fresh one (490 B), and `data-shrank` was raised
|
||||||
|
// for data that was never lost. A moved item is read against its new path only, so nothing is raised —
|
||||||
|
// and a genuine shrink at the new path, a week of history later, still is.
|
||||||
|
func TestAMovedPathIsNoShrinkAndAShrinkThereStillIs(t *testing.T) {
|
||||||
|
inv := inventory.ForTest(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
shelf := shelfFor(t, houseManifest)
|
||||||
|
declared := []inventory.DeclaredData{{Module: "house", Item: "config", Class: "irreplaceable", Owned: true}}
|
||||||
|
start := time.Now().Add(-6 * time.Hour)
|
||||||
|
measure := func(at time.Time, path string, size int64) []conditions.Observation {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := inv.RecordData(ctx, "home", declared, map[string]map[string]inventory.Measurement{"house": {
|
||||||
|
"config": {Path: path, Size: bytesOf(size), MeasuredAt: when(at), LastWrite: when(at),
|
||||||
|
LastBackup: when(at)}}}, "", at); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
records, err := inv.Data(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
peaks, err := inv.DataPeaks(ctx, at.Add(-shrinkWindow))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return dataFindings(records, peaks, shelf, nil, nil, at)
|
||||||
|
}
|
||||||
|
measure(start, "/home/operator/.claude", 94_700_000)
|
||||||
|
if got := findingsByKind(measure(start.Add(10*time.Minute), "/home/agent/.claude", 490)); got[kindDataShrank].Kind != "" {
|
||||||
|
t.Fatalf("a moved path raised a shrink: %+v", got[kindDataShrank])
|
||||||
|
}
|
||||||
|
measure(start.Add(2*time.Hour), "/home/agent/.claude", 300<<20)
|
||||||
|
got := findingsByKind(measure(start.Add(4*time.Hour), "/home/agent/.claude", 1<<20))[kindDataShrank]
|
||||||
|
if got.Severity != conditions.Urgent || !strings.Contains(got.Summary, "shrank") {
|
||||||
|
t.Fatalf("a genuine shrink at the new path was not raised: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Data said to be written all the time and not written; data with no backup or an old one — urgent when
|
// Data said to be written all the time and not written; data with no backup or an old one — urgent when
|
||||||
// irreplaceable, a warning when valuable; and a new item given its bound before it is said.
|
// irreplaceable, a warning when valuable; and a new item given its bound before it is said.
|
||||||
func TestQuietDataAndMissingBackupsAreSaidByClass(t *testing.T) {
|
func TestQuietDataAndMissingBackupsAreSaidByClass(t *testing.T) {
|
||||||
|
|||||||
@@ -144,11 +144,15 @@ func actOnDeadLetter(ctx context.Context, on *busHandles, act string, id uint64,
|
|||||||
}
|
}
|
||||||
switch act {
|
switch act {
|
||||||
case "deliver":
|
case "deliver":
|
||||||
_, to, err := link.DeliverAgain(on.js, id)
|
delivered, to, err := link.DeliverAgain(on.js, id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
answer["delivered_on"] = to
|
answer["delivered_on"] = to
|
||||||
|
if delivered.Original != "" {
|
||||||
|
// What became of the ask in its seat's queue (novox/hq issue 334): removed, or left, and why.
|
||||||
|
answer["original"] = delivered.Original
|
||||||
|
}
|
||||||
answer["done"] = fmt.Sprintf("dead letter %d was delivered again to %s, and nobody else; it is no longer kept",
|
answer["done"] = fmt.Sprintf("dead letter %d was delivered again to %s, and nobody else; it is no longer kept",
|
||||||
id, consumerWho(d.Stream, d.Consumer))
|
id, consumerWho(d.Stream, d.Consumer))
|
||||||
case "drop":
|
case "drop":
|
||||||
|
|||||||
@@ -103,6 +103,8 @@ func letGo(ctx context.Context, inv *inventory.Inventory, id, by, why string) (i
|
|||||||
return p, fmt.Errorf("%s was let go by %s at %s already", p.ID, p.Delivery.By,
|
return p, fmt.Errorf("%s was let go by %s at %s already", p.ID, p.Delivery.By,
|
||||||
p.Delivery.Go.Local().Format("15:04:05"))
|
p.Delivery.Go.Local().Format("15:04:05"))
|
||||||
}
|
}
|
||||||
|
// **One walk at a time** (novox/hq ADR 0276): the word is taken, and the walk starts once no other walk is
|
||||||
|
// started (advanceOnce), so the delivery's owner says it once and is not refused.
|
||||||
now := time.Now().UTC()
|
now := time.Now().UTC()
|
||||||
p.Delivery.Go, p.Delivery.By, p.Delivery.Why = &now, by, why
|
p.Delivery.Go, p.Delivery.By, p.Delivery.Why = &now, by, why
|
||||||
p.Note = "let go by " + by + "; its first tier is asked next"
|
p.Note = "let go by " + by + "; its first tier is asked next"
|
||||||
@@ -574,6 +576,12 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
|||||||
if walks, err = inv.OpenPlans(ctx); err != nil {
|
if walks, err = inv.OpenPlans(ctx); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
// And the batch being assembled (novox/hq ADR 0276): never a walk to link or let go, shown.
|
||||||
|
batches, err := inv.Batches(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
walks = append(walks, batches...)
|
||||||
recent, err := inv.RecentPlans(ctx, *limit)
|
recent, err := inv.RecentPlans(ctx, *limit)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -588,12 +596,81 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
// Each walk's phases, with the rest's reports (novox/hq ADR 0282 decision 6).
|
||||||
|
now := time.Now()
|
||||||
|
for i := range walks {
|
||||||
|
walks[i].Phases = walks[i].WalkPhases(now, appliedFrom(ctx, inv))
|
||||||
|
}
|
||||||
return answer(map[string]any{"held": deliverySeatHeld(entries), "walks": walks,
|
return answer(map[string]any{"held": deliverySeatHeld(entries), "walks": walks,
|
||||||
"own-path": sortedKeysOf(ownPathWords())})
|
"own-path": sortedKeysOf(ownPathWords())})
|
||||||
}
|
}
|
||||||
return fmt.Errorf("delivery %s: plan, order, check, go, stop or walks", sub)
|
return fmt.Errorf("delivery %s: plan, order, check, go, stop or walks", sub)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// appliedFrom answers a machine's first report after a send from the controller's `apply` durations; a lookup
|
||||||
|
// that fails is a report not read, which leaves the walk's end unknown rather than wrong.
|
||||||
|
func appliedFrom(ctx context.Context, inv *inventory.Inventory) inventory.AppliedLookup {
|
||||||
|
return func(node string, sent time.Time) (inventory.AppliedReport, bool) {
|
||||||
|
r, ok, err := inv.FirstAppliedAfter(ctx, node, sent)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "the report of %s after %s could not be read: %v\n", node, sent.Format(time.RFC3339), err)
|
||||||
|
return inventory.AppliedReport{}, false
|
||||||
|
}
|
||||||
|
return r, ok
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// recordWalkPhases keeps, once, each phase of every walk ended lately whose end is known, as a duration of kind
|
||||||
|
// walk-phase per class (novox/hq ADR 0282 decision 6): what `durations` summarises. A walk whose end is unknown
|
||||||
|
// past ApplySilentAfter keeps its measured phases without its total.
|
||||||
|
func recordWalkPhases(ctx context.Context, inv *inventory.Inventory, now time.Time) error {
|
||||||
|
recent, err := inv.RecentPlans(ctx, 30)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, p := range recent {
|
||||||
|
if p.State != inventory.PlanDone || p.Release != nil || now.Sub(p.Updated) > 2*time.Hour {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
anyKept, totalKept, err := inv.WalkPhasesKept(ctx, p.ID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if totalKept {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ph := p.WalkPhases(now, appliedFrom(ctx, inv))
|
||||||
|
if ph != nil && ph.End == nil && anyKept {
|
||||||
|
continue // kept without its end; kept again only once its end is known
|
||||||
|
}
|
||||||
|
if ph == nil || (ph.End == nil && now.Sub(p.Updated) < inventory.ApplySilentAfter+time.Minute) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
class := ph.Class
|
||||||
|
if class == "" {
|
||||||
|
class = "unclassed"
|
||||||
|
}
|
||||||
|
for _, x := range ph.Phases {
|
||||||
|
if x.State != inventory.PhaseMeasured || x.Start == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := inv.RecordDuration(ctx, inventory.Duration{Kind: inventory.DurationWalkPhase,
|
||||||
|
Subject: class + "/" + x.Name, Ref: fmt.Sprintf("%s/%s/%d", p.ID, x.Name, x.Tier), Started: *x.Start,
|
||||||
|
Took: time.Duration(x.TookMS) * time.Millisecond, Detail: p.Named()}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ph.End != nil && ph.From != nil {
|
||||||
|
if err := inv.RecordDuration(ctx, inventory.Duration{Kind: inventory.DurationWalkPhase,
|
||||||
|
Subject: class + "/total", Ref: p.ID + "/total", Started: *ph.From,
|
||||||
|
Took: time.Duration(ph.TotalMS) * time.Millisecond, Detail: ph.Said}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// ownPathWords is the controller's own path as words, for an answer.
|
// ownPathWords is the controller's own path as words, for an answer.
|
||||||
func ownPathWords() map[string]string { return onTheControllersPath }
|
func ownPathWords() map[string]string { return onTheControllersPath }
|
||||||
|
|
||||||
@@ -604,7 +681,7 @@ func theGraph(ctx context.Context, inv *inventory.Inventory) ([]inventory.Entry,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, nil, err
|
return nil, nil, nil, err
|
||||||
}
|
}
|
||||||
read, err := inv.ReadRepositories(ctx)
|
read, err := readForPlanning(ctx, inv)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, nil, err
|
return nil, nil, nil, err
|
||||||
}
|
}
|
||||||
@@ -720,6 +797,9 @@ func sayPlanMoved(ctx context.Context, bus link.Bus, p inventory.Plan) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func publishPlanMoved(ctx context.Context, bus link.Bus, p inventory.Plan) {
|
func publishPlanMoved(ctx context.Context, bus link.Bus, p inventory.Plan) {
|
||||||
|
// Its phases so far (novox/hq ADR 0282 decision 6): the rest's reports come after the walk ends, and are
|
||||||
|
// read by whoever asks for the walk (`delivery walks`).
|
||||||
|
p.Phases = p.WalkPhases(time.Now(), nil)
|
||||||
body, err := json.Marshal(p)
|
body, err := json.Marshal(p)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -37,12 +37,34 @@ type stalledLine struct {
|
|||||||
// Number is the pull request's, for a line of a head the forge never announced (novox/hq issue 347).
|
// Number is the pull request's, for a line of a head the forge never announced (novox/hq issue 347).
|
||||||
Number int `json:"number,omitempty"`
|
Number int `json:"number,omitempty"`
|
||||||
State string `json:"state"`
|
State string `json:"state"`
|
||||||
|
// Waiting are the merge checks a line of the state `unanswered` waits on, as mesh-delivery says each:
|
||||||
|
// "mesh/merge-gate pending since <UTC time>", "mesh/repo-check never set" (novox/hq issue 438).
|
||||||
|
Waiting []string `json:"waiting,omitempty"`
|
||||||
|
// Checks are the same merge checks as data, which the controller words in the operator's own time (novox/hq
|
||||||
|
// issue 443): a time inside a finished sentence cannot be said again in another zone. A mesh-delivery from before
|
||||||
|
// says none, and Waiting is said as it reads.
|
||||||
|
Checks []waitingCheck `json:"checks,omitempty"`
|
||||||
For string `json:"for"`
|
For string `json:"for"`
|
||||||
Bound string `json:"bound"`
|
Bound string `json:"bound"`
|
||||||
H2 string `json:"h2"`
|
H2 string `json:"h2"`
|
||||||
Says string `json:"says"`
|
Says string `json:"says"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// waitingCheck is one merge check a stalled line waits on, as mesh-delivery gives it: its context, its state —
|
||||||
|
// "pending", or "never-set" — and, for a pending one, since when in RFC 3339, empty when the forge did not say.
|
||||||
|
type waitingCheck struct {
|
||||||
|
Context string `json:"context"`
|
||||||
|
State string `json:"state"`
|
||||||
|
Since string `json:"since,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// wordsNow is the time the words are said at, which says whether a time needs its day; a seam a test replaces.
|
||||||
|
var wordsNow = time.Now
|
||||||
|
|
||||||
|
// wordsZone is the zone a stalled line's times are said in: the controller's local zone, as every other time in its
|
||||||
|
// messages. A seam a test replaces, so that no test writes time.Local, which every goroutine of the package reads.
|
||||||
|
var wordsZone = func() *time.Location { return time.Local }
|
||||||
|
|
||||||
// operatorsOnly is whether the table leaves H2 nothing to do for the line: the state is the operator's.
|
// operatorsOnly is whether the table leaves H2 nothing to do for the line: the state is the operator's.
|
||||||
func (l stalledLine) operatorsOnly() bool { return l.H2 == "" || strings.HasPrefix(l.H2, "none") }
|
func (l stalledLine) operatorsOnly() bool { return l.H2 == "" || strings.HasPrefix(l.H2, "none") }
|
||||||
|
|
||||||
|
|||||||
@@ -227,7 +227,9 @@ func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testin
|
|||||||
t.Fatalf("with no holder on record the walk waited (%v) or asked %v", p.Waiting(), *asked)
|
t.Fatalf("with no holder on record the walk waited (%v) or asked %v", p.Waiting(), *asked)
|
||||||
}
|
}
|
||||||
|
|
||||||
// The holder on record: the next merge waits, asking nothing, and an advance asks nothing either.
|
// The holder on record: the next merge waits, asking nothing, and an advance asks nothing either. One walk
|
||||||
|
// is open at a time (novox/hq ADR 0276): the first ends before the next merge's batch is cut.
|
||||||
|
finish(p.ID)
|
||||||
if _, err := inv.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
if _, err := inv.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -259,10 +261,11 @@ func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testin
|
|||||||
t.Fatalf("the word was not kept: %+v %v", got.Delivery, err)
|
t.Fatalf("the word was not kept: %+v %v", got.Delivery, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// The delivery's owner's own merge never waits for it — and takes over what the older walk had not
|
// The delivery's owner's own merge never waits for it. A walk that started is never taken over (novox/hq
|
||||||
// built (app, folded in: ADR 0218), which goes with it on the controller's own path.
|
// ADR 0276): the merge's batch is cut once it ended.
|
||||||
|
finish(p.ID)
|
||||||
p = merge("c3cccccccc", "modules/mesh-delivery/main.go")
|
p = merge("c3cccccccc", "modules/mesh-delivery/main.go")
|
||||||
if p.Waiting() || len(*asked) != 4 {
|
if p.Waiting() || len(*asked) != 3 {
|
||||||
t.Fatalf("mesh-delivery's own walk waited for mesh-delivery: %v %v", p.Waiting(), *asked)
|
t.Fatalf("mesh-delivery's own walk waited for mesh-delivery: %v %v", p.Waiting(), *asked)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -280,7 +283,7 @@ func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testin
|
|||||||
}
|
}
|
||||||
advanceHeld(ctx, open)
|
advanceHeld(ctx, open)
|
||||||
got, _ = inv.PlanByID(ctx, p.ID)
|
got, _ = inv.PlanByID(ctx, p.ID)
|
||||||
if got.Waiting() || !strings.HasPrefix(got.Delivery.By, "a person") || len(*asked) < 5 {
|
if got.Waiting() || !strings.HasPrefix(got.Delivery.By, "a person") || len(*asked) < 4 {
|
||||||
t.Fatalf("a person's word did not start the walk: %+v, asked %v", got.Delivery, *asked)
|
t.Fatalf("a person's word did not start the walk: %+v, asked %v", got.Delivery, *asked)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,333 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
"github.com/novox/mesh-controller/internal/secrets"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A module's own secret given at the operator's desk (novox/hq ADR 0259 §10, ADR 0277).
|
||||||
|
//
|
||||||
|
// mesh-controller secret ask <node> <module> <name> [--at <desk>]
|
||||||
|
//
|
||||||
|
// **The value never passes through whoever asked for it.** An agent, or the operator at the mesh MCP
|
||||||
|
// server, calls `secret-ask` (or `give`) with the machine, the module, the secret's name and the desk — never
|
||||||
|
// a value. The controller makes a sealing keypair for this one call, asks the desk's `node-launcher.secret` to
|
||||||
|
// prompt the operator without showing what is typed, and is answered with what was typed **sealed to that
|
||||||
|
// key**: no plaintext on the bus, in a runtime's log or in any call's record. It opens it here, seals it to the
|
||||||
|
// module's machine exactly as `secret accept` does, and forgets it. What it answers says only that the
|
||||||
|
// value was taken, or why not.
|
||||||
|
//
|
||||||
|
// **Bounded, and the prompt says who asked** (ADR 0277): one open prompt per secret and few an hour, read from
|
||||||
|
// the store before the prompt opens (inventory.OpenSecretAsk), so an agent cannot keep a prompt in front of the
|
||||||
|
// operator until they type. The prompt names the module, the secret, the machine and who asked — the caller as
|
||||||
|
// the bus named it, never a word the caller chose — written by the desk's launcher from those names alone.
|
||||||
|
//
|
||||||
|
// **What remains** (ADR 0234's accepted residual risk): on an X11 desk any program of the operator's
|
||||||
|
// account can read the keys as they are typed. And a program that calls the desk's prompt itself, with a
|
||||||
|
// key of its own, is answered with what the operator typed into a prompt they did not ask for — as it could
|
||||||
|
// draw a window of its own. The prompt says who asks and for what, so the operator types only into a
|
||||||
|
// prompt they started.
|
||||||
|
|
||||||
|
// deskPromptWithin is how long the prompt waits for the operator: below the runtime's thirty seconds for
|
||||||
|
// one call, as the launcher's menu is.
|
||||||
|
const deskPromptWithin = 25
|
||||||
|
|
||||||
|
// deskGive is the desk path, its four reaches given so a test needs no store and no bus.
|
||||||
|
type deskGive struct {
|
||||||
|
// declares refuses a module or a secret the mesh would refuse, before anybody is asked to type.
|
||||||
|
declares func(module, name string) error
|
||||||
|
// known refuses a machine the mesh does not know, before anybody is asked to type; nil knows every one
|
||||||
|
// (a test that does not look).
|
||||||
|
known func(machine string) error
|
||||||
|
// trusted says a module runs as an account of its own: its secret is never taken at a desk (below). Nil is
|
||||||
|
// never (a test that does not look).
|
||||||
|
trusted func(module string) (bool, error)
|
||||||
|
// ask asks one machine's node-launcher.secret and answers its result, or the holder's refusal.
|
||||||
|
ask func(machine string, args map[string]any) (json.RawMessage, error)
|
||||||
|
// accept seals the value as `secret accept` does, and says whether it lives until the module's start.
|
||||||
|
accept func(value string) (untilStart bool, err error)
|
||||||
|
// record writes the act in the hand-act log.
|
||||||
|
record func(link.HandAct) error
|
||||||
|
// announce raises the condition that says a module's own secret was given (secretGivenObservation), on
|
||||||
|
// every channel; nil announces nothing (a test that does not look).
|
||||||
|
announce func(node, module, name, how string) error
|
||||||
|
// askedBy is who asked, as the bus named the caller: said in the prompt and recorded.
|
||||||
|
askedBy string
|
||||||
|
// open records the ask and holds the bounds (one open per secret, few an hour), answering the record's id;
|
||||||
|
// nil keeps no record (a test that does not look). end closes it with how it ended.
|
||||||
|
open func(node, module, name, desk string) (int64, error)
|
||||||
|
end func(id int64, outcome string) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// askedByName is the caller as the prompt names it: the first clause of what the bus said, in the characters
|
||||||
|
// a name has, at most 80 of them. The desk's launcher refuses anything else, so no words of the caller's own
|
||||||
|
// reach the prompt.
|
||||||
|
func askedByName(caller string) string {
|
||||||
|
first, _, _ := strings.Cut(caller, ",")
|
||||||
|
var b strings.Builder
|
||||||
|
for _, r := range strings.TrimSpace(first) {
|
||||||
|
switch {
|
||||||
|
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '.', r == '_', r == '/', r == '@',
|
||||||
|
r == '-', r == ' ':
|
||||||
|
b.WriteRune(r)
|
||||||
|
default:
|
||||||
|
b.WriteRune('-')
|
||||||
|
}
|
||||||
|
if b.Len() >= 80 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
name := strings.TrimSpace(b.String())
|
||||||
|
if name == "" || strings.HasPrefix(name, "-") {
|
||||||
|
return "an unnamed caller"
|
||||||
|
}
|
||||||
|
return name
|
||||||
|
}
|
||||||
|
|
||||||
|
// errNothingGiven is a prompt dismissed, or not answered in time: nothing changes.
|
||||||
|
var errNothingGiven = errors.New("nothing was given")
|
||||||
|
|
||||||
|
// give asks the desk for the value and seals it; it answers the words said to the caller.
|
||||||
|
func (d deskGive) give(node, module, name, desk string) (string, error) {
|
||||||
|
for what, v := range map[string]string{"the machine": node, "the module": module, "the secret": name, "the desk": desk} {
|
||||||
|
if strings.TrimSpace(v) == "" {
|
||||||
|
return "", fmt.Errorf("%s is not named", what)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if d.known != nil {
|
||||||
|
for what, machine := range map[string]string{"the machine the secret is for": node, "the desk": desk} {
|
||||||
|
if err := d.known(machine); err != nil {
|
||||||
|
return "", fmt.Errorf("nobody was asked to type anything: %s, %s, is not a machine this mesh knows: %w",
|
||||||
|
what, machine, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := d.declares(module, name); err != nil {
|
||||||
|
return "", fmt.Errorf("nobody was asked to type anything: %w", err)
|
||||||
|
}
|
||||||
|
// **A trusted party's secret is not taken at a desk** (the confirmation review of 2026-10-09, N1-give). The
|
||||||
|
// prompt's answer comes back over the desk machine's bus, on a subject that machine's runtime answers — and
|
||||||
|
// on a desk machine agents run as the operator, who holds that runtime's credential. An agent could answer
|
||||||
|
// first, with a bot token of its own sealed to this call's key, and the channel the operator's answers are
|
||||||
|
// proven on would be the agent's. So the value of a module running as its own account is typed at the
|
||||||
|
// controller's terminal, where no bus carries it.
|
||||||
|
if d.trusted != nil {
|
||||||
|
trusted, err := d.trusted(module)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("nobody was asked to type anything: whether %s is a trusted party could not be read: %w", module, err)
|
||||||
|
}
|
||||||
|
if trusted {
|
||||||
|
return "", fmt.Errorf("nobody was asked to type anything: %s runs as an account of its own and proves the "+
|
||||||
|
"operator's answers, so its %s is given at the controller's terminal alone — there, run `mesh-controller "+
|
||||||
|
"secret accept %s %s %s` and type it at its prompt. A desk's prompt is answered over the desk machine's "+
|
||||||
|
"bus, where an agent may answer first (novox/hq ADR 0259 §10)", module, name, node, module, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The bounds, read and kept before anybody is asked to type (novox/hq ADR 0277): one open prompt per secret,
|
||||||
|
// few an hour. How the ask ends is recorded whatever happens below.
|
||||||
|
outcome := "failed"
|
||||||
|
if d.open != nil {
|
||||||
|
id, err := d.open(node, module, name, desk)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("nobody was asked to type anything: %w", err)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
if d.end != nil {
|
||||||
|
_ = d.end(id, outcome)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
public, private, err := secrets.Keypair()
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("no key could be made to take the value: %w", err)
|
||||||
|
}
|
||||||
|
// By name, never by words: the holder writes the prompt from these, and says the controller asks, which
|
||||||
|
// the bus alone makes true (broker.ControllerOnly). Who asked is the bus's word on the caller, cut to a
|
||||||
|
// name's characters — never an argument of the call.
|
||||||
|
raw, err := d.ask(desk, map[string]any{
|
||||||
|
"module": module,
|
||||||
|
"secret": name,
|
||||||
|
"node": node,
|
||||||
|
"asked_by": askedByName(d.askedBy),
|
||||||
|
"seal_to": public,
|
||||||
|
"timeout_seconds": deskPromptWithin,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
outcome = "refused"
|
||||||
|
return "", fmt.Errorf("the desk on %s could not be asked: %w", desk, err)
|
||||||
|
}
|
||||||
|
var answer struct {
|
||||||
|
Sealed string `json:"sealed"`
|
||||||
|
Cancelled bool `json:"cancelled"`
|
||||||
|
TimedOut bool `json:"timed_out"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &answer); err != nil {
|
||||||
|
return "", fmt.Errorf("the desk on %s answered something that is not the prompt's answer", desk)
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case answer.TimedOut:
|
||||||
|
outcome = "timed-out"
|
||||||
|
return "", fmt.Errorf("%w: the prompt on %s was not answered within %d seconds", errNothingGiven, desk, deskPromptWithin)
|
||||||
|
case answer.Cancelled:
|
||||||
|
outcome = "dismissed"
|
||||||
|
return "", fmt.Errorf("%w: the prompt on %s was dismissed", errNothingGiven, desk)
|
||||||
|
case answer.Sealed == "":
|
||||||
|
return "", fmt.Errorf("the desk on %s answered no sealed value", desk)
|
||||||
|
}
|
||||||
|
opened, err := secrets.Open(private, answer.Sealed)
|
||||||
|
if err != nil {
|
||||||
|
// Never the value, never what failed to open: only that it was not sealed to this call.
|
||||||
|
return "", fmt.Errorf("the desk on %s answered a value not sealed to this call; nothing was taken", desk)
|
||||||
|
}
|
||||||
|
value := asSupplied(string(opened))
|
||||||
|
for i := range opened {
|
||||||
|
opened[i] = 0
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(value) == "" {
|
||||||
|
outcome = "empty"
|
||||||
|
return "", fmt.Errorf("%w: the prompt on %s was answered empty", errNothingGiven, desk)
|
||||||
|
}
|
||||||
|
untilStart, err := d.accept(value)
|
||||||
|
value = ""
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
outcome = "given"
|
||||||
|
act := link.HandAct{Verb: "secret accept", Args: []string{node, module, name, "--at-desk", desk},
|
||||||
|
Why: fmt.Sprintf("the operator gave %s for %s on %s at the desk on %s, asked by %s", name, module, node, desk,
|
||||||
|
askedByName(d.askedBy)),
|
||||||
|
Cause: "given-at-the-desk"}
|
||||||
|
recorded := ""
|
||||||
|
if err := d.record(act); err != nil {
|
||||||
|
recorded = fmt.Sprintf("\n this act could NOT be recorded in the hand-act log, and is done anyway: %v", err)
|
||||||
|
}
|
||||||
|
if d.announce != nil {
|
||||||
|
if err := d.announce(node, module, name, "at the desk on "+desk); err != nil {
|
||||||
|
recorded += fmt.Sprintf("\n this change could NOT be announced on the operator's channels: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
words := fmt.Sprintf("%s on %s now holds %q, given at the desk on %s and sealed to %s; the mesh cannot read it "+
|
||||||
|
"back.\n run `push %s` to send it", module, node, name, desk, node, node)
|
||||||
|
if untilStart {
|
||||||
|
words += fmt.Sprintf("\n it lives until %s next starts well under the mesh, and is then replaced with a value "+
|
||||||
|
"the mesh makes (ADR 0228)", module)
|
||||||
|
}
|
||||||
|
return words + recorded, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// askAtDesk is `secret ask <node> <module> <name> [--at <machine>]`, and the terminal's `secret accept … --at-desk
|
||||||
|
// <machine>`: the desk path, on this controller's stores and bus. The desk is the module's machine unless named.
|
||||||
|
func askAtDesk(ctx context.Context, node, module, name, desk string) error {
|
||||||
|
if desk == "" {
|
||||||
|
desk = node
|
||||||
|
}
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
d := deskGive{
|
||||||
|
askedBy: link.Caller(),
|
||||||
|
open: func(node, module, name, desk string) (int64, error) {
|
||||||
|
return open.inventory.OpenSecretAsk(ctx, node, module, name, askedByName(link.Caller()), desk)
|
||||||
|
},
|
||||||
|
end: func(id int64, outcome string) error { return open.inventory.EndSecretAsk(ctx, id, outcome) },
|
||||||
|
declares: func(module, name string) error { return open.inventory.DeclaresOwnSecret(ctx, module, name) },
|
||||||
|
known: func(machine string) error {
|
||||||
|
_, err := open.inventory.NodeByName(ctx, machine)
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
trusted: func(module string) (bool, error) { return open.inventory.RunsAsItsOwnAccount(ctx, module) },
|
||||||
|
ask: func(machine string, args map[string]any) (json.RawMessage, error) {
|
||||||
|
var result json.RawMessage
|
||||||
|
err := onTheBus(func(conn *nats.Conn) error {
|
||||||
|
answer, err := link.AskSeatTool(ctx, conn, "node-launcher", "secret", machine, args,
|
||||||
|
time.Duration(deskPromptWithin+5)*time.Second)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if answer.Error != "" {
|
||||||
|
return errors.New(answer.Error)
|
||||||
|
}
|
||||||
|
result = answer.Result
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
return result, err
|
||||||
|
},
|
||||||
|
accept: func(value string) (bool, error) {
|
||||||
|
return open.inventory.AcceptGivenSecret(ctx, node, module, name, value)
|
||||||
|
},
|
||||||
|
record: func(act link.HandAct) error {
|
||||||
|
return onTheBus(func(conn *nats.Conn) error {
|
||||||
|
_, err := link.RecordHandAct(ctx, conn, act)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
},
|
||||||
|
announce: func(node, module, name, how string) error { return announceSecretGiven(ctx, node, module, name, how) },
|
||||||
|
}
|
||||||
|
words, err := d.give(node, module, name, desk)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(words)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// kindSecretGiven is the condition every value given for a module's own secret raises (the review of 2026-10-09,
|
||||||
|
// M4): on every channel, so a bot token changed by somebody else — a channel that now answers for them — is
|
||||||
|
// heard of. It stays until the operator silences or clears it.
|
||||||
|
const kindSecretGiven = "secret-given"
|
||||||
|
|
||||||
|
// secretGivenObservation is that condition: which secret, of which module on which machine, how and when.
|
||||||
|
// The summary, for whoever looks closer, names the secret and the time. The words the operator reads are
|
||||||
|
// held to the plain rule (conditions.PlainWords): no clock time — the channel says when, in the operator's
|
||||||
|
// time — and the secret's name said as words. Words that broke the rule were replaced by the keeper with
|
||||||
|
// "needs a look … a problem it calls secret given" (hq issue 359), which told the operator nothing.
|
||||||
|
func secretGivenObservation(node, module, name, how string, at time.Time) conditions.Observation {
|
||||||
|
key := node + "." + module + "." + name
|
||||||
|
where := module + " on " + node
|
||||||
|
// Within the bounds whatever the names' length: the module and machine, else the module, else the machine.
|
||||||
|
headline := "New secret given for " + where
|
||||||
|
for _, h := range []string{"New secret given for " + module, "New secret given on " + node} {
|
||||||
|
if len(headline) > conditions.HeadlineMax {
|
||||||
|
headline = h
|
||||||
|
}
|
||||||
|
}
|
||||||
|
resolved := "You saw that " + module + " was given a new secret"
|
||||||
|
if len(resolved) > conditions.HeadlineMax+20 {
|
||||||
|
resolved = "You saw that a new secret was given on " + node
|
||||||
|
}
|
||||||
|
return conditions.Observation{Scope: conditions.ScopeMachine, ID: key, Token: kindSecretGiven, Kind: kindSecretGiven,
|
||||||
|
Machine: node, Severity: conditions.Urgent, Source: kindSecretGiven,
|
||||||
|
Summary: fmt.Sprintf("%s of %s on %s was given %s at %s", name, module, node, how,
|
||||||
|
at.Local().Format("2006-01-02 15:04")),
|
||||||
|
Headline: headline,
|
||||||
|
Explanation: fmt.Sprintf("The secret %s of %s was given %s. If you gave it, nothing else is needed. If you "+
|
||||||
|
"did not, somebody else now holds what %s acts with.", secretNameWords(name), where, how, module),
|
||||||
|
Needs: "silence this if you just gave it; if you did not, give it again yourself so that only you hold it.",
|
||||||
|
Resolved: resolved,
|
||||||
|
Actions: []conditions.Action{conditions.SilenceAction(conditions.Key(conditions.ScopeMachine, key, kindSecretGiven))}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// secretNameWords is a secret's name as the operator reads it: "telegram-token" is "telegram token".
|
||||||
|
func secretNameWords(name string) string {
|
||||||
|
return strings.Join(strings.FieldsFunc(name, func(r rune) bool { return r == '-' || r == '_' || r == '.' }), " ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// announceSecretGiven raises it on this controller's keeper.
|
||||||
|
func announceSecretGiven(ctx context.Context, node, module, name, how string) error {
|
||||||
|
return withKeeper(ctx, func(k *conditions.Keeper) error {
|
||||||
|
_, err := k.Observe(ctx, secretGivenObservation(node, module, name, how, time.Now()))
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,512 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
"github.com/novox/mesh-controller/internal/secrets"
|
||||||
|
)
|
||||||
|
|
||||||
|
const typed = "123456789:AAEhBP0av28P4XFQnIuR-o-7Xnz1kkUzW3g"
|
||||||
|
|
||||||
|
// aDesk is the desk path with a prompt the test answers as the operator would, and what it was asked kept.
|
||||||
|
func aDesk(t *testing.T, answer func(args map[string]any) (json.RawMessage, error)) (deskGive, *[]string, *[]link.HandAct, *[]map[string]any) {
|
||||||
|
t.Helper()
|
||||||
|
var accepted []string
|
||||||
|
var acts []link.HandAct
|
||||||
|
var asked []map[string]any
|
||||||
|
return deskGive{
|
||||||
|
declares: func(module, name string) error {
|
||||||
|
if module != "telegram" || name != "telegram-token" {
|
||||||
|
return errors.New(module + " does not declare " + name + " as an own secret")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
ask: func(machine string, args map[string]any) (json.RawMessage, error) {
|
||||||
|
asked = append(asked, args)
|
||||||
|
return answer(args)
|
||||||
|
},
|
||||||
|
accept: func(value string) (bool, error) { accepted = append(accepted, value); return false, nil },
|
||||||
|
record: func(a link.HandAct) error { acts = append(acts, a); return nil },
|
||||||
|
}, &accepted, &acts, &asked
|
||||||
|
}
|
||||||
|
|
||||||
|
func sealedTo(t *testing.T, value string) func(args map[string]any) (json.RawMessage, error) {
|
||||||
|
return func(args map[string]any) (json.RawMessage, error) {
|
||||||
|
sealed, err := secrets.Seal(args["seal_to"].(string), []byte(value+"\n"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
raw, _ := json.Marshal(map[string]any{"sealed": sealed})
|
||||||
|
return raw, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0259 §10: the value typed at the desk is sealed as `secret accept` seals it, and is in no
|
||||||
|
// answer, no prompt argument and no act recorded.
|
||||||
|
func TestASecretGivenAtTheDeskIsSealedAndSaidNowhere(t *testing.T) {
|
||||||
|
d, accepted, acts, asked := aDesk(t, sealedTo(t, typed))
|
||||||
|
words, err := d.give("anchor", "telegram", "telegram-token", "laptop")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(*accepted) != 1 || (*accepted)[0] != typed {
|
||||||
|
t.Fatalf("the value sealed is not what was typed, its line ending taken off")
|
||||||
|
}
|
||||||
|
if len(*acts) != 1 || (*acts)[0].Verb != "secret accept" || (*acts)[0].Cause != "given-at-the-desk" ||
|
||||||
|
!strings.Contains((*acts)[0].Why, "at the desk on laptop") {
|
||||||
|
t.Errorf("the act: %+v", *acts)
|
||||||
|
}
|
||||||
|
raw, _ := json.Marshal(struct {
|
||||||
|
Words string
|
||||||
|
Acts []link.HandAct
|
||||||
|
Asked []map[string]any
|
||||||
|
}{words, *acts, *asked})
|
||||||
|
if strings.Contains(string(raw), typed) || strings.Contains(string(raw), "AAEhBP0") {
|
||||||
|
t.Fatal("the value appears in what was said, asked or recorded")
|
||||||
|
}
|
||||||
|
if !strings.Contains(words, "push anchor") || !strings.Contains(words, "given at the desk on laptop") {
|
||||||
|
t.Errorf("%q", words)
|
||||||
|
}
|
||||||
|
if p := (*asked)[0]; p["seal_to"] == "" || p["timeout_seconds"] != deskPromptWithin {
|
||||||
|
t.Errorf("the prompt was asked %v", p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNothingIsAskedForASecretTheMeshWouldRefuse(t *testing.T) {
|
||||||
|
for _, c := range [][2]string{{"telegram", "chat-id"}, {"nobody", "telegram-token"}} {
|
||||||
|
d, accepted, _, asked := aDesk(t, sealedTo(t, typed))
|
||||||
|
if _, err := d.give("anchor", c[0], c[1], "laptop"); err == nil || !strings.Contains(err.Error(), "nobody was asked") {
|
||||||
|
t.Errorf("%v: %v", c, err)
|
||||||
|
}
|
||||||
|
if len(*asked) != 0 || len(*accepted) != 0 {
|
||||||
|
t.Errorf("%v: the operator was asked anyway", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
d, _, _, _ := aDesk(t, sealedTo(t, typed))
|
||||||
|
if _, err := d.give("anchor", "telegram", "telegram-token", ""); err == nil {
|
||||||
|
t.Error("no desk was refused nowhere")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestADismissedEmptyLateOrForeignAnswerTakesNothing(t *testing.T) {
|
||||||
|
for want, answer := range map[string]func(map[string]any) (json.RawMessage, error){
|
||||||
|
"not answered within 25 seconds": func(map[string]any) (json.RawMessage, error) {
|
||||||
|
return json.RawMessage(`{"cancelled":true,"timed_out":true}`), nil
|
||||||
|
},
|
||||||
|
"was dismissed": func(map[string]any) (json.RawMessage, error) { return json.RawMessage(`{"cancelled":true}`), nil },
|
||||||
|
"answered empty": sealedTo(t, " "),
|
||||||
|
"not sealed to this call": func(map[string]any) (json.RawMessage, error) {
|
||||||
|
other, _, _ := secrets.Keypair()
|
||||||
|
sealed, _ := secrets.Seal(other, []byte(typed))
|
||||||
|
raw, _ := json.Marshal(map[string]any{"sealed": sealed})
|
||||||
|
return raw, nil
|
||||||
|
},
|
||||||
|
"could not be asked": func(map[string]any) (json.RawMessage, error) { return nil, errors.New("no session answers") },
|
||||||
|
} {
|
||||||
|
d, accepted, acts, _ := aDesk(t, answer)
|
||||||
|
_, err := d.give("anchor", "telegram", "telegram-token", "laptop")
|
||||||
|
if err == nil || !strings.Contains(err.Error(), want) || strings.Contains(err.Error(), typed) {
|
||||||
|
t.Errorf("want %q, got %v", want, err)
|
||||||
|
}
|
||||||
|
if len(*accepted) != 0 || len(*acts) != 0 {
|
||||||
|
t.Errorf("%s: something was taken or recorded", want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheGiveVerbRunsTheDeskPathAndTheControllerMayAskTheDesk(t *testing.T) {
|
||||||
|
argv, err := argvFor("give", map[string]any{"node": "anchor", "module": "telegram", "secret": "telegram-token", "at": "laptop"})
|
||||||
|
if err != nil || strings.Join(argv, " ") != "secret ask anchor telegram telegram-token --at laptop" {
|
||||||
|
t.Fatalf("%v %v", argv, err)
|
||||||
|
}
|
||||||
|
if _, err := argvFor("give", map[string]any{"node": "anchor", "module": "telegram", "secret": "telegram-token"}); err == nil {
|
||||||
|
t.Error("give without a desk was taken")
|
||||||
|
}
|
||||||
|
// secret-ask is the same line, with the desk the module's machine unless named (novox/hq ADR 0277).
|
||||||
|
argv, err = argvFor("secret-ask", map[string]any{"node": "anchor", "module": "telegram", "secret": "telegram-token"})
|
||||||
|
if err != nil || strings.Join(argv, " ") != "secret ask anchor telegram telegram-token" {
|
||||||
|
t.Fatalf("%v %v", argv, err)
|
||||||
|
}
|
||||||
|
argv, err = argvFor("secret-ask", map[string]any{"node": "anchor", "module": "telegram", "secret": "telegram-token", "at": "laptop"})
|
||||||
|
if err != nil || strings.Join(argv, " ") != "secret ask anchor telegram telegram-token --at laptop" {
|
||||||
|
t.Fatalf("%v %v", argv, err)
|
||||||
|
}
|
||||||
|
perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
found := false
|
||||||
|
for _, p := range perms.Publish {
|
||||||
|
found = found || p == "mesh.seat.node-launcher.tool.secret.*"
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Error("the controller may not ask the desk's prompt")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The review of 2026-10-09 (M4): the desk's prompt says who asks in words the caller does not choose — the
|
||||||
|
// controller, which the bus alone lets ask it — and what for, from names the controller checked; the prompt
|
||||||
|
// carries no free text of the caller's.
|
||||||
|
func TestThePromptIsAskedByNameNeverByWordsTheCallerChose(t *testing.T) {
|
||||||
|
d, _, _, asked := aDesk(t, sealedTo(t, typed))
|
||||||
|
if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
p := (*asked)[0]
|
||||||
|
if p["module"] != "telegram" || p["secret"] != "telegram-token" || p["node"] != "anchor" {
|
||||||
|
t.Errorf("the prompt was not asked by name: %v", p)
|
||||||
|
}
|
||||||
|
for _, free := range []string{"prompt", "message"} {
|
||||||
|
if _, there := p[free]; there {
|
||||||
|
t.Errorf("the prompt carries the caller's %s: %v", free, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every value given for a module's own secret is announced as a condition, on every channel (the review of
|
||||||
|
// 2026-10-09, M4): a bot token changed by somebody else is a channel that now answers for them.
|
||||||
|
func TestAValueGivenAtTheDeskIsAnnounced(t *testing.T) {
|
||||||
|
d, _, _, _ := aDesk(t, sealedTo(t, typed))
|
||||||
|
var said []string
|
||||||
|
d.announce = func(node, module, name, how string) error {
|
||||||
|
said = append(said, node+" "+module+" "+name+" "+how)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(said) != 1 || !strings.Contains(said[0], "anchor telegram telegram-token") || !strings.Contains(said[0], "laptop") {
|
||||||
|
t.Fatalf("announced %v", said)
|
||||||
|
}
|
||||||
|
o := secretGivenObservation("anchor", "telegram", "telegram-token", "at the desk on laptop", time.Date(2026, 10, 9, 12, 3, 0, 0, time.UTC))
|
||||||
|
if o.Severity != conditions.Urgent || !strings.Contains(o.Explanation, "telegram token") ||
|
||||||
|
len(o.Actions) == 0 || o.Key() == "" {
|
||||||
|
t.Errorf("the announcement %+v", o)
|
||||||
|
}
|
||||||
|
if strings.Contains(o.Summary+o.Explanation+o.Said, typed) {
|
||||||
|
t.Error("the announcement carries the value")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The secret-given condition says itself in its own plain words** (hq issue 359): the words it carries pass
|
||||||
|
// the plain rule, from the controller's terminal and from a desk, so the keeper keeps them — they once held a
|
||||||
|
// clock time, and the operator read "Novox needs a look … a problem it calls secret given" instead. Its
|
||||||
|
// severity and its answer stay: it is heard on every channel, and silenced by the operator.
|
||||||
|
func TestTheSecretGivenConditionSaysItselfInPlainWords(t *testing.T) {
|
||||||
|
at := time.Date(2026, 10, 9, 23, 32, 0, 0, time.Local)
|
||||||
|
for _, how := range []string{"at the controller's terminal", "at the desk on laptop"} {
|
||||||
|
o := secretGivenObservation("anchor", "telegram", "telegram-token", how, at)
|
||||||
|
w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs,
|
||||||
|
Actions: o.Actions}
|
||||||
|
if why, ok := conditions.PlainWords(w, o.Machine); !ok {
|
||||||
|
t.Fatalf("given %s, the words are not plain: %s", how, why)
|
||||||
|
}
|
||||||
|
k, _ := withConditionsInMemory(t)
|
||||||
|
c, err := k.Observe(t.Context(), o)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if c.Headline != "New secret given for telegram on anchor" || c.Severity != conditions.Urgent ||
|
||||||
|
!strings.HasPrefix(c.Explanation, conditions.NeedsYou+" silence this") ||
|
||||||
|
!strings.Contains(c.Explanation, "telegram token of telegram on anchor was given "+how) ||
|
||||||
|
len(c.Actions) != 1 || c.Actions[0].Label != "Silence for a week" {
|
||||||
|
t.Errorf("given %s, the keeper said %q / %q (%s, %v)", how, c.Headline, c.Explanation, c.Severity, c.Actions)
|
||||||
|
}
|
||||||
|
if strings.Contains(c.Headline+c.Explanation+c.Resolved+c.Needs, typed) {
|
||||||
|
t.Error("the words carry the value")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A long module name keeps the headline and the resolved line within their bounds.
|
||||||
|
for _, module := range []string{"a-module-with-a-rather-long-name-indeed",
|
||||||
|
"a-module-with-a-name-so-long-that-no-headline-could-ever-hold-it"} {
|
||||||
|
o := secretGivenObservation("anchor", module, "api-key", "at the controller's terminal", at)
|
||||||
|
if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Explanation: o.Explanation,
|
||||||
|
Resolved: o.Resolved, Needs: o.Needs, Actions: o.Actions}, o.Machine); !ok {
|
||||||
|
t.Errorf("the module %s: %s", module, why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The bus lets the controller alone ask the desk's prompt (the review of 2026-10-09, M4): the runtime, which
|
||||||
|
// carries every agent's calls, and a person granted every tool are denied it, however wide their grant.
|
||||||
|
func TestOnlyTheControllerMayAskTheDesksPrompt(t *testing.T) {
|
||||||
|
for _, p := range []broker.Principal{
|
||||||
|
{Kind: broker.KindNodeTools, Node: "laptop"},
|
||||||
|
{Kind: broker.KindPerson, Module: "operator", Invokes: []string{"*"}},
|
||||||
|
{Kind: broker.KindModule, Node: "laptop", Module: "lab", Invokes: []string{"seat:node-launcher.secret"}},
|
||||||
|
} {
|
||||||
|
perms, err := broker.PermissionsFor(p)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, subject := range []string{"mesh.seat.node-launcher.tool.secret.laptop", "mesh.seat.node-launcher.tool.secret",
|
||||||
|
"mesh.mod.rofi.tool.node-launcher.secret", "mesh.mod.rofi.tool.node-launcher.secret.laptop"} {
|
||||||
|
if broker.MayPublish(perms, subject) {
|
||||||
|
t.Errorf("%s may publish %s", p.Username(), subject)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
perms, _ := broker.PermissionsFor(broker.Principal{Kind: broker.KindController})
|
||||||
|
if !broker.MayPublish(perms, "mesh.seat.node-launcher.tool.secret.laptop") {
|
||||||
|
t.Error("the controller may not ask the desk's prompt")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A value for a secret comes from the terminal or the desk, never through a verb (the review of 2026-10-09,
|
||||||
|
// M4): `secret accept` with a value, run for a verb, is refused before anything is read.
|
||||||
|
func TestASecretValueIsNeverAcceptedThroughAVerb(t *testing.T) {
|
||||||
|
t.Setenv(verbVar, "mesh-controller.command")
|
||||||
|
for _, args := range [][]string{
|
||||||
|
{"accept", "anchor", "telegram", "telegram-token", "--from", "/dev/null"},
|
||||||
|
{"accept", "anchor", "app", "db", "--from", "/dev/null", "--provider", "store"},
|
||||||
|
} {
|
||||||
|
err := secretCommand(context.Background(), args)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "never through a verb") {
|
||||||
|
t.Errorf("%v: %v", args, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The `give` and `secret-ask` verbs' own line passes the terminal-only rule of ADR 0266, and no `secret accept`
|
||||||
|
// does: a value, a file, a provider or an extra word is still the terminal's alone (novox/hq ADR 0277).
|
||||||
|
func TestOnlyTheGiveLinePassesTheTerminalRuleForSecrets(t *testing.T) {
|
||||||
|
for _, argv := range [][]string{
|
||||||
|
{"secret", "ask", "anchor", "telegram", "telegram-token", "--at", "laptop"},
|
||||||
|
{"secret", "ask", "anchor", "telegram", "telegram-token"},
|
||||||
|
} {
|
||||||
|
if err := terminalOnly(argv); err != nil {
|
||||||
|
t.Errorf("%v refused: %v", argv, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, argv := range [][]string{
|
||||||
|
{"secret", "accept", "anchor", "telegram", "telegram-token", "--at-desk", "laptop"},
|
||||||
|
{"secret", "ask", "anchor", "telegram", "telegram-token", "--from", "/tmp/x"},
|
||||||
|
{"secret", "ask", "anchor", "telegram", "telegram-token", "--at", "laptop", "--local"},
|
||||||
|
{"secret", "ask", "anchor", "telegram", "--at", "laptop"},
|
||||||
|
{"secret", "accept", "anchor", "telegram", "telegram-token"},
|
||||||
|
{"secret", "accept", "anchor", "telegram", "telegram-token", "--from", "/tmp/x"},
|
||||||
|
{"secret", "accept", "anchor", "telegram", "telegram-token", "--at-desk", "laptop", "--local"},
|
||||||
|
{"secret", "accept", "anchor", "telegram", "--provider", "--at-desk", "laptop"},
|
||||||
|
{"secret", "export", "anchor", "telegram", "telegram-token", "--at-desk", "laptop"},
|
||||||
|
} {
|
||||||
|
if err := terminalOnly(argv); err == nil {
|
||||||
|
t.Errorf("%v passed the terminal rule", argv)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The confirmation review of 2026-10-09, N1-give: a desk's prompt is answered over the desk machine's bus, and
|
||||||
|
// on a desk machine agents run as the operator, who holds its runtime's credential — so a trusted party's
|
||||||
|
// secret (a module running as an account of its own: the Telegram bot's token) is never taken at a desk.
|
||||||
|
// Refused before anybody is asked to type, whoever called, naming the terminal's line.
|
||||||
|
func TestATrustedPartysSecretIsNeverTakenAtADesk(t *testing.T) {
|
||||||
|
d, accepted, acts, asked := aDesk(t, sealedTo(t, typed))
|
||||||
|
d.trusted = func(module string) (bool, error) { return module == "telegram", nil }
|
||||||
|
_, err := d.give("anchor", "telegram", "telegram-token", "laptop")
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "controller's terminal alone") ||
|
||||||
|
!strings.Contains(err.Error(), "secret accept anchor telegram telegram-token") {
|
||||||
|
t.Fatalf("a trusted party's secret was taken at the desk, or refused without the line: %v", err)
|
||||||
|
}
|
||||||
|
if len(*asked)+len(*accepted)+len(*acts) != 0 {
|
||||||
|
t.Errorf("asked %v, accepted %d, recorded %v", *asked, len(*accepted), *acts)
|
||||||
|
}
|
||||||
|
d.trusted = func(string) (bool, error) { return false, errors.New("the store did not answer") }
|
||||||
|
if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); err == nil || len(*asked) != 0 {
|
||||||
|
t.Errorf("a module not known to be untrusted was asked at the desk: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// And who may answer the desk's prompt at all: only the runtime of the machine it is asked on, carrying the
|
||||||
|
// launcher that holds the seat there — never the controller, another machine's runtime, or a module's own
|
||||||
|
// account (the confirmation review of 2026-10-09, N1-give).
|
||||||
|
func TestOnlyTheDeskMachinesLauncherMayAnswerItsPrompt(t *testing.T) {
|
||||||
|
launcher := broker.Declared{Module: "rofi", Holds: []broker.Seat{{Name: "node-launcher", Scope: "node",
|
||||||
|
Serves: []string{"run", "secret"}}}}
|
||||||
|
subject := "mesh.seat.node-launcher.tool.secret.laptop"
|
||||||
|
for _, c := range []struct {
|
||||||
|
p broker.Principal
|
||||||
|
answers bool
|
||||||
|
}{
|
||||||
|
{broker.Principal{Kind: broker.KindNodeTools, Node: "laptop", Module: broker.RuntimeModule, Carries: []broker.Declared{launcher}}, true},
|
||||||
|
{broker.Principal{Kind: broker.KindNodeTools, Node: "anchor", Module: broker.RuntimeModule, Carries: []broker.Declared{launcher}}, false},
|
||||||
|
{broker.Principal{Kind: broker.KindController}, false},
|
||||||
|
{broker.Principal{Kind: broker.KindModule, Node: "laptop", Module: "lab"}, false},
|
||||||
|
{broker.Principal{Kind: broker.KindNode, Node: "laptop"}, false},
|
||||||
|
} {
|
||||||
|
perms, err := broker.PermissionsFor(c.p)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := broker.MaySubscribe(perms, subject); got != c.answers {
|
||||||
|
t.Errorf("%s may answer %s: %v, want %v", c.p.Username(), subject, got, c.answers)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// N1-give at the controller's terminal (the confirmation review of 2026-10-09): a trusted party's secret is
|
||||||
|
// announced before it is kept, and not kept when the announcement fails; another module's is kept first and
|
||||||
|
// a failed announcement is said, not undone.
|
||||||
|
func TestATrustedPartysSecretGivenAtTheTerminalIsAnnouncedBeforeItIsKept(t *testing.T) {
|
||||||
|
var order []string
|
||||||
|
announce := func(fail bool) func() error {
|
||||||
|
return func() error {
|
||||||
|
order = append(order, "announce")
|
||||||
|
if fail {
|
||||||
|
return errors.New("no channel")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
keep := func() (bool, error) { order = append(order, "keep"); return false, nil }
|
||||||
|
|
||||||
|
order = nil
|
||||||
|
if _, unannounced, err := keepGiven(true, announce(false), keep); err != nil || unannounced != nil ||
|
||||||
|
strings.Join(order, ",") != "announce,keep" {
|
||||||
|
t.Errorf("trusted: %v %v, order %v; want announced, then kept", unannounced, err, order)
|
||||||
|
}
|
||||||
|
order = nil
|
||||||
|
if _, _, err := keepGiven(true, announce(true), keep); err == nil || strings.Join(order, ",") != "announce" {
|
||||||
|
t.Errorf("trusted, announcement failed: %v, order %v; want refused and nothing kept", err, order)
|
||||||
|
}
|
||||||
|
order = nil
|
||||||
|
if _, unannounced, err := keepGiven(false, announce(true), keep); err != nil || unannounced == nil ||
|
||||||
|
strings.Join(order, ",") != "keep,announce" {
|
||||||
|
t.Errorf("not trusted: %v %v, order %v; want kept, then the failed announcement said", unannounced, err, order)
|
||||||
|
}
|
||||||
|
order = nil
|
||||||
|
failing := func() (bool, error) { order = append(order, "keep"); return false, errors.New("store away") }
|
||||||
|
if _, _, err := keepGiven(false, announce(false), failing); err == nil || strings.Join(order, ",") != "keep" {
|
||||||
|
t.Errorf("not trusted, keep failed: %v, order %v; want refused and nothing announced", err, order)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A machine the mesh does not know, as the secret's or as the desk, is refused before anybody is asked to type.
|
||||||
|
func TestAGiveNamingAMachineTheMeshDoesNotKnowAsksNobody(t *testing.T) {
|
||||||
|
for _, unknown := range []string{"elsewhere", "nodesk"} {
|
||||||
|
d, accepted, acts, asked := aDesk(t, func(map[string]any) (json.RawMessage, error) {
|
||||||
|
t.Fatal("the desk was asked")
|
||||||
|
return nil, nil
|
||||||
|
})
|
||||||
|
d.known = func(machine string) error {
|
||||||
|
if machine == unknown {
|
||||||
|
return errors.New("no node " + machine)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
node, desk := "anchor", "laptop"
|
||||||
|
if unknown == "elsewhere" {
|
||||||
|
node = unknown
|
||||||
|
} else {
|
||||||
|
desk = unknown
|
||||||
|
}
|
||||||
|
_, err := d.give(node, "telegram", "telegram-token", desk)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "nobody was asked") || !strings.Contains(err.Error(), unknown) {
|
||||||
|
t.Errorf("%s: %v", unknown, err)
|
||||||
|
}
|
||||||
|
if len(*accepted)+len(*acts)+len(*asked) != 0 {
|
||||||
|
t.Errorf("%s: something happened: %v %v %v", unknown, *accepted, *acts, *asked)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0277: the prompt names who asked — the controller's word on the bus's caller, cut to a name's
|
||||||
|
// characters — and never a word the caller chose: there is no argument for it.
|
||||||
|
func TestThePromptNamesWhoAskedFromTheBussWordAlone(t *testing.T) {
|
||||||
|
d, _, acts, asked := aDesk(t, sealedTo(t, typed))
|
||||||
|
d.askedBy = "g14/claude-code, through the mesh-controller seat"
|
||||||
|
if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := (*asked)[0]["asked_by"]; got != "g14/claude-code" {
|
||||||
|
t.Errorf("asked_by %q", got)
|
||||||
|
}
|
||||||
|
if len(*acts) != 1 || !strings.Contains((*acts)[0].Why, "asked by g14/claude-code") {
|
||||||
|
t.Errorf("the record: %+v", *acts)
|
||||||
|
}
|
||||||
|
for in, want := range map[string]string{
|
||||||
|
"jochen at a shell on novox": "jochen at a shell on novox",
|
||||||
|
"laptop/agent": "laptop/agent",
|
||||||
|
"Your bank asks\nType your PIN, now": "Your bank asks-Type your PIN",
|
||||||
|
"": "an unnamed caller",
|
||||||
|
"<b>x</b>": "an unnamed caller",
|
||||||
|
strings.Repeat("a", 100): strings.Repeat("a", 80),
|
||||||
|
"--prompt, something else": "an unnamed caller",
|
||||||
|
} {
|
||||||
|
if got := askedByName(in); got != want {
|
||||||
|
t.Errorf("askedByName(%q) = %q, want %q", in, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The verb's schema has no argument that reaches the prompt's words.
|
||||||
|
for _, verb := range []string{"give", "secret-ask"} {
|
||||||
|
for _, free := range []string{"asked_by", "prompt", "message", "value", "from"} {
|
||||||
|
if _, err := argvFor(verb, map[string]any{"node": "anchor", "module": "telegram", "secret": "telegram-token",
|
||||||
|
"at": "laptop", free: "x"}); err == nil {
|
||||||
|
t.Errorf("%s takes %s", verb, free)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An ask for a secret is bounded before anybody is asked to type (ADR 0277): the record refuses it, nothing is
|
||||||
|
// asked; and how every ask ends is recorded.
|
||||||
|
func TestASecretAskIsBoundedAndItsEndRecorded(t *testing.T) {
|
||||||
|
d, accepted, _, asked := aDesk(t, sealedTo(t, typed))
|
||||||
|
var ended []string
|
||||||
|
d.open = func(node, module, name, desk string) (int64, error) { return 7, nil }
|
||||||
|
d.end = func(id int64, outcome string) error {
|
||||||
|
ended = append(ended, fmt.Sprintf("%d %s", id, outcome))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
d.open = func(node, module, name, desk string) (int64, error) {
|
||||||
|
return 0, errors.New("an ask for telegram-token of telegram on anchor is still open")
|
||||||
|
}
|
||||||
|
_, err := d.give("anchor", "telegram", "telegram-token", "laptop")
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "nobody was asked to type anything") || !strings.Contains(err.Error(), "still open") {
|
||||||
|
t.Errorf("a second ask: %v", err)
|
||||||
|
}
|
||||||
|
if len(*asked) != 1 || len(*accepted) != 1 {
|
||||||
|
t.Errorf("asked %d, accepted %d", len(*asked), len(*accepted))
|
||||||
|
}
|
||||||
|
d.open = func(node, module, name, desk string) (int64, error) { return 8, nil }
|
||||||
|
d.ask = func(string, map[string]any) (json.RawMessage, error) {
|
||||||
|
return json.RawMessage(`{"cancelled":true}`), nil
|
||||||
|
}
|
||||||
|
if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); !errors.Is(err, errNothingGiven) {
|
||||||
|
t.Errorf("a dismissed prompt: %v", err)
|
||||||
|
}
|
||||||
|
if strings.Join(ended, "; ") != "7 given; 8 dismissed" {
|
||||||
|
t.Errorf("ended: %v", ended)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A secret ask cannot be turned into a secret read: the line carries no value, the answer carries none, and every
|
||||||
|
// other `secret` line is the terminal's.
|
||||||
|
func TestASecretAskIsNeverASecretRead(t *testing.T) {
|
||||||
|
t.Setenv(verbVar, "mesh-controller.secret-ask")
|
||||||
|
for _, args := range [][]string{
|
||||||
|
{"ask", "anchor", "telegram", "telegram-token", "the-value"},
|
||||||
|
{"ask", "anchor", "telegram"},
|
||||||
|
{"ask", "anchor", "telegram", "telegram-token", "--from", "/dev/null"},
|
||||||
|
} {
|
||||||
|
if err := secretCommand(context.Background(), args); err == nil {
|
||||||
|
t.Errorf("secret %v was taken", args)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, args := range [][]string{{"recover", "anchor", "telegram", "telegram-token"}, {"export"}} {
|
||||||
|
if err := terminalOnly(append([]string{"secret"}, args...)); err == nil {
|
||||||
|
t.Errorf("secret %v passed the terminal rule", args)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -116,6 +116,11 @@ var probeRegistry = []probe{
|
|||||||
{ID: probeDeliveriesID, Asserts: "no delivery is held past its state's bound unsaid: mesh-delivery's " +
|
{ID: probeDeliveriesID, Asserts: "no delivery is held past its state's bound unsaid: mesh-delivery's " +
|
||||||
"`stalled`, each with the transition its table lets healer H2 take", From: "ADR 0239",
|
"`stalled`, each with the transition its table lets healer H2 take", From: "ADR 0239",
|
||||||
Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries},
|
Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries},
|
||||||
|
// The delivery budgets (novox/hq ADR 0282 decision 7): the newest delivery of each class within its budget,
|
||||||
|
// read from mesh-delivery's `times`; a measurement said, never a delivery held.
|
||||||
|
{ID: probeBudgetsID, Asserts: "the newest delivery of a leaf module ran on every machine within five minutes of " +
|
||||||
|
"its merge, and of a core module within ten: mesh-delivery's `times`", From: "ADR 0282",
|
||||||
|
Kind: kindOverBudget, Phase: 3, run: probeBudgets},
|
||||||
// A client of the bus reconnecting in a loop (novox/hq issue 327), from the server's record of closed
|
// A client of the bus reconnecting in a loop (novox/hq issue 327), from the server's record of closed
|
||||||
// connections, which the bus's own module reads.
|
// connections, which the bus's own module reads.
|
||||||
{ID: probeReconnectsID, Asserts: "no user of the bus had its connection dropped more than twelve times in the " +
|
{ID: probeReconnectsID, Asserts: "no user of the bus had its connection dropped more than twelve times in the " +
|
||||||
@@ -131,6 +136,12 @@ var probeRegistry = []probe{
|
|||||||
{ID: "D-root", Asserts: "no agent can become root without a person on a machine where the router or a channel " +
|
{ID: "D-root", Asserts: "no agent can become root without a person on a machine where the router or a channel " +
|
||||||
"proving its sender runs: not by its own account, and not through a tool that runs its command as an account " +
|
"proving its sender runs: not by its own account, and not through a tool that runs its command as an account " +
|
||||||
"that can", From: "ADR 0259 §8", Kind: kindRootNotFree, Phase: 2, run: probeAgentRoot},
|
"that can", From: "ADR 0259 §8", Kind: kindRootNotFree, Phase: 2, run: probeAgentRoot},
|
||||||
|
// A module's bucket or log filling toward its cap (novox/hq ADR 0297 §6): said at three quarters, cleared
|
||||||
|
// below seven tenths, so one at its threshold is not raised and cleared on every run. One that cannot be
|
||||||
|
// read is said on its own, and every other is still judged.
|
||||||
|
{ID: probeFillID, Asserts: "every module's bucket and log holds less than three quarters of its cap; one " +
|
||||||
|
"said filling is cleared once it holds less than seven tenths", From: "ADR 0297, issue 501",
|
||||||
|
Kind: kindBucketOrLogFilling, Raises: []string{kindBucketOrLogUnread}, Phase: 1, run: probeFill},
|
||||||
{ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals",
|
{ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals",
|
||||||
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs},
|
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs},
|
||||||
// The core's health definitions (novox/hq to-be 45 §8, ADR 0236): what a core component's new build is
|
// The core's health definitions (novox/hq to-be 45 §8, ADR 0236): what a core component's new build is
|
||||||
|
|||||||
@@ -205,7 +205,7 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return snapshot.Facts{}, err
|
return snapshot.Facts{}, err
|
||||||
}
|
}
|
||||||
read, err := inv.ReadRepositories(ctx)
|
read, err := readForPlanning(ctx, inv)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return snapshot.Facts{}, err
|
return snapshot.Facts{}, err
|
||||||
}
|
}
|
||||||
@@ -411,7 +411,18 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot
|
|||||||
Commit: e.Source.BuiltFrom, Provided: e.Provided, RollOut: current[e.Manifest.Module].RollOut,
|
Commit: e.Source.BuiltFrom, Provided: e.Provided, RollOut: current[e.Manifest.Module].RollOut,
|
||||||
Manifest: raw}
|
Manifest: raw}
|
||||||
for _, r := range read[e.Manifest.Module] {
|
for _, r := range read[e.Manifest.Module] {
|
||||||
|
// The module's own build source is said apart: a gate that predates it would read an own
|
||||||
|
// entry among Reads as a context of its own repository.
|
||||||
|
if r.Own {
|
||||||
|
if len(r.Paths) > 0 {
|
||||||
|
mod.Sources = append(mod.Sources, snapshot.BuildSource{Own: true, Paths: r.Paths})
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
mod.Reads = append(mod.Reads, snapshot.RepositoryName(r.Repository))
|
mod.Reads = append(mod.Reads, snapshot.RepositoryName(r.Repository))
|
||||||
|
if len(r.Paths) > 0 {
|
||||||
|
mod.Sources = append(mod.Sources, snapshot.BuildSource{Repository: snapshot.RepositoryName(r.Repository), Paths: r.Paths})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
f.Modules = append(f.Modules, mod)
|
f.Modules = append(f.Modules, mod)
|
||||||
if e.Provided || e.Source.Repository == "" {
|
if e.Provided || e.Source.Repository == "" {
|
||||||
@@ -503,6 +514,10 @@ func composedAndValidated(ctx context.Context, open *stores, node string, gens m
|
|||||||
if declared.Epoch, err = open.inventory.SentEpoch(ctx, record.ID); err != nil {
|
if declared.Epoch, err = open.inventory.SentEpoch(ctx, record.ID); err != nil {
|
||||||
return sendable{}, nil, err
|
return sendable{}, nil, err
|
||||||
}
|
}
|
||||||
|
// And the generation it was last sent, as the would-send is (novox/hq issue 234).
|
||||||
|
if declared.Generation, err = open.inventory.SentGeneration(ctx, record.ID); err != nil {
|
||||||
|
return sendable{}, nil, err
|
||||||
|
}
|
||||||
body, err := declared.Body()
|
body, err := declared.Body()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return sendable{}, nil, err
|
return sendable{}, nil, err
|
||||||
|
|||||||
@@ -0,0 +1,66 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A provider whose wait fails the controller's check lists who waits on it (novox/hq issue 450).
|
||||||
|
//
|
||||||
|
// A wait that does not check out is judged unhealthy (ADR 0283 decision 3), so the provider raises its unhealthy
|
||||||
|
// condition and its consumers are held under it (ADR 0240 rule 5). What is stored is what the machine said, the
|
||||||
|
// wait unchecked: read as said, the provider seems to wait for the operator, and the held consumers were never
|
||||||
|
// listed on the condition that is open.
|
||||||
|
|
||||||
|
// refusedWait is a wait for a secret the database's manifest does not declare: it fails the check.
|
||||||
|
var refusedWait = inventory.Wait{Part: "postgres-database", Secret: "certificate", What: "the database's certificate"}
|
||||||
|
|
||||||
|
// judgedRefused is the provider's resource as the controller judges it: unhealthy, saying why.
|
||||||
|
func judgedRefused() inventory.ResourceHealth {
|
||||||
|
r := waitingDatabaseFor(refusedWait)
|
||||||
|
r.State, r.Waits = link.StateUnhealthy, nil
|
||||||
|
r.Reason = "says it waits for the secret certificate, which db does not declare"
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// The consumer's statement arrives after the provider's, so it is the consumer's judging (sayWaiters) that must list
|
||||||
|
// it at the provider's unhealthy condition, made urgent by who waits on it.
|
||||||
|
func TestAProviderWhoseWaitFailedItsCheckListsWhoWaitsOnIt(t *testing.T) {
|
||||||
|
k, _ := withConditionsInMemory(t)
|
||||||
|
stored := waitingDatabaseFor(refusedWait)
|
||||||
|
open := judgeBoth(t, k, []inventory.ResourceHealth{judgedRefused()},
|
||||||
|
map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Resources: []inventory.ResourceHealth{stored}}},
|
||||||
|
shopFailingBeside(stored))
|
||||||
|
provider := conditionOf(open, moduleUnhealthyKey("db", "anchor"))
|
||||||
|
if len(open) != 1 || provider == nil {
|
||||||
|
t.Fatalf("a provider whose wait failed its check and a consumer of it raised %v; want the provider's "+
|
||||||
|
"unhealthy alone", openKeysOf(open))
|
||||||
|
}
|
||||||
|
if said := provider.Evidence[0].Said; !strings.Contains(said, "shop on laptop") {
|
||||||
|
t.Fatalf("the provider's unhealthy condition does not list shop on laptop as waiting on it: %s", said)
|
||||||
|
}
|
||||||
|
if provider.Severity != conditions.Urgent {
|
||||||
|
t.Fatalf("the provider's unhealthy condition is %s with a consumer waiting on it; want urgent", provider.Severity)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A provider that waits for a secret already given is unhealthy to its consumers too, before its own condition opens:
|
||||||
|
// they are held under it as under any unhealthy provider, never as waiting for the operator, and its condition, when
|
||||||
|
// open, is said as unhealthy with who waits on it.
|
||||||
|
func TestAProviderWaitingForASecretAlreadyGivenIsUnhealthyToItsConsumers(t *testing.T) {
|
||||||
|
given := holdingOf(nil, shopFailingBeside(waitingDatabase()), shopOnTheDatabase)
|
||||||
|
given.waits.given["db@anchor"] = map[string]time.Time{"licence": time.Now().Add(-time.Hour)}
|
||||||
|
by, held := given.heldWith("laptop", "shop", []inventory.ResourceHealth{failingConsumer("shop")})
|
||||||
|
if !held || by.provider != theDatabase || len(by.waits) > 0 {
|
||||||
|
t.Fatalf("shop under a database waiting for a licence already given: held %v under %v with waits %v; want "+
|
||||||
|
"held under db on anchor as unhealthy, no waits", held, by.provider, by.waits)
|
||||||
|
}
|
||||||
|
if _, uncovered := given.waitingUncovered("laptop", "shop", []inventory.ResourceHealth{failingConsumer("shop")}); uncovered {
|
||||||
|
t.Fatalf("a provider whose wait failed its check is said as waiting for another part")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -18,7 +18,7 @@ func foundDirectory(module string, since time.Time) inventory.ResourceHealth {
|
|||||||
return inventory.ResourceHealth{Module: module, Resource: module + ".data", Kind: link.KindDirectory,
|
return inventory.ResourceHealth{Module: module, Resource: module + ".data", Kind: link.KindDirectory,
|
||||||
Target: "/srv/" + module, State: link.StateUnhealthy, Since: since,
|
Target: "/srv/" + module, State: link.StateUnhealthy, Since: since,
|
||||||
Reason: link.ReasonUsedAsFound + " owned by 1000:1000, mode 700, as found; root, mode 755 was declared and " +
|
Reason: link.ReasonUsedAsFound + " owned by 1000:1000, mode 700, as found; root, mode 755 was declared and " +
|
||||||
"not given it — `mesh-host hand-over` at the machine hands it to the mesh"}
|
"not given it — `nox node hand-over laptop <directory>` on the control-node, with its path, hands it to the mesh"}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) {
|
func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) {
|
||||||
|
|||||||
+232
-12
@@ -87,6 +87,9 @@ const (
|
|||||||
healthWaiting
|
healthWaiting
|
||||||
healthNotYet
|
healthNotYet
|
||||||
healthBroken
|
healthBroken
|
||||||
|
// healthSuperseded is a judging that cannot go on: the machine was sent another build of the module
|
||||||
|
// after the gate's send (novox/hq issue 352). No verdict on the build judged, and nothing put back.
|
||||||
|
healthSuperseded
|
||||||
)
|
)
|
||||||
|
|
||||||
// served is what one machine's node tools answered the bus's discovery with.
|
// served is what one machine's node tools answered the bus's discovery with.
|
||||||
@@ -118,10 +121,69 @@ type gateFacts struct {
|
|||||||
health map[string]inventory.NodeHealth
|
health map[string]inventory.NodeHealth
|
||||||
healthErr error
|
healthErr error
|
||||||
// heldOn is, per "<module>@<machine>", the provider its findings are held under (ADR 0240 rule 5).
|
// heldOn is, per "<module>@<machine>", the provider its findings are held under (ADR 0240 rule 5).
|
||||||
heldOn map[string]string
|
heldOn map[string]heldReading
|
||||||
// groupsAdded is, per module, whether the move judged puts an account in a group its previous build did
|
// groupsAdded is, per module, whether the move judged puts an account in a group its previous build did
|
||||||
// not (issue 318 review): the only move whose wait for a new login is excused.
|
// not (issue 318 review): the only move whose wait for a new login is excused.
|
||||||
groupsAdded map[string]bool
|
groupsAdded map[string]bool
|
||||||
|
// waits is what the controller holds to check a module's wait for the operator (novox/hq ADR 0283): the
|
||||||
|
// manifest of each module's build judged, and the secrets given on each machine.
|
||||||
|
waits operatorWaitFacts
|
||||||
|
// sent is, per machine, the declaration the gate's own send carried there (novox/hq issue 352): a
|
||||||
|
// report is held against it, never against the send made last. sentBuilds is what each machine was
|
||||||
|
// last sent of every module, and judged the commit of each module this gate judges: a machine last
|
||||||
|
// sent another build of the module is not running the build judged.
|
||||||
|
sent map[string]inventory.SentDeclaration
|
||||||
|
sentBuilds map[string]map[string]string
|
||||||
|
commits map[string]string
|
||||||
|
}
|
||||||
|
|
||||||
|
// heldReading is the provider a module's findings are held under, as "<module> on <machine>", and, when that
|
||||||
|
// provider only waits for the operator for the part the module needs, its wait in one sentence (novox/hq issue
|
||||||
|
// 405): the module's gate then reads as a wait for a person (ADR 0254), a pass carrying the wait, as ADR 0283
|
||||||
|
// decision 4 reads the waiting provider itself. A walk never waits on the operator's secret, there or here.
|
||||||
|
type heldReading struct {
|
||||||
|
on, waits string
|
||||||
|
}
|
||||||
|
|
||||||
|
// heldReadings is, per "<module>@<machine>" in every machine's newest statement, what its findings are held under.
|
||||||
|
func heldReadings(hold *holding) map[string]heldReading {
|
||||||
|
out := map[string]heldReading{}
|
||||||
|
for machine := range hold.healths {
|
||||||
|
for module, by := range hold.heldModules(machine) {
|
||||||
|
reading := heldReading{on: by.provider.Module + " on " + by.provider.Node}
|
||||||
|
if len(by.waits) > 0 {
|
||||||
|
reading.waits = operatorWaitSaid(by.provider.Module, by.provider.Node, by.waits)
|
||||||
|
}
|
||||||
|
out[module+"@"+machine] = reading
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// reportedOn says a machine's last report is on what the gate sent it (novox/hq issue 352): on that
|
||||||
|
// declaration, or one it was sent after it — or, for a gate kept before sends were kept on it, on the
|
||||||
|
// declaration last sent. On 2026-10-09 a release's gate read the control node's report against a newer
|
||||||
|
// send another plan had just made there, and failed three builds the machine had reported healthy as
|
||||||
|
// "has not reported on what it was sent".
|
||||||
|
func (f gateFacts) reportedOn(machine string, r inventory.Reported) bool {
|
||||||
|
if sent, kept := f.sent[machine]; kept {
|
||||||
|
return sent.ReportsOn(r)
|
||||||
|
}
|
||||||
|
return r.Current
|
||||||
|
}
|
||||||
|
|
||||||
|
// supersededOn says the machine was last sent another build of the module than the one this gate judges
|
||||||
|
// (novox/hq issue 352): the judging cannot go on, whatever the machine reports. On 2026-10-09 a controller
|
||||||
|
// put back by one gate judged another gate's newer controller build passed on the same machine, reading
|
||||||
|
// the put-back build's health as the newer one's.
|
||||||
|
func (f gateFacts) supersededOn(module, machine string) (string, bool) {
|
||||||
|
judged, known := f.commits[module]
|
||||||
|
sent, has := f.sentBuilds[machine][module]
|
||||||
|
if !known || !has || judged == "" || sent == "" || sameCommit(sent, judged) {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s was sent %s %s after this gate's %s: the build judged no longer runs there, and "+
|
||||||
|
"this judging is superseded by that send's", machine, module, short(sent), short(judged)), true
|
||||||
}
|
}
|
||||||
|
|
||||||
// gatherGateFacts reads what a judging needs, from the store, the bus and this controller's memory. A
|
// gatherGateFacts reads what a judging needs, from the store, the bus and this controller's memory. A
|
||||||
@@ -163,12 +225,7 @@ var gatherGateFacts = func(ctx context.Context, open *stores, component string)
|
|||||||
// Whose findings wait on an unhealthy provider (ADR 0240 rule 5): their gates wait, not fail.
|
// Whose findings wait on an unhealthy provider (ADR 0240 rule 5): their gates wait, not fail.
|
||||||
if f.healthErr == nil && f.openErr == nil {
|
if f.healthErr == nil && f.openErr == nil {
|
||||||
if hold, err := readHolding(ctx, inv, f.open); err == nil {
|
if hold, err := readHolding(ctx, inv, f.open); err == nil {
|
||||||
f.heldOn = map[string]string{}
|
f.heldOn = heldReadings(hold)
|
||||||
for machine := range f.health {
|
|
||||||
for module, p := range hold.heldModules(machine) {
|
|
||||||
f.heldOn[module+"@"+machine] = p.Module + " on " + p.Node
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if theLease != nil {
|
if theLease != nil {
|
||||||
@@ -199,9 +256,12 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string,
|
|||||||
return healthBroken, fmt.Sprintf("the witness on %s judged the %s %s and %s: %s", machine, r.Component,
|
return healthBroken, fmt.Sprintf("the witness on %s judged the %s %s and %s: %s", machine, r.Component,
|
||||||
short(r.From), r.Outcome, r.Why)
|
short(r.From), r.Outcome, r.Why)
|
||||||
}
|
}
|
||||||
|
if why, superseded := f.supersededOn(module, machine); superseded {
|
||||||
|
return healthSuperseded, why
|
||||||
|
}
|
||||||
r, said := f.reports[machine]
|
r, said := f.reports[machine]
|
||||||
switch {
|
switch {
|
||||||
case !said || r.At == nil || !r.Current:
|
case !said || r.At == nil || !f.reportedOn(machine, r):
|
||||||
return healthNotYet, fmt.Sprintf("%s has not reported on what it was sent", machine)
|
return healthNotYet, fmt.Sprintf("%s has not reported on what it was sent", machine)
|
||||||
case r.Outcome == inventory.OutcomeFailed || r.Outcome == inventory.OutcomeRefused:
|
case r.Outcome == inventory.OutcomeFailed || r.Outcome == inventory.OutcomeRefused:
|
||||||
return healthBroken, fmt.Sprintf("%s %s what it was sent", machine, r.Outcome)
|
return healthBroken, fmt.Sprintf("%s %s what it was sent", machine, r.Outcome)
|
||||||
@@ -218,10 +278,11 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string,
|
|||||||
firstLine(f.openErr.Error())
|
firstLine(f.openErr.Error())
|
||||||
}
|
}
|
||||||
for _, c := range f.open {
|
for _, c := range f.open {
|
||||||
// A wait for a person's new login, or for a directory used as found to be handed over, is the module's
|
// A wait for a person's new login, for a directory used as found to be handed over, or for the
|
||||||
// reading, not a fault raised since the send: the gate reads it from the statement below (ADR 0254,
|
// operator's secret or setting, is the module's reading, not a fault raised since the send: the gate
|
||||||
// novox/hq issue 339).
|
// reads it from the statement below (ADR 0254, novox/hq issue 339, ADR 0283).
|
||||||
if c.Source == gateProbe || c.OpenAt(since) || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound {
|
if c.Source == gateProbe || c.OpenAt(since) || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound ||
|
||||||
|
c.Kind == kindNeedsOperator {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
onIt := c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) ||
|
onIt := c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) ||
|
||||||
@@ -438,6 +499,22 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
|||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
facts.groupsAdded = movesAddingGroups(ctx, open.inventory, g, pairs, shelf)
|
facts.groupsAdded = movesAddingGroups(ctx, open.inventory, g, pairs, shelf)
|
||||||
|
facts.waits = gateWaitFacts(ctx, open.inventory, g, pairs, shelf, facts.health)
|
||||||
|
facts.sent = g.Sent
|
||||||
|
facts.commits, facts.sentBuilds = judgedCommits(g, pairs), map[string]map[string]string{}
|
||||||
|
// A module this gate put back at once (putBackBroken) was sent its earlier build by the gate itself:
|
||||||
|
// not another send, and not a judging superseded.
|
||||||
|
for _, m := range g.Returned {
|
||||||
|
delete(facts.commits, m)
|
||||||
|
}
|
||||||
|
for _, j := range pairs {
|
||||||
|
if _, read := facts.sentBuilds[j.node]; read {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if builds, known, err := open.inventory.SentBuilds(ctx, j.node); err == nil && known {
|
||||||
|
facts.sentBuilds[j.node] = builds
|
||||||
|
}
|
||||||
|
}
|
||||||
// **What is wrong with a machine itself is the machine's** (novox/hq issue 281): read once for each
|
// **What is wrong with a machine itself is the machine's** (novox/hq issue 281): read once for each
|
||||||
// machine judged, apart from what is wrong with a module there, and never pinned on the module the
|
// machine judged, apart from what is wrong with a module there, and never pinned on the module the
|
||||||
// gate happens to be kept on.
|
// gate happens to be kept on.
|
||||||
@@ -474,6 +551,13 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
|||||||
if _, seen := reading[j.module]; !seen {
|
if _, seen := reading[j.module]; !seen {
|
||||||
modules = append(modules, j.module)
|
modules = append(modules, j.module)
|
||||||
}
|
}
|
||||||
|
if h == healthSuperseded {
|
||||||
|
// Decided at once (novox/hq issue 352): nothing of this gate's can be judged on a machine that
|
||||||
|
// was sent another build of it, and nothing is put back — the later send is what runs there.
|
||||||
|
g.Failing, g.Last = nil, ""
|
||||||
|
decide(g, inventory.GateSuperseded, said, now)
|
||||||
|
return g.Verdict, nil
|
||||||
|
}
|
||||||
if h == healthBroken && !slices.Contains(g.Broken, j.module) {
|
if h == healthBroken && !slices.Contains(g.Broken, j.module) {
|
||||||
g.Broken = append(g.Broken, j.module)
|
g.Broken = append(g.Broken, j.module)
|
||||||
if g.BrokenWhy == "" {
|
if g.BrokenWhy == "" {
|
||||||
@@ -542,6 +626,7 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
|||||||
pastBound := now.Sub(*g.Since) > gateBound
|
pastBound := now.Sub(*g.Since) > gateBound
|
||||||
switch {
|
switch {
|
||||||
case worst == healthBroken:
|
case worst == healthBroken:
|
||||||
|
g.Read(now, false, g.BrokenWhy)
|
||||||
var judging []string
|
var judging []string
|
||||||
for _, m := range modules {
|
for _, m := range modules {
|
||||||
if reading[m] != healthBroken && !passedAlone(m) {
|
if reading[m] != healthBroken && !passedAlone(m) {
|
||||||
@@ -560,8 +645,10 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
|||||||
// Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either —
|
// Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either —
|
||||||
// the provider's own condition says what is wrong (ADR 0240 rule 5).
|
// the provider's own condition says what is wrong (ADR 0240 rule 5).
|
||||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
||||||
|
g.Read(now, false, why)
|
||||||
case worst == healthNotYet:
|
case worst == healthNotYet:
|
||||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
||||||
|
g.Read(now, false, why)
|
||||||
if pastBound {
|
if pastBound {
|
||||||
fail(fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why))
|
fail(fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why))
|
||||||
}
|
}
|
||||||
@@ -569,6 +656,7 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
|||||||
// Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict.
|
// Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict.
|
||||||
g.Passes++
|
g.Passes++
|
||||||
g.LastPass, g.Last, g.Failing = &now, "", nil
|
g.LastPass, g.Last, g.Failing = &now, "", nil
|
||||||
|
g.Read(now, true, "")
|
||||||
if g.Passes >= gatePasses && settled {
|
if g.Passes >= gatePasses && settled {
|
||||||
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
|
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
|
||||||
now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now)
|
now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now)
|
||||||
@@ -577,6 +665,40 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
|||||||
return g.Verdict, nil
|
return g.Verdict, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// judgedCommits is the commit of each module a gate judges: the gate's own To for its module, and each
|
||||||
|
// carried move's. Pure.
|
||||||
|
func judgedCommits(g *inventory.PlanGate, pairs []judged) map[string]string {
|
||||||
|
out := map[string]string{}
|
||||||
|
for _, c := range g.Carried {
|
||||||
|
if c.To != "" {
|
||||||
|
out[c.Module] = c.To
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if g.To != "" {
|
||||||
|
for _, j := range pairs {
|
||||||
|
if _, has := out[j.module]; !has && !slices.ContainsFunc(g.Carried, func(c inventory.CarriedMove) bool { return c.Module == j.module }) {
|
||||||
|
out[j.module] = g.To
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// sentNow is what each machine was just sent, read after a send for the gate to keep (novox/hq issue
|
||||||
|
// 352): a machine whose send is not on record is left out, and its report is read as before.
|
||||||
|
func sentNow(ctx context.Context, inv *inventory.Inventory, machines []string) map[string]inventory.SentDeclaration {
|
||||||
|
out := map[string]inventory.SentDeclaration{}
|
||||||
|
for _, n := range machines {
|
||||||
|
if s, found, err := inv.SentTo(ctx, n); err == nil && found {
|
||||||
|
out[n] = s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(out) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// whyFor is a passing gate's why as one module's verdict says it: the send's, and that module's own wait
|
// whyFor is a passing gate's why as one module's verdict says it: the send's, and that module's own wait
|
||||||
// for a person, never another's (issue 318 review).
|
// for a person, never another's (issue 318 review).
|
||||||
func whyFor(g *inventory.PlanGate, module string) string {
|
func whyFor(g *inventory.PlanGate, module string) string {
|
||||||
@@ -652,6 +774,47 @@ func movesAddingGroups(ctx context.Context, inv *inventory.Inventory, g *invento
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// gateWaitFacts reads what checks the waits for the operator of the modules a gate judges (novox/hq ADR 0283): the
|
||||||
|
// manifest of the build judged — the one it moves to, else the catalogue's — and the secrets given on each machine
|
||||||
|
// that says a module of them waits.
|
||||||
|
func gateWaitFacts(ctx context.Context, inv *inventory.Inventory, g *inventory.PlanGate, pairs []judged,
|
||||||
|
shelf map[string]catalogue.Manifest, health map[string]inventory.NodeHealth) operatorWaitFacts {
|
||||||
|
var f operatorWaitFacts
|
||||||
|
judgedManifests := map[string]catalogue.Manifest{}
|
||||||
|
byMachine := map[string][]string{}
|
||||||
|
for _, j := range pairs {
|
||||||
|
waiting := false
|
||||||
|
for _, r := range health[j.node].Resources {
|
||||||
|
if r.Module == j.module && r.State == link.StateWaiting {
|
||||||
|
waiting = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !waiting {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
byMachine[j.node] = append(byMachine[j.node], j.module)
|
||||||
|
if _, done := judgedManifests[j.module]; done {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
to := g.To
|
||||||
|
for _, c := range g.Carried {
|
||||||
|
if c.Module == j.module {
|
||||||
|
to = c.To
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if m, found, err := inv.ManifestAt(ctx, j.module, to); err == nil && found {
|
||||||
|
judgedManifests[j.module] = m
|
||||||
|
} else if m, known := shelf[j.module]; known {
|
||||||
|
judgedManifests[j.module] = m
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for machine, modules := range byMachine {
|
||||||
|
readWaitFacts(ctx, inv, machine, modules, judgedManifests, &f)
|
||||||
|
}
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|
||||||
// decide sets a gate's verdict.
|
// decide sets a gate's verdict.
|
||||||
func decide(g *inventory.PlanGate, verdict, why string, now time.Time) {
|
func decide(g *inventory.PlanGate, verdict, why string, now time.Time) {
|
||||||
g.Verdict, g.Why, g.JudgedAt = verdict, why, &now
|
g.Verdict, g.Why, g.JudgedAt = verdict, why, &now
|
||||||
@@ -802,6 +965,16 @@ func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module str
|
|||||||
"back", module, short(state.Previous), inventory.KeptBuilds))
|
"back", module, short(state.Previous), inventory.KeptBuilds))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if g.Component == lease.ComponentController {
|
||||||
|
// **The controller is never put back to a build older than the store's schema** (novox/hq issue
|
||||||
|
// 352): the build before it carries fewer migrations than the failed one applied, starts behind
|
||||||
|
// its own records, and judges the next gate with what it can read. The current build is kept and
|
||||||
|
// the condition says so; a person decides.
|
||||||
|
if why, ok := controllerSchemaAllows(ctx, inv, previous); !ok {
|
||||||
|
notBack(why)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
if err := inv.RestoreModule(ctx, previous); err != nil {
|
if err := inv.RestoreModule(ctx, previous); err != nil {
|
||||||
notBack(err.Error())
|
notBack(err.Error())
|
||||||
return
|
return
|
||||||
@@ -835,6 +1008,53 @@ func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module str
|
|||||||
sayRollback(ctx, open, module, g, "")
|
sayRollback(ctx, open, module, g, "")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// controllerSchemaAllows says the store's schema lets this build of the controller be put back: the
|
||||||
|
// build recorded, when it served, a reach at or past the highest migration the store has applied. One
|
||||||
|
// that never recorded a reach is not proved safe, and is refused as such (novox/hq issue 352). Why
|
||||||
|
// says what is kept and why when it is not.
|
||||||
|
func controllerSchemaAllows(ctx context.Context, inv *inventory.Inventory, previous inventory.Build) (string, bool) {
|
||||||
|
applied, err := inv.SchemaApplied(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "what the store's schema reaches cannot be read, so whether the build before it can read it is not " +
|
||||||
|
"known; the current build is kept: " + err.Error(), false
|
||||||
|
}
|
||||||
|
build := versionOfBuild(previous)
|
||||||
|
if build == "" {
|
||||||
|
return fmt.Sprintf("the build before it (%s) names no bundle to know it by, so whether it can read the store's "+
|
||||||
|
"schema (migration %04d) is not known; the current build is kept, and a person decides", short(previous.Commit), applied), false
|
||||||
|
}
|
||||||
|
reach, known, err := inv.SchemaReachOf(ctx, build)
|
||||||
|
if err != nil {
|
||||||
|
return "what the build before it knows of the store's schema cannot be read; the current build is kept: " + err.Error(), false
|
||||||
|
}
|
||||||
|
if !known {
|
||||||
|
return fmt.Sprintf("the build before it (%s, %s) never recorded how far it reads the store's schema — a "+
|
||||||
|
"controller records that when it serves — so it is not proved to read migration %04d, which the store "+
|
||||||
|
"has applied; a controller older than its store starts behind its own records and judges with what it "+
|
||||||
|
"can read, so the current build is kept, and a person decides", short(previous.Commit), build, applied), false
|
||||||
|
}
|
||||||
|
if reach < applied {
|
||||||
|
return fmt.Sprintf("the build before it (%s, %s) reads the store's schema up to migration %04d, and the store "+
|
||||||
|
"is at %04d: a controller older than its store starts behind its own records and judges with what it "+
|
||||||
|
"can read, so the current build is kept, and a person decides", short(previous.Commit), build, reach, applied), false
|
||||||
|
}
|
||||||
|
return "", true
|
||||||
|
}
|
||||||
|
|
||||||
|
// versionOfBuild is the version a build's bundle is delivered as — its archive's digest, short, as the
|
||||||
|
// catalogue names it (`${version}`) — read from the build's artifacts; empty when none is a bundle.
|
||||||
|
func versionOfBuild(b inventory.Build) string {
|
||||||
|
for _, a := range b.Made {
|
||||||
|
if a.Kind != catalogue.ArtifactBundle && a.Kind != catalogue.ArtifactArchive {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, hex, found := strings.Cut(a.Reference, "sha256:"); found && len(hex) >= 12 {
|
||||||
|
return hex[:12]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
// rollbacks is what a failed send puts back, sent together (novox/hq issue 281): a gate that judged one
|
// rollbacks is what a failed send puts back, sent together (novox/hq issue 281): a gate that judged one
|
||||||
// send judges what it moved as one, and what it found wanting goes back in one send per machine — not
|
// send judges what it moved as one, and what it found wanting goes back in one send per machine — not
|
||||||
// in a send for each module, which is the churn that failed the gate in the first place.
|
// in a send for each module, which is the churn that failed the gate in the first place.
|
||||||
|
|||||||
@@ -113,9 +113,10 @@ func aGateMesh(t *testing.T) *gateMesh {
|
|||||||
}
|
}
|
||||||
for node, h := range g.health {
|
for node, h := range g.health {
|
||||||
if h == healthNotYet {
|
if h == healthNotYet {
|
||||||
|
// Not reported on the send: neither the send made last, nor the gate's own (issue 352).
|
||||||
f.rolledBack[node] = nil
|
f.rolledBack[node] = nil
|
||||||
r := f.reports[node]
|
r := f.reports[node]
|
||||||
r.Current = false
|
r.Current, r.Declared, r.ReportedSequence = false, "", 0
|
||||||
f.reports[node] = r
|
f.reports[node] = r
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,222 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The assignment generation a declaration was composed from, and the record of every send (novox/hq
|
||||||
|
// issue 234).
|
||||||
|
//
|
||||||
|
// On 2026-10-04 a declaration newer in sequence than every other named four fewer modules than the
|
||||||
|
// assignments held, and a machine applied it and undeclared all four. The sequence orders arrival and
|
||||||
|
// cannot tell a later send that carries an older view of the assignments. So a declaration now carries
|
||||||
|
// the generation of the assignments it was composed from — a counter the store raises in the same
|
||||||
|
// transaction as every assignment change — and a machine refuses one older than it applied, unless it is
|
||||||
|
// a gate's put-back. And every send is written down with who sent it, from which generation and naming
|
||||||
|
// which modules: the controller logged no send then, and which process sent the stale declaration could
|
||||||
|
// not be read back from anything the mesh kept.
|
||||||
|
|
||||||
|
// kindOlderGeneration is S20's kind: a machine refused a send for the generation it was composed from.
|
||||||
|
const kindOlderGeneration = "older-generation"
|
||||||
|
|
||||||
|
// generationRefusalsSaid is how long a refused send is said after its refusal: an hour, as an advisory is.
|
||||||
|
const generationRefusalsSaid = advisoryQuiet
|
||||||
|
|
||||||
|
// stamp gives a composed declaration the order allotted to it before it was composed: its sequence, and
|
||||||
|
// the epoch and generation when the machine reads them — and keeps the generation and acting epoch for the
|
||||||
|
// record of the send whether or not the machine is sent them.
|
||||||
|
//
|
||||||
|
// **No put-back mark is sent.** A gate puts a machine back by composing it again (sendRollout), so its
|
||||||
|
// declaration is allotted here like any other and carries the generation as it stands — never older than
|
||||||
|
// what the machine applied. The node-engine reads a `put_back` key (mesh-host#82); this controller never
|
||||||
|
// needs to send one.
|
||||||
|
func (o order) stamp(d *sendable) {
|
||||||
|
d.Sequence, d.Epoch = o.sequence, o.epoch
|
||||||
|
d.composedFrom, d.actingEpoch = o.generation, o.acting
|
||||||
|
d.Generation = 0
|
||||||
|
if o.readsGeneration && o.generation > 0 {
|
||||||
|
d.Generation = o.generation
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// sentRecord is what the record of a send keeps beside its digest.
|
||||||
|
type sentRecord struct {
|
||||||
|
sequence int64
|
||||||
|
epoch uint64
|
||||||
|
generation int64
|
||||||
|
// toldGeneration is the generation on the wire, which the would-send is stamped with: zero for a machine
|
||||||
|
// whose node-engine has not said it reads one.
|
||||||
|
toldGeneration int64
|
||||||
|
sender string
|
||||||
|
modules []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// sentRecordOf is a composed send's record: its sender is the caller this process acts for.
|
||||||
|
func sentRecordOf(ctx context.Context, d sendable) sentRecord {
|
||||||
|
return sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom,
|
||||||
|
toldGeneration: d.Generation, sender: senderOf(callerOf(ctx)), modules: d.modules}
|
||||||
|
}
|
||||||
|
|
||||||
|
// callerOf is who asked for what this process does: the seat call's caller when ctx belongs to one, the
|
||||||
|
// caller the controller ran this command for, or the account at the shell.
|
||||||
|
func callerOf(ctx context.Context) string {
|
||||||
|
if c := link.CallerIn(ctx); c != "" {
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
return link.Caller()
|
||||||
|
}
|
||||||
|
|
||||||
|
// senderOf is a send's sender in words: who asked, and the process and build that composed it — the
|
||||||
|
// answer issue 234 could not find, because two controllers and a one-shot push were all sending then.
|
||||||
|
func senderOf(caller string) string {
|
||||||
|
host, _ := os.Hostname()
|
||||||
|
return fmt.Sprintf("%s (pid %d on %s, build %s)", caller, os.Getpid(), host, version)
|
||||||
|
}
|
||||||
|
|
||||||
|
// namedModules are the modules a declaration names: its machine's set, less what was left out of it.
|
||||||
|
func namedModules(plan catalogue.Resolution, leftOut map[string]string) []string {
|
||||||
|
out := make([]string, 0, len(plan.Modules))
|
||||||
|
for _, m := range plan.Modules {
|
||||||
|
if _, left := leftOut[m.Module]; !left {
|
||||||
|
out = append(out, m.Module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// heardGenerationRefusal keeps a machine's refusal of a send for its generation, so S20 names its sender
|
||||||
|
// and says it (novox/hq issue 234): on the send it refused or — when the mesh has no record of sending that
|
||||||
|
// sequence — as a refusal of its own, naming the sender as unknown. Either way S20 is raised: a refusal the
|
||||||
|
// mesh cannot attribute is the louder fact, not a quieter one.
|
||||||
|
//
|
||||||
|
// **And raises the mesh's counter past what the machine applied, when the refused send was composed from
|
||||||
|
// the counter as it stands** (counterBehind): then the sender's view was not stale — the counter is behind
|
||||||
|
// the machine, which is a store put back from a backup — and every send after would be refused for ever. A
|
||||||
|
// stale sender's generation is below the counter, and the counter is left alone for it.
|
||||||
|
//
|
||||||
|
// Nothing is sent from here. This runs in the controller's receive loop, and a push from it would hold that
|
||||||
|
// loop, and the machine's hold, for as long as the push takes — the deaf controller of issues 184 and 185.
|
||||||
|
// S20 names `push <node>` for that case instead.
|
||||||
|
func heardGenerationRefusal(ctx context.Context, inv *inventory.Inventory, report link.Report) {
|
||||||
|
r := report.OlderGeneration
|
||||||
|
if r == nil || inv == nil || report.Node == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
node, err := inv.NodeByName(ctx, report.Node)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the machine cannot be "+
|
||||||
|
"read, so it is not raised: %v\n", report.Node, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var raised int64
|
||||||
|
if now, err := inv.AssignmentGeneration(ctx); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the mesh's own cannot be "+
|
||||||
|
"read: %v\n", report.Node, err)
|
||||||
|
} else if counterBehind(*r, now) {
|
||||||
|
if raised, err = inv.RaiseAssignmentGeneration(ctx, r.Applied); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation (%d) is behind what %s applied (%d), "+
|
||||||
|
"and could not be raised: %v\n", now, report.Node, r.Applied, err)
|
||||||
|
raised = 0
|
||||||
|
} else {
|
||||||
|
fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation was %d, behind what %s applied (%d) — "+
|
||||||
|
"a store put back from a backup — and is raised to %d; `push %s` sends it what the mesh holds now\n",
|
||||||
|
now, report.Node, r.Applied, raised, report.Node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
send, found, err := inv.RefusedSend(ctx, node.ID, report.Sequence, r.Applied, raised)
|
||||||
|
if err == nil && !found {
|
||||||
|
send, err = inv.RecordUnrecordedRefusal(ctx, inventory.Send{Node: node.ID, Sequence: report.Sequence,
|
||||||
|
Epoch: report.Epoch, Generation: r.Generation, Digest: report.Declared, RefusedApplied: r.Applied,
|
||||||
|
CounterRaisedTo: raised})
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d for its generation, and the refusal could not be "+
|
||||||
|
"kept, so it is not raised: %v\n", report.Node, report.Sequence, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d from %s: composed from assignment generation %d, "+
|
||||||
|
"and it applied %d\n", report.Node, report.Sequence, send.Sender, r.Generation, r.Applied)
|
||||||
|
}
|
||||||
|
|
||||||
|
// counterBehind says a refusal shows the mesh's counter behind the machine rather than a stale sender: the
|
||||||
|
// refused send carried the counter as it stands now, and the machine applied more than that.
|
||||||
|
func counterBehind(r link.GenerationRefusal, now int64) bool {
|
||||||
|
return r.Generation >= now && r.Applied > r.Generation
|
||||||
|
}
|
||||||
|
|
||||||
|
// watchGenerationRefusals is S20: every send a machine refused within the hour for the generation it was
|
||||||
|
// composed from, naming who sent it (novox/hq issue 234). One per machine, the newest refusal.
|
||||||
|
func watchGenerationRefusals(f *signalFacts) []conditions.Observation {
|
||||||
|
seen := map[string]bool{}
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, s := range f.refusedSends {
|
||||||
|
if s.RefusedAt == nil || f.now.Sub(*s.RefusedAt) > generationRefusalsSaid || seen[s.NodeName] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[s.NodeName] = true
|
||||||
|
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: s.NodeName, Kind: kindOlderGeneration,
|
||||||
|
Machine: s.NodeName, Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, and "+
|
||||||
|
"%s applied generation %d — a sender composing from a view of the assignments the mesh has moved "+
|
||||||
|
"past; it named %s", s.NodeName, s.Sequence, s.Sender, s.Generation, s.NodeName, s.RefusedApplied,
|
||||||
|
modulesWords(s.Modules)),
|
||||||
|
Said: fmt.Sprintf("refused at %s", s.RefusedAt.UTC().Format(time.RFC3339)),
|
||||||
|
Headline: fmt.Sprintf("%s refused an out-of-date update", s.NodeName),
|
||||||
|
Explanation: fmt.Sprintf("Something sent %s an update made from an older list of what runs there. %s "+
|
||||||
|
"refused it, so nothing was removed. Nothing for you to do unless it repeats.", s.NodeName, s.NodeName),
|
||||||
|
Resolved: fmt.Sprintf("%s has had no out-of-date update for an hour", s.NodeName)}
|
||||||
|
if s.CounterRaisedTo > 0 {
|
||||||
|
// Not a stale sender: the mesh's counter was behind the machine (a store put back from a backup) and
|
||||||
|
// was raised; nothing has sent the machine its declaration since, so a person is asked to.
|
||||||
|
o.Summary = fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, "+
|
||||||
|
"the mesh's own, and %s applied generation %d — the mesh's counter was behind the machine (a store "+
|
||||||
|
"put back from a backup?) and is raised to %d; `push %s` sends it what the mesh holds now",
|
||||||
|
s.NodeName, s.Sequence, s.Sender, s.Generation, s.NodeName, s.RefusedApplied, s.CounterRaisedTo,
|
||||||
|
s.NodeName)
|
||||||
|
o.Explanation = fmt.Sprintf("Needs you: push %s. The mesh's record was older than %s, so %s refused its "+
|
||||||
|
"update and kept what it had. The record is repaired; a push sends the update again.",
|
||||||
|
s.NodeName, s.NodeName, s.NodeName)
|
||||||
|
}
|
||||||
|
out = append(out, o)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// modulesWords is a send's modules as a sentence says them.
|
||||||
|
func modulesWords(modules []string) string {
|
||||||
|
if len(modules) == 0 {
|
||||||
|
return "no module the mesh recorded"
|
||||||
|
}
|
||||||
|
return strings.Join(modules, ", ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234):
|
||||||
|
// nothing when the mesh has not recorded a send to it.
|
||||||
|
func writeLastSend(ctx context.Context, w io.Writer, inv *inventory.Inventory, node string) error {
|
||||||
|
s, found, err := inv.LastSend(ctx, node)
|
||||||
|
if err != nil || !found {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
generation := "no generation recorded"
|
||||||
|
if s.Generation > 0 {
|
||||||
|
generation = fmt.Sprintf("assignment generation %d", s.Generation)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(w, "%s was last sent sequence %d at %s by %s, composed from %s, naming %s\n", node, s.Sequence, s.SentAt.Local().Format("2006-01-02 15:04:05"), s.Sender, generation, modulesWords(s.Modules))
|
||||||
|
if s.RefusedAt != nil {
|
||||||
|
fmt.Fprintf(w, " and refused it at %s: it had applied generation %d\n",
|
||||||
|
s.RefusedAt.Local().Format("2006-01-02 15:04:05"), s.RefusedApplied)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,148 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The assignment generation a declaration was composed from (novox/hq issue 234).
|
||||||
|
|
||||||
|
func bodyKeys(t *testing.T, s sendable) map[string]any {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := s.Body()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var keys map[string]any
|
||||||
|
if err := json.Unmarshal(raw, &keys); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return keys
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The generation goes on the wire only to a machine whose node-engine said it reads one**: an older
|
||||||
|
// node-engine decodes strictly and refuses an unknown key, whole. No put-back mark is ever sent: a gate
|
||||||
|
// composes its put-back afresh, with the generation as it stands.
|
||||||
|
func TestTheGenerationIsSentOnlyToAMachineThatReadsOne(t *testing.T) {
|
||||||
|
resources := []map[string]any{{"id": "a", "type": "file", "path": "/etc/a", "content": "x\n"}}
|
||||||
|
reads := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: true, acting: 57}
|
||||||
|
var told sendable
|
||||||
|
told.Resources = resources
|
||||||
|
reads.stamp(&told)
|
||||||
|
keys := bodyKeys(t, told)
|
||||||
|
if keys["generation"] != float64(40) || keys["sequence"] != float64(12) {
|
||||||
|
t.Fatalf("a machine that reads a generation was sent %v", keys)
|
||||||
|
}
|
||||||
|
if _, there := keys["put_back"]; there {
|
||||||
|
t.Fatalf("a put-back mark was sent: %v", keys)
|
||||||
|
}
|
||||||
|
if told.composedFrom != 40 || told.actingEpoch != 57 {
|
||||||
|
t.Errorf("the send does not keep what it was composed from for its record: %+v", told)
|
||||||
|
}
|
||||||
|
|
||||||
|
older := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: false, acting: 57}
|
||||||
|
var untold sendable
|
||||||
|
untold.Resources = resources
|
||||||
|
older.stamp(&untold)
|
||||||
|
if _, there := bodyKeys(t, untold)["generation"]; there {
|
||||||
|
t.Fatal("a machine whose node-engine never said it reads a generation was sent one")
|
||||||
|
}
|
||||||
|
if untold.composedFrom != 40 {
|
||||||
|
t.Errorf("the generation it was composed from is recorded whether or not it was sent: %+v", untold)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The sender is named**: the caller of the seat call when there is one, else the shell's account, and the
|
||||||
|
// process and build that composed it.
|
||||||
|
func TestASendNamesItsSender(t *testing.T) {
|
||||||
|
said := senderOf("g14.node-tools, through the mesh-controller seat")
|
||||||
|
if !strings.Contains(said, "g14.node-tools") || !strings.Contains(said, "pid ") || !strings.Contains(said, "build ") {
|
||||||
|
t.Fatalf("the sender reads %q", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// refusedSend is a send a machine refused for its generation at a moment.
|
||||||
|
func refusedSend(at time.Time) inventory.Send {
|
||||||
|
return inventory.Send{NodeName: "anchor", Sequence: 12, Epoch: 57, Generation: 38, RefusedApplied: 40,
|
||||||
|
Sender: "a one-shot push by jochen at a shell on anchor (pid 4242 on anchor, build 2026.10.11)",
|
||||||
|
Modules: []string{"docker"}, SentAt: at.Add(-time.Second), RefusedAt: &at, Recorded: true}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A refused send is raised naming its sender** (novox/hq issue 234): who sent it, from which generation,
|
||||||
|
// against which the machine applied, and its sequence — the facts that took a morning to look for.
|
||||||
|
func TestARefusedSendIsRaisedNamingItsSender(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC)
|
||||||
|
f := calm(now)
|
||||||
|
f.refusedSends = []inventory.Send{refusedSend(now.Add(-time.Minute))}
|
||||||
|
got := watchGenerationRefusals(f)
|
||||||
|
if len(got) != 1 {
|
||||||
|
t.Fatalf("%+v", got)
|
||||||
|
}
|
||||||
|
o := got[0]
|
||||||
|
if o.Key() != "machine.anchor.older-generation" || o.Machine != "anchor" {
|
||||||
|
t.Errorf("raised as %s about %q", o.Key(), o.Machine)
|
||||||
|
}
|
||||||
|
for _, want := range []string{"a one-shot push by jochen", "generation 38", "generation 40", "sequence 12"} {
|
||||||
|
if !strings.Contains(o.Summary, want) {
|
||||||
|
t.Errorf("the summary does not say %q: %s", want, o.Summary)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if o.Headline == "" || o.Explanation == "" || o.Resolved == "" {
|
||||||
|
t.Errorf("the condition is not worded for the operator: %+v", o)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A refusal of a send the mesh has no record of is raised too, its sender said to be unknown** — never
|
||||||
|
// only a line on stderr.
|
||||||
|
func TestARefusalOfAnUnrecordedSendIsRaisedSayingTheSenderIsUnknown(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC)
|
||||||
|
f := calm(now)
|
||||||
|
s := refusedSend(now.Add(-time.Minute))
|
||||||
|
s.Recorded, s.Sender, s.Modules = false, "a sender the mesh has no record of (no send of this sequence was recorded)", nil
|
||||||
|
f.refusedSends = []inventory.Send{s}
|
||||||
|
got := watchGenerationRefusals(f)
|
||||||
|
if len(got) != 1 || !strings.Contains(got[0].Summary, "no record of") {
|
||||||
|
t.Fatalf("an unattributed refusal was not raised as one: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **When the refusal showed the mesh's counter behind the machine, the condition asks for a push** — it was
|
||||||
|
// raised, and nothing has sent the machine its declaration since — and does not say there is nothing to do.
|
||||||
|
func TestACounterBehindTheMachineAsksForAPush(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC)
|
||||||
|
f := calm(now)
|
||||||
|
s := refusedSend(now.Add(-time.Minute))
|
||||||
|
s.Generation, s.RefusedApplied, s.CounterRaisedTo = 12, 40, 41
|
||||||
|
f.refusedSends = []inventory.Send{s}
|
||||||
|
got := watchGenerationRefusals(f)
|
||||||
|
if len(got) != 1 {
|
||||||
|
t.Fatalf("%+v", got)
|
||||||
|
}
|
||||||
|
if !strings.Contains(got[0].Summary, "`push anchor`") || !strings.Contains(got[0].Explanation, "push anchor") ||
|
||||||
|
strings.Contains(got[0].Explanation, "Nothing for you to do") {
|
||||||
|
t.Errorf("a counter behind the machine does not ask for a push: %s / %s", got[0].Summary, got[0].Explanation)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Only a refusal of the counter as it stands is the counter behind**: a stale sender's generation is below
|
||||||
|
// it, and the counter is left alone for that.
|
||||||
|
func TestOnlyARefusalOfTheCounterAsItStandsRaisesIt(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
refused, applied, now int64
|
||||||
|
behind bool
|
||||||
|
}{
|
||||||
|
{refused: 12, applied: 40, now: 12, behind: true}, // the store was put back: the mesh says 12, the machine had 40
|
||||||
|
{refused: 38, applied: 40, now: 41, behind: false}, // a stale sender: the counter is already past
|
||||||
|
{refused: 38, applied: 40, now: 40, behind: false}, // a stale sender: the counter is where the machine is
|
||||||
|
} {
|
||||||
|
r := link.GenerationRefusal{Generation: c.refused, Applied: c.applied}
|
||||||
|
if got := counterBehind(r, c.now); got != c.behind {
|
||||||
|
t.Errorf("refused %d, applied %d, counter %d: behind %v, want %v", c.refused, c.applied, c.now, got, c.behind)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -110,6 +110,9 @@ var handActVerbs = []handActVerb{
|
|||||||
// to that judgement. Several values rotate for one leak, and a leak that recurs is a defect of the
|
// to that judgement. Several values rotate for one leak, and a leak that recurs is a defect of the
|
||||||
// module that prints them, an issue against it, not a healer that rotates. A rotation for any other
|
// module that prints them, an issue against it, not a healer that rotates. A rotation for any other
|
||||||
// cause — a credential that stopped working — counts: a schedule or a healer could take it over.
|
// cause — a credential that stopped working — counts: a schedule or a healer could take it over.
|
||||||
|
// A value given at the desk (novox/hq ADR 0259 §10): an outside party's key, such as a bot token, which
|
||||||
|
// only a person can give. Their word, never a repair.
|
||||||
|
{Verb: "secret accept", Decision: "a value an outside party issued is given by a person, at their desk"},
|
||||||
{Verb: "secret rotate", Decision: "a value a person judged disclosed is replaced on their word",
|
{Verb: "secret rotate", Decision: "a value a person judged disclosed is replaced on their word",
|
||||||
DecidedFor: []string{causeLeakedInLogs}},
|
DecidedFor: []string{causeLeakedInLogs}},
|
||||||
}
|
}
|
||||||
@@ -248,6 +251,9 @@ func handActCommand(ctx context.Context, args []string) error {
|
|||||||
if len(args) > 0 && args[0] == "drill" {
|
if len(args) > 0 && args[0] == "drill" {
|
||||||
return handActDrill(ctx, args[1:])
|
return handActDrill(ctx, args[1:])
|
||||||
}
|
}
|
||||||
|
if len(args) > 0 && args[0] == "warrant" {
|
||||||
|
return handActWarrantCommand(ctx, args[1:])
|
||||||
|
}
|
||||||
if len(args) > 0 && args[0] != "list" && !strings.HasPrefix(args[0], "-") {
|
if len(args) > 0 && args[0] != "list" && !strings.HasPrefix(args[0], "-") {
|
||||||
return errors.New("hand-act record <what> --why <text> --cause <word> | hand-act drill <what> --why <text> " +
|
return errors.New("hand-act record <what> --why <text> --cause <word> | hand-act drill <what> --why <text> " +
|
||||||
"| hand-acts [--days N] [--json]")
|
"| hand-acts [--days N] [--json]")
|
||||||
|
|||||||
@@ -0,0 +1,169 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A hand-over's line is judged before anything is asked (novox/hq issue 356): a node's name and the directory's
|
||||||
|
// absolute path exactly as the engine states it — no `..`, no doubled or trailing separator, nothing relative.
|
||||||
|
func TestAHandOverLineIsJudgedBeforeItIsAsked(t *testing.T) {
|
||||||
|
for _, args := range [][]string{
|
||||||
|
{},
|
||||||
|
{"laptop"},
|
||||||
|
{"laptop", "/srv/notes", "extra"},
|
||||||
|
{"", "/srv/notes"},
|
||||||
|
{"--node", "/srv/notes"},
|
||||||
|
{"laptop", "srv/notes"},
|
||||||
|
{"laptop", "/srv/../etc"},
|
||||||
|
{"laptop", "/srv//notes"},
|
||||||
|
{"laptop", "/srv/notes/"},
|
||||||
|
{"laptop", "/srv/notes/."},
|
||||||
|
} {
|
||||||
|
if _, _, err := handOverLine(args); err == nil {
|
||||||
|
t.Errorf("%q was taken", args)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
node, path, err := handOverLine([]string{"laptop", "/srv/notes"})
|
||||||
|
if err != nil || node != "laptop" || path != "/srv/notes" {
|
||||||
|
t.Fatalf("read as %q %q %v", node, path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Who hands over is the line's caller in the words every verb's caller is recorded in — the operator through
|
||||||
|
// mesh-cli — or the controller's terminal when nobody is named.
|
||||||
|
func TestAHandOverNamesWhoAsked(t *testing.T) {
|
||||||
|
t.Setenv(link.CallerVar, " jo through mesh-cli on anchor ")
|
||||||
|
if by := handOverBy(); by != "jo through mesh-cli on anchor" {
|
||||||
|
t.Fatalf("by %q", by)
|
||||||
|
}
|
||||||
|
t.Setenv(link.CallerVar, "")
|
||||||
|
if by := handOverBy(); by != "the controller's terminal" {
|
||||||
|
t.Fatalf("by %q", by)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A hand-over is the controller's terminal's alone** (novox/hq issue 356, ADR 0266): through any verb, and
|
||||||
|
// through mesh-cli outside the terminal, `node hand-over` is refused and nothing runs — at the next apply root
|
||||||
|
// gives the directory to the account the module declares, and whoever may call a verb includes agents.
|
||||||
|
func TestAHandOverIsRefusedThroughEveryVerb(t *testing.T) {
|
||||||
|
line := []string{"node", "hand-over", "laptop", "/srv/notes"}
|
||||||
|
if err := terminalOnly(line); err == nil {
|
||||||
|
t.Fatal("node hand-over passed as a verb's line")
|
||||||
|
}
|
||||||
|
if _, err := argvFor("command", map[string]any{"command": "node hand-over laptop /srv/notes"}); err == nil {
|
||||||
|
t.Fatal("the command verb composed node hand-over")
|
||||||
|
}
|
||||||
|
if _, err := ordinaryLine(line); err == nil {
|
||||||
|
t.Fatal("node hand-over composed as an ordinary mesh-cli line")
|
||||||
|
}
|
||||||
|
if _, err := argvFor("node", map[string]any{"node": "laptop", "hand-over": "/srv/notes"}); err == nil {
|
||||||
|
t.Fatal("the node verb composed a hand-over")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The condition's words are plain and name no machine's binary; the operator's line, with the node and the path
|
||||||
|
// (novox/hq ADR 0272), is in the summary the module's health gives (moduleHealthWord) and in the evidence.
|
||||||
|
func TestTheUsedAsFoundConditionNamesTheOperatorsLine(t *testing.T) {
|
||||||
|
rs := []inventory.ResourceHealth{foundDirectory("notes", time.Now().Add(-24*time.Hour))}
|
||||||
|
o := usedAsFoundObservation("notes", "laptop", "notes on laptop uses notes.data as found", rs)
|
||||||
|
if strings.Contains(o.Needs, "mesh-host") || strings.Contains(o.Explanation, "mesh-host") ||
|
||||||
|
!strings.Contains(o.Needs, "control-node") {
|
||||||
|
t.Fatalf("the condition's words: %q %q", o.Needs, o.Explanation)
|
||||||
|
}
|
||||||
|
if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Needs: o.Needs,
|
||||||
|
Resolved: o.Resolved}, "laptop"); !ok {
|
||||||
|
t.Fatalf("not plain: %s", why)
|
||||||
|
}
|
||||||
|
f := gateFacts{now: time.Now(), health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: time.Now(),
|
||||||
|
Resources: rs}}}
|
||||||
|
h, why := moduleHealthWord("notes", "laptop", time.Now().Add(-time.Hour), f)
|
||||||
|
if h != healthPerson || !strings.Contains(why, "`nox node hand-over laptop <directory>` on the control-node") ||
|
||||||
|
strings.Contains(why, "mesh-host") || strings.Contains(why, "/srv/") {
|
||||||
|
t.Fatalf("the module's health reads %v %q; want the operator's line", h, why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Nothing is asked of a node the mesh does not know, and the engine's refusal is the command's failure**
|
||||||
|
// (review of issue 356): a refused hand-over never exits as a success.
|
||||||
|
func TestAHandOverAsksOnlyAKnownNodeAndFailsOnARefusal(t *testing.T) {
|
||||||
|
t.Setenv(link.CallerVar, "jo through mesh-cli on anchor")
|
||||||
|
asked := 0
|
||||||
|
ask := func(answer link.HandOverAnswer) func(node, path, by string) (link.HandOverAnswer, error) {
|
||||||
|
return func(node, path, by string) (link.HandOverAnswer, error) {
|
||||||
|
asked++
|
||||||
|
if node != "laptop" || path != "/srv/notes" || by != "jo through mesh-cli on anchor" {
|
||||||
|
t.Fatalf("asked %q %q %q", node, path, by)
|
||||||
|
}
|
||||||
|
return answer, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
unknown := func(string) error { return errors.New("no node called laptop") }
|
||||||
|
known := func(string) error { return nil }
|
||||||
|
var out bytes.Buffer
|
||||||
|
err := handOverAsked([]string{"laptop", "/srv/notes"}, unknown, ask(link.HandOverAnswer{Said: "x"}), &out)
|
||||||
|
if err == nil || asked != 0 || !strings.Contains(err.Error(), "nothing was asked") {
|
||||||
|
t.Fatalf("an unknown node: %v, asked %d", err, asked)
|
||||||
|
}
|
||||||
|
err = handOverAsked([]string{"laptop", "/srv/../etc"}, known, ask(link.HandOverAnswer{Said: "x"}), &out)
|
||||||
|
if err == nil || asked != 0 {
|
||||||
|
t.Fatalf("a refused line was asked: %v, asked %d", err, asked)
|
||||||
|
}
|
||||||
|
err = handOverAsked([]string{"laptop", "/srv/notes"}, known,
|
||||||
|
ask(link.HandOverAnswer{Refused: "/srv/notes is not used as found; nothing was handed over"}), &out)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "laptop refused: /srv/notes is not used as found") || out.Len() != 0 {
|
||||||
|
t.Fatalf("a refusal: %v, printed %q", err, out.String())
|
||||||
|
}
|
||||||
|
err = handOverAsked([]string{"laptop", "/srv/notes"}, known, ask(link.HandOverAnswer{Said: "handed over"}), &out)
|
||||||
|
if err != nil || !strings.HasPrefix(out.String(), "handed over\n") || !strings.Contains(out.String(), "`nox push laptop`") {
|
||||||
|
t.Fatalf("a record: %v, printed %q", err, out.String())
|
||||||
|
}
|
||||||
|
failing := func(string, string, string) (link.HandOverAnswer, error) {
|
||||||
|
return link.HandOverAnswer{}, errors.New("no engine")
|
||||||
|
}
|
||||||
|
if err := handOverAsked([]string{"laptop", "/srv/notes"}, known, failing, &out); err == nil {
|
||||||
|
t.Fatal("an ask that failed was a success")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The setuid search's line asks only a known node, one name and nothing else, and fails on a refusal
|
||||||
|
// (novox/hq issue 361).
|
||||||
|
func TestASetuidSearchAsksOnlyAKnownNodeAndFailsOnARefusal(t *testing.T) {
|
||||||
|
t.Setenv(link.CallerVar, "jo through mesh-cli on anchor")
|
||||||
|
asked := 0
|
||||||
|
ask := func(answer link.HandOverAnswer) func(node, by string) (link.HandOverAnswer, error) {
|
||||||
|
return func(node, by string) (link.HandOverAnswer, error) {
|
||||||
|
asked++
|
||||||
|
if node != "novox" || by != "jo through mesh-cli on anchor" {
|
||||||
|
t.Fatalf("asked %q %q", node, by)
|
||||||
|
}
|
||||||
|
return answer, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
known := func(string) error { return nil }
|
||||||
|
var out bytes.Buffer
|
||||||
|
for _, args := range [][]string{nil, {"novox", "extra"}, {"-x"}} {
|
||||||
|
if err := setuidSearchAsked(args, known, ask(link.HandOverAnswer{Said: "x"}), &out); err == nil || asked != 0 {
|
||||||
|
t.Fatalf("%v was asked: %v", args, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := setuidSearchAsked([]string{"novox"}, func(string) error { return errors.New("no node called novox") },
|
||||||
|
ask(link.HandOverAnswer{Said: "x"}), &out); err == nil || asked != 0 {
|
||||||
|
t.Fatalf("an unknown node: %v", err)
|
||||||
|
}
|
||||||
|
if err := setuidSearchAsked([]string{"novox"}, known, ask(link.HandOverAnswer{Refused: "no; no search was started"}),
|
||||||
|
&out); err == nil || !strings.Contains(err.Error(), "novox refused") || out.Len() != 0 {
|
||||||
|
t.Fatalf("a refusal: %v, printed %q", err, out.String())
|
||||||
|
}
|
||||||
|
if err := setuidSearchAsked([]string{"novox"}, known, ask(link.HandOverAnswer{Said: "a new search starts"}),
|
||||||
|
&out); err != nil || !strings.HasPrefix(out.String(), "a new search starts\n") {
|
||||||
|
t.Fatalf("a start: %v, printed %q", err, out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -625,7 +625,7 @@ func planStale(ctx context.Context, inv *inventory.Inventory, p inventory.Plan)
|
|||||||
}
|
}
|
||||||
for _, newer := range recent {
|
for _, newer := range recent {
|
||||||
if newer.ID == p.ID || !newer.Created.After(p.Created) || !repositoryMatches(newer.Repository, p.Repository) ||
|
if newer.ID == p.ID || !newer.Created.After(p.Created) || !repositoryMatches(newer.Repository, p.Repository) ||
|
||||||
newer.Branch != p.Branch || newer.State == inventory.PlanSuperseded {
|
newer.Branch != p.Branch || newer.State == inventory.PlanSuperseded || newer.Batch() {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
return inventory.PlanSuperseded, fmt.Sprintf("superseded by %s (%s %s), a newer merge of the same repository "+
|
return inventory.PlanSuperseded, fmt.Sprintf("superseded by %s (%s %s), a newer merge of the same repository "+
|
||||||
|
|||||||
@@ -128,7 +128,7 @@ func (r *recordedDelivery) grant(context.Context, []readyNode) error { return ni
|
|||||||
|
|
||||||
func (r *recordedDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
|
func (r *recordedDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
|
||||||
r.declared = append(r.declared, s.node)
|
r.declared = append(r.declared, s.node)
|
||||||
return recordSent(ctx, r.inv, s.node, body, s.declared.Builds, s.declared.Epoch)
|
return recordSent(ctx, r.inv, s.node, body, s.declared.Builds, s.declared.Epoch, sentRecordOf(ctx, s.declared))
|
||||||
}
|
}
|
||||||
|
|
||||||
// aResolver is a module built from a repository, at a commit, with something on the machine that
|
// aResolver is a module built from a repository, at a commit, with something on the machine that
|
||||||
|
|||||||
@@ -0,0 +1,26 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
)
|
||||||
|
|
||||||
|
// hideTyping turns a terminal's echo off while a secret is typed at it, and gives back what restores it. On
|
||||||
|
// anything that is not a terminal (a pipe, a file) it does nothing.
|
||||||
|
func hideTyping(f *os.File) func() {
|
||||||
|
fd := int(f.Fd())
|
||||||
|
before, err := unix.IoctlGetTermios(fd, unix.TCGETS)
|
||||||
|
if err != nil {
|
||||||
|
return func() {}
|
||||||
|
}
|
||||||
|
hidden := *before
|
||||||
|
hidden.Lflag &^= unix.ECHO
|
||||||
|
if err := unix.IoctlSetTermios(fd, unix.TCSETS, &hidden); err != nil {
|
||||||
|
return func() {}
|
||||||
|
}
|
||||||
|
return func() {
|
||||||
|
_ = unix.IoctlSetTermios(fd, unix.TCSETS, before)
|
||||||
|
_, _ = os.Stderr.WriteString("\n")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,194 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// novox/hq issue 349: on 2026-10-09 the plan of mesh-catalog at a082615b (the merge of a security fix to the
|
|
||||||
// forge's module) was "superseded at tier 0 by" the plan at 8ff8197a, the merge before it, which the
|
|
||||||
// catch-up acted on late; what the later plan had not built was folded into a plan at the commit before the
|
|
||||||
// fix. Plans of one branch are ordered by when the forge made their merges, and a merge older than an open
|
|
||||||
// plan of its branch is planned at that plan's commit.
|
|
||||||
|
|
||||||
// TestTwoMergesActedOnInReverseOrderBuildTheNewerCommit replays it: the later merge acted on first, the
|
|
||||||
// earlier one second (the catch-up). One plan is left open, at the later commit, and it builds both.
|
|
||||||
func TestTwoMergesActedOnInReverseOrderBuildTheNewerCommit(t *testing.T) {
|
|
||||||
open := aCatalogueMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
asksWithPaths(t)
|
|
||||||
for _, m := range []string{"gitea", "notes"} {
|
|
||||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"},
|
|
||||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + m, Ref: "main",
|
|
||||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
at := time.Now().UTC().Add(-20 * time.Minute).Truncate(time.Second)
|
|
||||||
fix := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "a082615bfix",
|
|
||||||
MergedAt: at.Add(2 * time.Minute).Format(time.RFC3339Nano),
|
|
||||||
Paths: []string{"modules/gitea/module.json"}, ModuleDirs: []string{"modules/gitea"}, ModuleDirsSaid: true}
|
|
||||||
before := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "8ff8197abefore",
|
|
||||||
MergedAt: at.Format(time.RFC3339Nano),
|
|
||||||
Paths: []string{"modules/notes/module.json"}, ModuleDirs: []string{"modules/notes"}, ModuleDirsSaid: true}
|
|
||||||
for _, m := range []link.SourceMoved{fix, before} {
|
|
||||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
plans, err := open.inventory.OpenPlans(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(plans) != 1 {
|
|
||||||
var said []string
|
|
||||||
for _, p := range plans {
|
|
||||||
said = append(said, p.ID+" "+p.Commit+" "+p.Note)
|
|
||||||
}
|
|
||||||
t.Fatalf("open plans: %s", strings.Join(said, "; "))
|
|
||||||
}
|
|
||||||
p := plans[0]
|
|
||||||
if p.Commit != fix.Commit {
|
|
||||||
t.Fatalf("the open plan builds %s, not the newer commit %s", p.Commit, fix.Commit)
|
|
||||||
}
|
|
||||||
for _, m := range []string{"gitea", "notes"} {
|
|
||||||
if _, has := p.Modules[m]; !has {
|
|
||||||
t.Fatalf("the open plan at the newer commit does not build %s: %v", m, p.Modules)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A late older merge after the newer plan is done (review of PR 179, A1): what it moved is built from the
|
|
||||||
// newer commit, and what the newer merge already looked at is not built again at the older one.
|
|
||||||
func TestALateMergeAfterTheNewerPlanEndedBuildsTheNewerCommit(t *testing.T) {
|
|
||||||
open := aCatalogueMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
asksWithPaths(t)
|
|
||||||
for _, m := range []string{"gitea", "notes"} {
|
|
||||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"},
|
|
||||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + m, Ref: "main",
|
|
||||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
at := time.Now().UTC().Add(-20 * time.Minute).Truncate(time.Second)
|
|
||||||
fix := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "a082615bfix",
|
|
||||||
MergedAt: at.Add(2*time.Minute + 500*time.Millisecond).Format(time.RFC3339Nano),
|
|
||||||
Paths: []string{"modules/gitea/module.json"}, ModuleDirs: []string{"modules/gitea"}, ModuleDirsSaid: true}
|
|
||||||
if err := (following{open: open}).SourceMoved(ctx, fix); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
plans, err := open.inventory.OpenPlans(ctx)
|
|
||||||
if err != nil || len(plans) != 1 {
|
|
||||||
t.Fatalf("%v %v", plans, err)
|
|
||||||
}
|
|
||||||
done := plans[0]
|
|
||||||
done.State = inventory.PlanDone
|
|
||||||
if err := open.inventory.SavePlan(ctx, &done); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got, err := open.inventory.PlanByID(ctx, done.ID); err != nil || !got.Merged.Equal(at.Add(2*time.Minute+500*time.Millisecond)) {
|
|
||||||
t.Fatalf("the merge time kept is %s, not to the nanosecond (%v)", got.Merged, err)
|
|
||||||
}
|
|
||||||
before := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "8ff8197abefore",
|
|
||||||
MergedAt: at.Format(time.RFC3339Nano),
|
|
||||||
Paths: []string{"modules/notes/module.json", "modules/gitea/module.json"},
|
|
||||||
ModuleDirs: []string{"modules/notes", "modules/gitea"}, ModuleDirsSaid: true}
|
|
||||||
if err := (following{open: open}).SourceMoved(ctx, before); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
plans, err = open.inventory.OpenPlans(ctx)
|
|
||||||
if err != nil || len(plans) != 1 {
|
|
||||||
t.Fatalf("open plans after the late merge: %+v %v", plans, err)
|
|
||||||
}
|
|
||||||
p := plans[0]
|
|
||||||
if p.Commit != fix.Commit {
|
|
||||||
t.Fatalf("the late merge was planned at %s, not the newer commit %s", p.Commit, fix.Commit)
|
|
||||||
}
|
|
||||||
if _, has := p.Modules["notes"]; !has {
|
|
||||||
t.Fatalf("what only the late merge moved is not built: %v", p.Modules)
|
|
||||||
}
|
|
||||||
if _, has := p.Modules["gitea"]; has {
|
|
||||||
t.Fatalf("what the newer merge already built is built again: %v", p.Modules)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Pure: the branch's order is the merges', where both plans know it; and a later merge of the branch is
|
|
||||||
// found in any state, never a release's, another repository's or another branch's.
|
|
||||||
func TestTheBranchOrderIsTheMerges(t *testing.T) {
|
|
||||||
t0 := time.Date(2026, 10, 9, 10, 0, 0, 0, time.UTC)
|
|
||||||
newerMerge := inventory.Plan{ID: "plan-1", Repository: "novox/mesh-catalog", Branch: "main", Commit: "a082615b",
|
|
||||||
Merged: t0.Add(time.Minute), Created: t0.Add(2 * time.Minute), State: inventory.PlanRolling}
|
|
||||||
olderMerge := inventory.Plan{ID: "plan-2", Repository: "novox/mesh-catalog", Branch: "main", Commit: "8ff8197a",
|
|
||||||
Merged: t0, Created: t0.Add(10 * time.Minute), State: inventory.PlanBuilding}
|
|
||||||
if earlierOnTheBranch(newerMerge, olderMerge) || !earlierOnTheBranch(olderMerge, newerMerge) {
|
|
||||||
t.Fatal("ordered by when the plans were made, not by when the merges were")
|
|
||||||
}
|
|
||||||
if _, closed := supersededBy(olderMerge, []inventory.Plan{newerMerge}, func(string) bool { return true }); len(closed) != 0 {
|
|
||||||
t.Fatalf("the plan of an older merge superseded a newer one: %s", closed[0].Note)
|
|
||||||
}
|
|
||||||
unknown := newerMerge
|
|
||||||
unknown.Merged = time.Time{}
|
|
||||||
if !earlierOnTheBranch(unknown, olderMerge) {
|
|
||||||
t.Fatal("without a merge time the plans' own order does not stand")
|
|
||||||
}
|
|
||||||
same := olderMerge
|
|
||||||
same.Merged = newerMerge.Merged
|
|
||||||
if !earlierOnTheBranch(newerMerge, same) || earlierOnTheBranch(same, newerMerge) {
|
|
||||||
t.Fatal("one merge time: the plans' own order does not stand")
|
|
||||||
}
|
|
||||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "8ff8197a",
|
|
||||||
MergedAt: t0.Add(500 * time.Millisecond).Format(time.RFC3339Nano)}
|
|
||||||
newerMerge.State = inventory.PlanDone
|
|
||||||
if !laterOnTheBranch(m, newerMerge) {
|
|
||||||
t.Fatal("a later merge of the branch, its plan done, was not found")
|
|
||||||
}
|
|
||||||
for name, change := range map[string]func(p *inventory.Plan, m *link.SourceMoved){
|
|
||||||
"another branch": func(_ *inventory.Plan, m *link.SourceMoved) { m.Base = "release" },
|
|
||||||
"another repository": func(p *inventory.Plan, _ *link.SourceMoved) { p.Repository = "novox/mesh-controller" },
|
|
||||||
"a release": func(p *inventory.Plan, _ *link.SourceMoved) { p.Release = &inventory.PlanRelease{} },
|
|
||||||
"no merge time": func(p *inventory.Plan, _ *link.SourceMoved) { p.Merged = time.Time{} },
|
|
||||||
"the same commit": func(p *inventory.Plan, m *link.SourceMoved) { m.Commit = p.Commit },
|
|
||||||
"an older merge": func(p *inventory.Plan, _ *link.SourceMoved) { p.Merged = t0 },
|
|
||||||
"the same moment": func(p *inventory.Plan, _ *link.SourceMoved) { p.Merged = t0.Add(500 * time.Millisecond) },
|
|
||||||
"an unreadable time": func(_ *inventory.Plan, m *link.SourceMoved) { m.MergedAt = "yesterday" },
|
|
||||||
} {
|
|
||||||
p, mm := newerMerge, m
|
|
||||||
change(&p, &mm)
|
|
||||||
if laterOnTheBranch(mm, p) {
|
|
||||||
t.Errorf("%s was taken as a later merge of the branch", name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// To the nanosecond: two merges within a second keep their order.
|
|
||||||
m.MergedAt = t0.Add(time.Minute + 200*time.Millisecond).Format(time.RFC3339Nano)
|
|
||||||
if !laterOnTheBranch(m, inventory.Plan{Repository: "novox/mesh-catalog", Branch: "main", Commit: "x",
|
|
||||||
Merged: t0.Add(time.Minute + 700*time.Millisecond)}) {
|
|
||||||
t.Fatal("merges within one second lost their order")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A merge time with a fraction of a second is kept whole in its plan, and two merges within one second keep
|
|
||||||
// their order through the plans and the lookup (review of PR 179).
|
|
||||||
func TestAMergeTimeKeepsItsFractionOfASecond(t *testing.T) {
|
|
||||||
at := "2026-10-09T10:57:52.123456789Z"
|
|
||||||
p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1", MergedAt: at}, nil, nil)
|
|
||||||
want := time.Date(2026, 10, 9, 10, 57, 52, 123456789, time.UTC)
|
|
||||||
if !p.Merged.Equal(want) {
|
|
||||||
t.Fatalf("the plan's merge time is %s, not %s", p.Merged, want)
|
|
||||||
}
|
|
||||||
earlier := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c0",
|
|
||||||
MergedAt: "2026-10-09T10:57:52.123456788Z"}, nil, nil)
|
|
||||||
earlier.Created = p.Created.Add(time.Second) // made after, merged before
|
|
||||||
if !earlierOnTheBranch(earlier, p) || earlierOnTheBranch(p, earlier) {
|
|
||||||
t.Fatal("two merges a nanosecond apart lost their order")
|
|
||||||
}
|
|
||||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c0", MergedAt: "2026-10-09T10:57:52.123456788Z"}
|
|
||||||
if !laterOnTheBranch(m, p) {
|
|
||||||
t.Fatal("a merge a nanosecond later was not found as the later one")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,356 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/lease"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq issue 352: on 2026-10-09 a release's gate on the control node read the machine's report against a
|
||||||
|
// newer send another plan had just made there — not against its own send — and failed three builds the
|
||||||
|
// machine had reported healthy ("has not reported on what it was sent"), put them back on every machine,
|
||||||
|
// to a controller older than the store's schema, and that controller then judged the newer plan's
|
||||||
|
// controller passed from the put-back build's health.
|
||||||
|
|
||||||
|
// TestReplay352 replays the walk on the backlog fixture: the release sends anchor and anchor reports;
|
||||||
|
// another send reaches anchor, unreported; the gate still passes. And a send that moves a judged module
|
||||||
|
// to another build supersedes the judging: no verdict, nothing put back.
|
||||||
|
func TestReplay352(t *testing.T) {
|
||||||
|
t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn)
|
||||||
|
t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging)
|
||||||
|
}
|
||||||
|
|
||||||
|
func testANewerSendDoesNotUnreportTheGatesOwn(t *testing.T) {
|
||||||
|
b := aBacklog(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := b.open.inventory
|
||||||
|
advancePlans(ctx, b.open) // anchor is sent, and the fixture reports it applied
|
||||||
|
// 16:31:37 — another plan sends anchor a newer declaration, which it has not reported on.
|
||||||
|
carried, _, _ := inv.SentBuilds(ctx, "anchor")
|
||||||
|
if err := inv.RecordSent(ctx, nodeID(t, b.open, "anchor"), "d-anchor-newer", carried); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
reports, _ := inv.LastReports(ctx)
|
||||||
|
for _, r := range reports {
|
||||||
|
if r.Node == "anchor" && r.Current {
|
||||||
|
t.Fatal("the fixture's newer send reads as reported")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
gateEvery, gateBound = 0, 0 // past the bound at once: before the fix, "has not reported" fails it here
|
||||||
|
for i := 0; i < 4; i++ {
|
||||||
|
advancePlans(ctx, b.open)
|
||||||
|
}
|
||||||
|
p := b.release(t)
|
||||||
|
if p.State == inventory.PlanFailed || strings.Contains(p.Note, "has not reported") {
|
||||||
|
t.Fatalf("the release failed on the newer send: %s %s", p.State, p.Note)
|
||||||
|
}
|
||||||
|
if g := p.Release.Gate; g != nil && (g.Sent == nil || g.Sent["anchor"].Digest == "") {
|
||||||
|
t.Fatalf("the gate does not keep what it sent: %+v", g)
|
||||||
|
}
|
||||||
|
if v, found, err := inv.GateOf(ctx, "build-app-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||||
|
t.Fatalf("app's pass on anchor was not kept: %+v %v %v", v, found, err)
|
||||||
|
}
|
||||||
|
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" {
|
||||||
|
t.Fatalf("app was put back to %s", current["app"].Commit)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A plan's own first send waits while a release judges the same module on that machine with another build.
|
||||||
|
func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
|
||||||
|
b := aBacklog(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
advancePlans(ctx, b.open) // the release judges app c2 on anchor
|
||||||
|
_, _, err := gatedSend(ctx, b.open, "anchor", []inventory.CarriedMove{{Module: "app", Node: "anchor", From: "c2", To: "c3", Build: "build-app-c3"}})
|
||||||
|
if !errors.Is(err, errWalkedElsewhere) || !strings.Contains(err.Error(), "release-") {
|
||||||
|
t.Fatalf("a newer build of a judged module was sent under the release's gate: %v", err)
|
||||||
|
}
|
||||||
|
if len(b.sent) != 1 {
|
||||||
|
t.Fatalf("sent %v", b.sent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A merge plan's judging is superseded the same way: another send moved its module on the first machine.
|
||||||
|
func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) {
|
||||||
|
g := aGateMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := g.open.inventory
|
||||||
|
advancePlans(ctx, g.open) // anchor is sent app c2 first
|
||||||
|
if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-c3", map[string]string{"app": "c3"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
gateEvery = 0
|
||||||
|
advancePlans(ctx, g.open)
|
||||||
|
p := g.plan(t)
|
||||||
|
if p.State != inventory.PlanSuperseded || !strings.Contains(p.Note, "superseded") || !strings.Contains(p.Note, "c3") {
|
||||||
|
t.Fatalf("the plan is %s: %s", p.State, p.Note)
|
||||||
|
}
|
||||||
|
// Nothing put back: the registered build stands, the build is not marked, and the plan's gate made no
|
||||||
|
// rollback (a release may walk what the other send left waiting on anchor; that is not a put-back).
|
||||||
|
if r := p.Modules["app"].Gate.Rollback; r != "" {
|
||||||
|
t.Fatalf("a superseded judging made a rollback: %q", r)
|
||||||
|
}
|
||||||
|
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" {
|
||||||
|
t.Fatalf("app was put back to %s", current["app"].Commit)
|
||||||
|
}
|
||||||
|
if failed, _ := inv.GateFailed(ctx, "build-2"); failed {
|
||||||
|
t.Fatal("a superseded build was marked failed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testASendThatMovesTheModuleSupersedesTheJudging(t *testing.T) {
|
||||||
|
b := aBacklog(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := b.open.inventory
|
||||||
|
advancePlans(ctx, b.open)
|
||||||
|
// Another send moves app on anchor to a build this gate does not judge.
|
||||||
|
carried, _, _ := inv.SentBuilds(ctx, "anchor")
|
||||||
|
carried["app"] = "c3"
|
||||||
|
if err := inv.RecordSent(ctx, nodeID(t, b.open, "anchor"), "d-anchor-c3", carried); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
gateEvery = 0
|
||||||
|
advancePlans(ctx, b.open)
|
||||||
|
p := b.release(t)
|
||||||
|
if p.State != inventory.PlanSuperseded || !strings.Contains(p.Note, "superseded") || !strings.Contains(p.Note, "c3") {
|
||||||
|
t.Fatalf("the release is %s: %s", p.State, p.Note)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}}) {
|
||||||
|
t.Fatalf("sent %v: a superseded judging puts nothing back", b.sent)
|
||||||
|
}
|
||||||
|
if _, found, _ := inv.GateOf(ctx, "build-app-c2"); found {
|
||||||
|
t.Fatal("a superseded judging kept a verdict")
|
||||||
|
}
|
||||||
|
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" {
|
||||||
|
t.Fatalf("app was put back to %s", current["app"].Commit)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A report is on the gate's own send: the declaration itself, or one sequenced after it; a gate kept
|
||||||
|
// without its send reads the report against the send made last, as before. Pure.
|
||||||
|
func TestAReportIsHeldAgainstTheGatesOwnSend(t *testing.T) {
|
||||||
|
sent := inventory.SentDeclaration{Digest: "d-490", Sequence: 490}
|
||||||
|
for _, c := range []struct {
|
||||||
|
r inventory.Reported
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{inventory.Reported{Declared: "d-490", Current: false}, true},
|
||||||
|
{inventory.Reported{Declared: "d-491", ReportedSequence: 491, Current: true}, true},
|
||||||
|
{inventory.Reported{Declared: "d-489", ReportedSequence: 489, Current: false}, false},
|
||||||
|
{inventory.Reported{Declared: "other", ReportedSequence: 490}, true}, // the same sequence, said by another digest
|
||||||
|
{inventory.Reported{Declared: "d-495", ReportedSequence: 495, Current: true}, true}, // the last send: this one or a later one
|
||||||
|
{inventory.Reported{Declared: "", Current: false}, false},
|
||||||
|
} {
|
||||||
|
if got := sent.ReportsOn(c.r); got != c.want {
|
||||||
|
t.Errorf("%+v on %+v: %v", c.r, sent, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
byDigest := inventory.SentDeclaration{Digest: "d-1"}
|
||||||
|
if !byDigest.ReportsOn(inventory.Reported{Declared: "d-1"}) || byDigest.ReportsOn(inventory.Reported{ReportedSequence: 5}) ||
|
||||||
|
!byDigest.ReportsOn(inventory.Reported{Current: true}) {
|
||||||
|
t.Fatal("a send kept without a sequence is matched by its digest and by the last send alone")
|
||||||
|
}
|
||||||
|
f := gateFacts{sent: map[string]inventory.SentDeclaration{"anchor": sent}}
|
||||||
|
if !f.reportedOn("anchor", inventory.Reported{Declared: "d-490"}) || f.reportedOn("anchor", inventory.Reported{Declared: "d-1"}) {
|
||||||
|
t.Fatal("a gate that kept its send read the report against something other than it")
|
||||||
|
}
|
||||||
|
if !f.reportedOn("laptop", inventory.Reported{Current: true}) || f.reportedOn("laptop", inventory.Reported{Current: false}) {
|
||||||
|
t.Fatal("a gate that did not keep its send does not read the report against the send made last")
|
||||||
|
}
|
||||||
|
// Through the merge plan's first-machine wait too.
|
||||||
|
at := time.Now()
|
||||||
|
state := inventory.PlanModule{First: []string{"anchor"}, FirstAt: &at,
|
||||||
|
Gate: &inventory.PlanGate{Machines: []string{"anchor"}, Sent: map[string]inventory.SentDeclaration{"anchor": sent}}}
|
||||||
|
reports := []inventory.Reported{{Node: "anchor", At: &at, Outcome: inventory.OutcomeApplied, Current: false, Declared: "d-490"}}
|
||||||
|
if step := nextRollout(state, []string{"anchor", "laptop"}, false, reports, at.Add(time.Minute), time.Hour); step.waiting != "" || step.failed != "" {
|
||||||
|
t.Fatalf("the first machine's report on the plan's own send read as none: %+v", step)
|
||||||
|
}
|
||||||
|
reports[0].Declared = "d-480"
|
||||||
|
if step := nextRollout(state, []string{"anchor", "laptop"}, false, reports, at.Add(time.Minute), time.Hour); step.waiting == "" {
|
||||||
|
t.Fatalf("a report on an older send read as the plan's: %+v", step)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A gate judges only the build the machine was last sent: last sent another build of the module, the
|
||||||
|
// judging is superseded, whatever the machine reports. Pure.
|
||||||
|
func TestAGateJudgesOnlyTheBuildTheMachineWasLastSent(t *testing.T) {
|
||||||
|
at := time.Now()
|
||||||
|
f := gateFacts{now: at, reports: map[string]inventory.Reported{"anchor": {Node: "anchor", Outcome: inventory.OutcomeApplied,
|
||||||
|
At: &at, Current: true}}, engines: map[string]string{}, served: map[string]served{}, rolledBack: map[string][]lease.Rollback{},
|
||||||
|
commits: map[string]string{"mesh-controller": "e6b00e2e"}, sentBuilds: map[string]map[string]string{"anchor": {"mesh-controller": "ef26d4cb"}}}
|
||||||
|
taken := at.Add(-30 * time.Second)
|
||||||
|
f.holder = &lease.Holder{Taken: taken, Health: &lease.Health{Ready: true}}
|
||||||
|
h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "anchor", at.Add(-time.Minute), f)
|
||||||
|
if h != healthSuperseded || !strings.Contains(why, "ef26d4cb") || !strings.Contains(why, "e6b00e2e") {
|
||||||
|
t.Fatalf("a controller build the machine no longer runs: %v %q", h, why)
|
||||||
|
}
|
||||||
|
f.sentBuilds["anchor"]["mesh-controller"] = "e6b00e2e"
|
||||||
|
if h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "anchor", at.Add(-time.Minute), f); h == healthSuperseded {
|
||||||
|
t.Fatalf("the build sent read as another: %q", why)
|
||||||
|
}
|
||||||
|
delete(f.sentBuilds, "anchor")
|
||||||
|
if h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "anchor", at.Add(-time.Minute), f); h == healthSuperseded {
|
||||||
|
t.Fatalf("a machine whose send is not known read as superseded: %q", why)
|
||||||
|
}
|
||||||
|
g := &inventory.PlanGate{To: "c2", Carried: []inventory.CarriedMove{{Module: "late", Node: "anchor", To: "c5"}}}
|
||||||
|
if got := judgedCommits(g, []judged{{"app", "anchor"}, {"late", "anchor"}}); got["app"] != "c2" || got["late"] != "c5" {
|
||||||
|
t.Fatalf("judged commits %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A move of another build of a module to a machine where a release or a plan is judging that module
|
||||||
|
// waits for that judging; the same build to that machine is already there. Pure.
|
||||||
|
func TestAWalkWaitsForAJudgingOfTheSameModuleOnThatMachine(t *testing.T) {
|
||||||
|
at := time.Now()
|
||||||
|
release := inventory.Plan{ID: "release-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{
|
||||||
|
Gate: &inventory.PlanGate{Machines: []string{"novox"}, Carried: []inventory.CarriedMove{
|
||||||
|
{Module: "mesh-controller", Node: "novox", From: "ef26d4cb", To: "2913c54c"}}}}}
|
||||||
|
merge := inventory.Plan{ID: "plan-1", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
|
||||||
|
"app": {First: []string{"anchor"}, FirstAt: &at, Commit: "c2", Gate: &inventory.PlanGate{Machines: []string{"anchor"}}}}}
|
||||||
|
f := moveFacts{plans: []inventory.Plan{release, merge}}
|
||||||
|
for _, c := range []struct {
|
||||||
|
module, node, to, want string
|
||||||
|
}{
|
||||||
|
{"mesh-controller", "novox", "e6b00e2e", "release-1"}, // the day's case: a newer controller to the judged machine
|
||||||
|
{"mesh-controller", "novox", "2913c54c", ""}, // the same build: already there
|
||||||
|
{"mesh-controller", "ace", "2913c54c", "release-1"}, // another machine while the first is judged
|
||||||
|
{"mesh-host", "novox", "x", ""}, // a module the release does not carry
|
||||||
|
{"app", "anchor", "c2", ""},
|
||||||
|
{"app", "anchor", "c3", "plan-1"},
|
||||||
|
{"app", "laptop", "c2", "plan-1"},
|
||||||
|
} {
|
||||||
|
if got := f.walkedBy(c.module, c.node, c.to); got != c.want {
|
||||||
|
t.Errorf("%s %s to %s: walked by %q, want %q", c.module, c.to, c.node, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
release.Release.Gate.Verdict = inventory.GatePassed
|
||||||
|
merge.Modules["app"].Gate.Verdict = inventory.GatePassed
|
||||||
|
if f.walkedBy("mesh-controller", "novox", "e6b00e2e") != "" || f.walkedBy("app", "laptop", "c3") != "" {
|
||||||
|
t.Fatal("a passed judging still holds a move")
|
||||||
|
}
|
||||||
|
release.Release.Gate.Verdict = ""
|
||||||
|
f.plans[0].State = inventory.PlanSuperseded
|
||||||
|
if f.walkedBy("mesh-controller", "novox", "e6b00e2e") != "" {
|
||||||
|
t.Fatal("a closed release still holds a move")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The controller is never put back to a build that reaches less of the store's schema than the store
|
||||||
|
// has, or to one that never said what it reaches: the current build is kept, and the condition says so.
|
||||||
|
func TestTheControllerIsNotPutBackToABuildOlderThanTheStoresSchema(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
keeper, _ := withConditionsInMemory(t)
|
||||||
|
told := &conditions.Told{}
|
||||||
|
was := doctorFrom
|
||||||
|
doctorFrom = &doctor{open: open, keeper: keeper, teller: told}
|
||||||
|
t.Cleanup(func() { doctorFrom = was })
|
||||||
|
wasSend := sendRollout
|
||||||
|
var sent [][]string
|
||||||
|
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||||
|
sent = append(sent, names)
|
||||||
|
return names, nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { sendRollout = wasSend })
|
||||||
|
|
||||||
|
build := func(id, commit, digest string, asked time.Time) inventory.Build {
|
||||||
|
manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: commit})
|
||||||
|
b := inventory.Build{ID: id, Module: "mesh-controller", Commit: commit, Repository: "novox/mesh-controller", Path: ".",
|
||||||
|
Manifest: manifest, Asked: asked, At: asked, Made: []inventory.Artifact{{Name: "controller", Kind: catalogue.ArtifactBundle,
|
||||||
|
Reference: "mesh-artifact://mesh-controller/controller/blobs/sha256:" + digest}}}
|
||||||
|
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: commit},
|
||||||
|
inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Path: ".", BuiltFrom: commit, Head: commit, Asked: asked}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
previous := build("build-old", "ef26d4cb", strings.Repeat("1", 64), time.Now().Add(-2*time.Hour))
|
||||||
|
failed := build("build-new", "e6b00e2e", strings.Repeat("2", 64), time.Now().Add(-time.Minute))
|
||||||
|
if versionOfBuild(previous) != strings.Repeat("1", 12) {
|
||||||
|
t.Fatalf("the build's version is %q", versionOfBuild(previous))
|
||||||
|
}
|
||||||
|
applied, err := inv.SchemaApplied(ctx)
|
||||||
|
if err != nil || applied < 87 {
|
||||||
|
t.Fatalf("the store's schema reaches %d (%v)", applied, err)
|
||||||
|
}
|
||||||
|
// The build before never recorded what it reads: not proved, refused.
|
||||||
|
if why, ok := controllerSchemaAllows(ctx, inv, previous); ok || !strings.Contains(why, "never recorded") {
|
||||||
|
t.Fatalf("an unknown reach: %v %q", ok, why)
|
||||||
|
}
|
||||||
|
// It reads less than the store has: refused, naming both.
|
||||||
|
if err := inv.RecordSchemaReach(ctx, versionOfBuild(previous), applied-1); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if why, ok := controllerSchemaAllows(ctx, inv, previous); ok || !strings.Contains(why, "is at") {
|
||||||
|
t.Fatalf("a reach behind the store: %v %q", ok, why)
|
||||||
|
}
|
||||||
|
// Through the gate: the failed build is marked, nothing is put back, nothing is sent, the condition is urgent.
|
||||||
|
at := time.Now().Add(-5 * time.Minute)
|
||||||
|
state := &inventory.PlanModule{Build: failed.ID, Commit: failed.Commit, Previous: previous.Commit, First: []string{"anchor"}, FirstAt: &at}
|
||||||
|
p := inventory.Plan{ID: "plan-352", Repository: "novox/mesh-controller", Branch: "main", Commit: failed.Commit, Created: at,
|
||||||
|
State: inventory.PlanRolling, Tiers: [][]string{{"mesh-controller"}}, Modules: map[string]*inventory.PlanModule{"mesh-controller": state}}
|
||||||
|
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
gateFailed(ctx, open, &p, "mesh-controller", state, []string{"anchor"}, "not healthy within 10m0s of its apply")
|
||||||
|
if state.Gate.Rollback != inventory.NotRolledBack || !strings.Contains(p.Note, "NOT put back") || !strings.Contains(p.Note, "the current build is kept") {
|
||||||
|
t.Fatalf("rollback %q: %s", state.Gate.Rollback, p.Note)
|
||||||
|
}
|
||||||
|
if len(sent) != 0 {
|
||||||
|
t.Fatalf("sent %v: nothing is put back", sent)
|
||||||
|
}
|
||||||
|
if current, _ := inv.CurrentBuilds(ctx); current["mesh-controller"].Commit != failed.Commit {
|
||||||
|
t.Fatalf("the module was put back to %s", current["mesh-controller"].Commit)
|
||||||
|
}
|
||||||
|
if marked, _ := inv.GateFailed(ctx, failed.ID); !marked {
|
||||||
|
t.Fatal("the failed build is not marked failed at its gate")
|
||||||
|
}
|
||||||
|
open2, _ := keeper.Open(ctx)
|
||||||
|
var found bool
|
||||||
|
for _, c := range open2 {
|
||||||
|
if c.Kind == kindRollbackFailed && c.Severity == conditions.Urgent && strings.Contains(c.Summary, "current build is kept") {
|
||||||
|
found = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatalf("no urgent rollback-failed condition saying the current build is kept: %+v", open2)
|
||||||
|
}
|
||||||
|
// Reaching the store: allowed.
|
||||||
|
if err := inv.RecordSchemaReach(ctx, versionOfBuild(previous), applied); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if why, ok := controllerSchemaAllows(ctx, inv, previous); !ok {
|
||||||
|
t.Fatalf("a build that reads the whole schema was refused: %q", why)
|
||||||
|
}
|
||||||
|
// A build with no bundle to know it by: refused.
|
||||||
|
if why, ok := controllerSchemaAllows(ctx, inv, inventory.Build{Commit: "x"}); ok || !strings.Contains(why, "names no bundle") {
|
||||||
|
t.Fatalf("a build without a bundle: %v %q", ok, why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The lease's holder names the build the declaration told it it is, and the version stamp only without one.
|
||||||
|
func TestTheHolderNamesTheBuildTheDeclarationToldIt(t *testing.T) {
|
||||||
|
t.Setenv(RunningBuildVar, " ad62528c47c7 ")
|
||||||
|
if h := holderOf("x"); h.Build != "ad62528c47c7" {
|
||||||
|
t.Fatalf("the holder's build is %q", h.Build)
|
||||||
|
}
|
||||||
|
t.Setenv(RunningBuildVar, "")
|
||||||
|
if h := holderOf("x"); h.Build != version {
|
||||||
|
t.Fatalf("without a declared version the holder's build is %q", h.Build)
|
||||||
|
}
|
||||||
|
if reach, err := schemaReach(); err != nil || reach < 87 {
|
||||||
|
t.Fatalf("this build's reach is %d (%v)", reach, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -77,7 +77,7 @@ func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) {
|
|||||||
}
|
}
|
||||||
cancel() // the sender is going away: its context is cancelled between the send and the record
|
cancel() // the sender is going away: its context is cancelled between the send and the record
|
||||||
body := []byte(`{"declaration":1,"resources":[]}`)
|
body := []byte(`{"declaration":1,"resources":[]}`)
|
||||||
digest, err := recordSent(ctx, inv, "anchor", body, nil, 0)
|
digest, err := recordSent(ctx, inv, "anchor", body, nil, 0, sentRecord{sender: "a test"})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// NodeByName on the cancelled context may itself refuse; the record must still be possible
|
// NodeByName on the cancelled context may itself refuse; the record must still be possible
|
||||||
// through the detached context, so look the node up again on a live one.
|
// through the detached context, so look the node up again on a live one.
|
||||||
|
|||||||
@@ -311,7 +311,7 @@ func usage() {
|
|||||||
the self-check: the last verdict, a run now, the probes, the signals' ages
|
the self-check: the last verdict, a run now, the probes, the signals' ages
|
||||||
healers [--days N] [--json] the healers, what they did lately, and their brake (to-be 45 §7)
|
healers [--days N] [--json] the healers, what they did lately, and their brake (to-be 45 §7)
|
||||||
durations [--kind K] [--days N] [--json]
|
durations [--kind K] [--days N] [--json]
|
||||||
apply, heartbeat, plan-tier and build durations, per machine or module
|
apply, heartbeat, plan-tier, build and walk-phase durations
|
||||||
collection [--json] kept archives held/unheld by a manifest, and what the sweep may let go
|
collection [--json] kept archives held/unheld by a manifest, and what the sweep may let go
|
||||||
builder issue <name> a broker account for a build machine, scoped to build work,
|
builder issue <name> a broker account for a build machine, scoped to build work,
|
||||||
delivered as the builder module's broker secret (module add it first)
|
delivered as the builder module's broker secret (module add it first)
|
||||||
|
|||||||
@@ -1204,7 +1204,18 @@ func graphOfFacts(f snapshot.Facts) ([]inventory.Entry, map[string][]inventory.R
|
|||||||
entries = append(entries, inventory.Entry{Manifest: manifest, Provided: mod.Provided,
|
entries = append(entries, inventory.Entry{Manifest: manifest, Provided: mod.Provided,
|
||||||
Source: inventory.Source{Repository: mod.Repository, Path: mod.Path, BuiltFrom: mod.Commit}})
|
Source: inventory.Source{Repository: mod.Repository, Path: mod.Path, BuiltFrom: mod.Commit}})
|
||||||
for _, r := range mod.Reads {
|
for _, r := range mod.Reads {
|
||||||
read[mod.Name] = append(read[mod.Name], inventory.ReadRepository{Repository: r})
|
entry := inventory.ReadRepository{Repository: r}
|
||||||
|
for _, s := range mod.Sources {
|
||||||
|
if !s.Own && s.Repository == r && len(s.Paths) > 0 {
|
||||||
|
entry.Paths = s.Paths
|
||||||
|
}
|
||||||
|
}
|
||||||
|
read[mod.Name] = append(read[mod.Name], entry)
|
||||||
|
}
|
||||||
|
for _, s := range mod.Sources {
|
||||||
|
if s.Own && len(s.Paths) > 0 {
|
||||||
|
read[mod.Name] = append(read[mod.Name], inventory.ReadRepository{Own: true, Paths: s.Paths})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
var edges []inventory.Edge
|
var edges []inventory.Edge
|
||||||
|
|||||||
@@ -392,7 +392,8 @@ func TestACheckByHandClonesWhatTheSeatClones(t *testing.T) {
|
|||||||
for dir, refs := range map[string]map[string]string{
|
for dir, refs := range map[string]map[string]string{
|
||||||
"mesh-catalog": {"mesh-catalog": "main"},
|
"mesh-catalog": {"mesh-catalog": "main"},
|
||||||
"mesh-host": {"mesh-host": "c0ffee"},
|
"mesh-host": {"mesh-host": "c0ffee"},
|
||||||
"mesh-controller": {"mesh-controller": "c0ffee", "mesh-controller-main": "main", "mesh-lab": "main"},
|
"mesh-controller": {"mesh-controller": "c0ffee", "mesh-controller-main": "main", "mesh-lab": "main",
|
||||||
|
"mesh-sdk": sdkPinnedByGoMod(t)},
|
||||||
} {
|
} {
|
||||||
got := besideRefs(dir, "c0ffee")
|
got := besideRefs(dir, "c0ffee")
|
||||||
for d, ref := range refs {
|
for d, ref := range refs {
|
||||||
@@ -412,3 +413,24 @@ func TestACheckByHandClonesWhatTheSeatClones(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// sdkPinnedByGoMod is the SDK commit this tree's go.mod pins, read from the file rather than the build, so
|
||||||
|
// sdkPinned is held to what the repository says (novox/hq issue 449).
|
||||||
|
func sdkPinnedByGoMod(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := os.ReadFile(filepath.Join("..", "..", "go.mod"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, line := range strings.Split(string(raw), "\n") {
|
||||||
|
fields := strings.Fields(line)
|
||||||
|
if len(fields) >= 2 && fields[0] == sdkModule {
|
||||||
|
if m := pseudoCommit.FindStringSubmatch(fields[1]); m != nil {
|
||||||
|
return m[1]
|
||||||
|
}
|
||||||
|
return "go/" + fields[1]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("go.mod requires no %s", sdkModule)
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|||||||
@@ -141,6 +141,12 @@ func runForMeshCLI(ctx context.Context, node string, asked link.CLIAsked, v cliV
|
|||||||
if v.refused != "" {
|
if v.refused != "" {
|
||||||
return link.CLIRefusal(v.refused)
|
return link.CLIRefusal(v.refused)
|
||||||
}
|
}
|
||||||
|
// Standard input is the terminal's alone: a secret given at the terminal reaches `secret accept`, and no ordinary
|
||||||
|
// call is handed what the asker's standard input held (novox/hq ADR 0259 §10, ADR 0272).
|
||||||
|
if len(asked.Stdin) > 0 && !v.terminal {
|
||||||
|
return link.CLIAnswer{Exit: 1, Why: v.why, Refused: "standard input is given to a line that runs as the " +
|
||||||
|
"controller's terminal alone, and this one does not. Nothing ran"}
|
||||||
|
}
|
||||||
if cliServers[asked.Line[0]] {
|
if cliServers[asked.Line[0]] {
|
||||||
return link.CLIAnswer{Exit: 1, Why: v.why, Refused: fmt.Sprintf("%s serves until stopped, and is not a "+
|
return link.CLIAnswer{Exit: 1, Why: v.why, Refused: fmt.Sprintf("%s serves until stopped, and is not a "+
|
||||||
"command line mesh-cli runs. Nothing ran", asked.Line[0])}
|
"command line mesh-cli runs. Nothing ran", asked.Line[0])}
|
||||||
@@ -155,8 +161,12 @@ func runForMeshCLI(ctx context.Context, node string, asked link.CLIAsked, v cliV
|
|||||||
}
|
}
|
||||||
cmd := selfCommand(ctx, line)
|
cmd := selfCommand(ctx, line)
|
||||||
cmd.Env = commandEnvironment(fmt.Sprintf("%s through mesh-cli on %s", asked.Account, node), verb, v.terminal)
|
cmd.Env = commandEnvironment(fmt.Sprintf("%s through mesh-cli on %s", asked.Account, node), verb, v.terminal)
|
||||||
// No standard input: a command that reads one gets nothing, and fails saying so (ADR 0272 §5).
|
// No standard input unless mesh-cli carried one for a terminal line: a command that reads one gets nothing, and
|
||||||
|
// fails saying so (ADR 0272 §5).
|
||||||
cmd.Stdin = nil
|
cmd.Stdin = nil
|
||||||
|
if len(asked.Stdin) > 0 {
|
||||||
|
cmd.Stdin = bytes.NewReader(asked.Stdin)
|
||||||
|
}
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
cmd.Stdout, cmd.Stderr = &stdout, &stderr
|
cmd.Stdout, cmd.Stderr = &stdout, &stderr
|
||||||
err := cmd.Run()
|
err := cmd.Run()
|
||||||
|
|||||||
@@ -0,0 +1,99 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// secretAcceptWants makes the test binary, run as a command line, read a secret as `secret accept` reads it and
|
||||||
|
// say whether it is the value whose SHA-256 the variable names (TestMain).
|
||||||
|
const secretAcceptWants = "MESH_TEST_SECRET_ACCEPT_WANTS"
|
||||||
|
|
||||||
|
// readAsSecretAccept is that process: `secret accept <node> <module> <name> [--from -]`, the value read by
|
||||||
|
// valueFor, compared by digest, and only the verdict printed.
|
||||||
|
func readAsSecretAccept(want string, argv []string) int {
|
||||||
|
if len(argv) < 5 || argv[0] != "secret" || argv[1] != "accept" {
|
||||||
|
fmt.Printf("not a secret accept line: %q\n", argv)
|
||||||
|
return 2
|
||||||
|
}
|
||||||
|
from := ""
|
||||||
|
if len(argv) == 7 && argv[5] == "--from" {
|
||||||
|
from = argv[6]
|
||||||
|
}
|
||||||
|
value, err := valueFor(argv[2], argv[3], argv[4], from)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("secret accept read nothing: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
sum := sha256.Sum256([]byte(asSupplied(value)))
|
||||||
|
if hex.EncodeToString(sum[:]) != want {
|
||||||
|
fmt.Printf("secret accept read something else (%d bytes)\n", len(value))
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
fmt.Println("secret accept read the value it was given")
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0259 §10, ADR 0272: what mesh-cli's standard input held reaches `secret accept` on a line that runs
|
||||||
|
// as the controller's terminal, and appears nowhere else — not in the answer, not in the journal, not in the calls
|
||||||
|
// record; an ordinary line carrying it is refused and nothing runs.
|
||||||
|
func TestStandardInputReachesSecretAcceptAtTheTerminalAndNowhereElse(t *testing.T) {
|
||||||
|
token := "123456789:AAEhBOweik6ad9r_QxGivenAtTheTerminal"
|
||||||
|
sum := sha256.Sum256([]byte(token))
|
||||||
|
t.Setenv(secretAcceptWants, hex.EncodeToString(sum[:]))
|
||||||
|
var journal []string
|
||||||
|
var mu sync.Mutex
|
||||||
|
was := cliJournal
|
||||||
|
cliJournal = func(line string) { mu.Lock(); journal = append(journal, line); mu.Unlock() }
|
||||||
|
t.Cleanup(func() { cliJournal = was })
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
for _, line := range [][]string{
|
||||||
|
{"secret", "accept", "anchor", "telegram", "telegram-token", "--from", "-"},
|
||||||
|
{"secret", "accept", "anchor", "telegram", "telegram-token"}, // the prompt's path, a line on standard input
|
||||||
|
} {
|
||||||
|
asked := cliAsked("operator", 1000, line...)
|
||||||
|
asked.Stdin = []byte(token + "\n")
|
||||||
|
a := runForMeshCLI(ctx, "control", asked, cliVerdict{terminal: true, why: "the terminal"})
|
||||||
|
if a.Exit != 0 || !strings.Contains(string(a.Stdout), "read the value it was given") {
|
||||||
|
t.Fatalf("%q: secret accept did not read what mesh-cli carried: %+v (%s)", line, a, a.Stdout)
|
||||||
|
}
|
||||||
|
if body, _ := json.Marshal(a); strings.Contains(string(body), "AAEh") || strings.Contains(string(body), base64.StdEncoding.EncodeToString([]byte(token))) {
|
||||||
|
t.Fatalf("the answer carries the secret: %s", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Without standard input, the line reads nothing, as before.
|
||||||
|
a := runForMeshCLI(ctx, "control", cliAsked("operator", 1000, "secret", "accept", "anchor", "telegram",
|
||||||
|
"telegram-token", "--from", "-"), cliVerdict{terminal: true, why: "the terminal"})
|
||||||
|
if a.Exit == 0 {
|
||||||
|
t.Fatalf("a line with no standard input read a value: %+v", a)
|
||||||
|
}
|
||||||
|
|
||||||
|
// An ordinary call is never handed it: refused, and nothing ran.
|
||||||
|
asked := cliAsked("operator", 1000, "status")
|
||||||
|
asked.Stdin = []byte(token)
|
||||||
|
a = runForMeshCLI(ctx, "laptop", asked, cliVerdict{why: "not the terminal"})
|
||||||
|
if a.Exit == 0 || !strings.Contains(a.Refused, "terminal alone") || len(a.Stdout) != 0 {
|
||||||
|
t.Fatalf("an ordinary line was given standard input: %+v", a)
|
||||||
|
}
|
||||||
|
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
for _, l := range journal {
|
||||||
|
if strings.Contains(l, "AAEh") {
|
||||||
|
t.Fatalf("the journal says the secret: %s", l)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(journal) == 0 {
|
||||||
|
t.Fatal("the lines were not said in the journal at all")
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -40,6 +40,50 @@ type merges interface {
|
|||||||
AnnouncedMerges(ctx context.Context, since time.Time) ([]link.AnnouncedMerge, error)
|
AnnouncedMerges(ctx context.Context, since time.Time) ([]link.AnnouncedMerge, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// unheardMerges is the announcements of merges the controller has not heard (novox/hq ADR 0276): a merge kept
|
||||||
|
// in a batch is not acted on until its batch is cut, which can be past mergeGrace behind a long walk, and is
|
||||||
|
// not missed for that.
|
||||||
|
type unheardMerges struct {
|
||||||
|
merges
|
||||||
|
inv *inventory.Inventory
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u unheardMerges) AnnouncedMerges(ctx context.Context, since time.Time) ([]link.AnnouncedMerge, error) {
|
||||||
|
all, err := u.merges.AnnouncedMerges(ctx, since)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var out []link.AnnouncedMerge
|
||||||
|
for _, a := range all {
|
||||||
|
_, kept, err := u.inv.MergeOf(ctx, a.Owner+"/"+a.Repo, a.Commit)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if !kept {
|
||||||
|
out = append(out, a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// owedToTheRecord says a merge read as history is still owed to the record (novox/hq ADR 0276): the merges
|
||||||
|
// reader knows, when it keeps them (unheardMerges).
|
||||||
|
func owedToTheRecord(ctx context.Context, announced merges, m link.SourceMoved, entries []inventory.Entry,
|
||||||
|
read map[string][]inventory.ReadRepository) bool {
|
||||||
|
u, ok := announced.(unheardMerges)
|
||||||
|
if !ok {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
merged, err := time.Parse(time.RFC3339Nano, m.MergedAt)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
raw := m
|
||||||
|
raw.MergedAt = ""
|
||||||
|
owed, err := owedLate(ctx, u.inv, m, merged, wouldMove(raw, entries, read))
|
||||||
|
return err == nil && owed
|
||||||
|
}
|
||||||
|
|
||||||
// catchingUpOnMerges reads back the forge's announcements on a timer, until the context ends.
|
// catchingUpOnMerges reads back the forge's announcements on a timer, until the context ends.
|
||||||
func catchingUpOnMerges(ctx context.Context, open *stores, announced merges) {
|
func catchingUpOnMerges(ctx context.Context, open *stores, announced merges) {
|
||||||
f := following{open}
|
f := following{open}
|
||||||
@@ -48,7 +92,7 @@ func catchingUpOnMerges(ctx context.Context, open *stores, announced merges) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
read, err := open.inventory.ReadRepositories(ctx)
|
read, err := readForPlanning(ctx, open.inventory)
|
||||||
return entries, read, err
|
return entries, read, err
|
||||||
}
|
}
|
||||||
failing := ""
|
failing := ""
|
||||||
@@ -61,7 +105,7 @@ func catchingUpOnMerges(ctx context.Context, open *stores, announced merges) {
|
|||||||
case <-tick.C:
|
case <-tick.C:
|
||||||
}
|
}
|
||||||
watchedMerges.begin()
|
watchedMerges.begin()
|
||||||
err := catchUpOnMerges(ctx, time.Now(), announced, catalogued, f.SourceMoved, func(format string, args ...any) {
|
err := catchUpOnMerges(ctx, time.Now(), unheardMerges{announced, open.inventory}, catalogued, f.SourceMoved, func(format string, args ...any) {
|
||||||
fmt.Printf(format+"\n", args...)
|
fmt.Printf(format+"\n", args...)
|
||||||
})
|
})
|
||||||
// What the pass found is what S5 says (novox/hq to-be 45 §3); a pass that could not read
|
// What the pass found is what S5 says (novox/hq to-be 45 §3); a pass that could not read
|
||||||
@@ -102,10 +146,15 @@ func catchUpOnMerges(ctx context.Context, now time.Time, announced merges,
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
for _, a := range all {
|
for _, a := range all {
|
||||||
|
// A merge the forge said no time of is dated by its announcement, so a packaging module's look can
|
||||||
|
// make it history once acted on, and the catch-up does not act on it again every pass (ADR 0267).
|
||||||
|
if a.SourceMoved.MergedAt == "" && !a.At.IsZero() {
|
||||||
|
a.SourceMoved.MergedAt = a.At.UTC().Format(time.RFC3339)
|
||||||
|
}
|
||||||
if now.Sub(a.At) < mergeGrace {
|
if now.Sub(a.At) < mergeGrace {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if len(wouldMove(a.SourceMoved, entries, read)) == 0 {
|
if len(wouldMove(a.SourceMoved, entries, read)) == 0 && !owedToTheRecord(ctx, announced, a.SourceMoved, entries, read) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if entries, read, err = catalogued(ctx); err != nil {
|
if entries, read, err = catalogued(ctx); err != nil {
|
||||||
@@ -113,8 +162,13 @@ func catchUpOnMerges(ctx context.Context, now time.Time, announced merges,
|
|||||||
}
|
}
|
||||||
moves := wouldMove(a.SourceMoved, entries, read)
|
moves := wouldMove(a.SourceMoved, entries, read)
|
||||||
if len(moves) == 0 {
|
if len(moves) == 0 {
|
||||||
|
if !owedToTheRecord(ctx, announced, a.SourceMoved, entries, read) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
raw := a.SourceMoved
|
||||||
|
raw.MergedAt = ""
|
||||||
|
moves = wouldMove(raw, entries, read)
|
||||||
|
}
|
||||||
var names []string
|
var names []string
|
||||||
for _, e := range moves {
|
for _, e := range moves {
|
||||||
names = append(names, e.Manifest.Module)
|
names = append(names, e.Manifest.Module)
|
||||||
|
|||||||
@@ -74,9 +74,21 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke
|
|||||||
kept := inventory.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target,
|
kept := inventory.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target,
|
||||||
State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts,
|
State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts,
|
||||||
Check: r.Check, Needs: r.Needs, Account: r.Account, Root: r.Root}
|
Check: r.Check, Needs: r.Needs, Account: r.Account, Root: r.Root}
|
||||||
|
for _, w := range r.Waits {
|
||||||
|
kept.Waits = append(kept.Waits, inventory.Wait{Part: w.Part, Secret: w.Secret, Setting: w.Setting, What: w.What})
|
||||||
|
}
|
||||||
resources = append(resources, kept)
|
resources = append(resources, kept)
|
||||||
if r.State == link.StateUnhealthy && r.Module != "" {
|
}
|
||||||
unhealthy[r.Module] = append(unhealthy[r.Module], kept)
|
// **A wait for the operator is checked before it is excused** (novox/hq ADR 0283): a waiting resource whose
|
||||||
|
// wait does not check out is judged unhealthy, saying why; one that does is kept beside the unhealthy ones, so
|
||||||
|
// judgeModuleHealth can say it as needs-operator. What is stored is what the machine said.
|
||||||
|
var wf operatorWaitFacts
|
||||||
|
if mods := waitingModules(resources); len(mods) > 0 {
|
||||||
|
readWaitFacts(ctx, inv, node, mods, nil, &wf)
|
||||||
|
}
|
||||||
|
for _, r := range checkWaiting(node, resources, wf) {
|
||||||
|
if (r.State == link.StateUnhealthy || r.State == link.StateWaiting) && r.Module != "" {
|
||||||
|
unhealthy[r.Module] = append(unhealthy[r.Module], r)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
streaks := map[string]int{}
|
streaks := map[string]int{}
|
||||||
@@ -135,17 +147,16 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
|||||||
}
|
}
|
||||||
standing := map[string]conditions.Condition{}
|
standing := map[string]conditions.Condition{}
|
||||||
for _, c := range open {
|
for _, c := range open {
|
||||||
if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound) &&
|
if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound ||
|
||||||
|
c.Kind == kindNeedsOperator) &&
|
||||||
c.Subject.Machine == node {
|
c.Subject.Machine == node {
|
||||||
standing[c.Key] = c
|
standing[c.Key] = c
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
var hold *holding
|
hold, err := readHoldingFor(ctx, inv, open)
|
||||||
if inv != nil {
|
if err != nil {
|
||||||
if hold, err = readHolding(ctx, inv, open); err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
|
||||||
var problems []string
|
var problems []string
|
||||||
modules := make([]string, 0, len(unhealthy))
|
modules := make([]string, 0, len(unhealthy))
|
||||||
for m := range unhealthy {
|
for m := range unhealthy {
|
||||||
@@ -159,6 +170,39 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
|||||||
heldOn := map[string]string{}
|
heldOn := map[string]string{}
|
||||||
providers := map[catalogue.Chosen]bool{}
|
providers := map[catalogue.Chosen]bool{}
|
||||||
for _, m := range modules {
|
for _, m := range modules {
|
||||||
|
// **A part that waits for the operator is said as that** (novox/hq ADR 0283): its waits already checked,
|
||||||
|
// the operator's, never urgent, its words naming the act.
|
||||||
|
if waits, waiting := operatorWait(m, unhealthy[m]); waiting {
|
||||||
|
o := needsOperatorObservation(m, node, waits, unhealthy[m])
|
||||||
|
// **A provider waiting for the operator says who waits on it** (novox/hq issue 405), as one waiting for a
|
||||||
|
// login does: its consumers are held under it.
|
||||||
|
if hold != nil {
|
||||||
|
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
|
||||||
|
}
|
||||||
|
seen[o.Key()] = true
|
||||||
|
became[m] = kindNeedsOperator
|
||||||
|
if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := k.Observe(ctx, o); err != nil {
|
||||||
|
problems = append(problems, err.Error())
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// **A wait for the operator beside anything else is said too** (novox/hq issue 405): a module with a checked
|
||||||
|
// wait beside a new login owed, a directory used as found or a fault of its own is said as that, and the
|
||||||
|
// operator's secret or setting it waits for was not mentioned until the other cleared. Its needs-operator
|
||||||
|
// condition stands beside the other, on the same looks; what follows judges the rest without the wait.
|
||||||
|
if waits, rest := besideAWait(m, unhealthy[m]); len(waits) > 0 {
|
||||||
|
o := needsOperatorObservation(m, node, waits, waitingOf(m, unhealthy[m]))
|
||||||
|
seen[o.Key()] = true
|
||||||
|
if _, isOpen := standing[o.Key()]; streaks[m] >= moduleUnhealthyAfter || isOpen {
|
||||||
|
if _, err := k.Observe(ctx, o); err != nil {
|
||||||
|
problems = append(problems, err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
unhealthy[m] = rest
|
||||||
|
}
|
||||||
// **A directory used as found is said as that** (novox/hq issue 339): the operator's to hand over at the
|
// **A directory used as found is said as that** (novox/hq issue 339): the operator's to hand over at the
|
||||||
// machine, never urgent — nothing is broken by the wait that a person was not told of — and its own kind,
|
// machine, never urgent — nothing is broken by the wait that a person was not told of — and its own kind,
|
||||||
// so the gate never reads it as a fault of the build that happened to be sent beside it.
|
// so the gate never reads it as a fault of the build that happened to be sent beside it.
|
||||||
@@ -195,13 +239,24 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
|||||||
}
|
}
|
||||||
o := moduleUnhealthyObservation(m, node, unhealthy[m])
|
o := moduleUnhealthyObservation(m, node, unhealthy[m])
|
||||||
if hold != nil {
|
if hold != nil {
|
||||||
if p, held := hold.heldUnder(node, m, unhealthy[m]); held {
|
if by, held := hold.heldWith(node, m, unhealthy[m]); held {
|
||||||
// Held under the provider's condition: nothing of its own, and the provider's says it waits.
|
// Held under the provider's condition: nothing of its own, and the provider's says it waits. The
|
||||||
heldOn[o.Key()] = p.Module + " on " + p.Node
|
// clearing line says which the provider is: unhealthy, or waiting for the operator (issue 405).
|
||||||
|
p := by.provider
|
||||||
|
heldOn[o.Key()] = p.Module + " on " + p.Node + ", which is unhealthy"
|
||||||
|
if len(by.waits) > 0 {
|
||||||
|
heldOn[o.Key()] = p.Module + " on " + p.Node + ", which waits for you"
|
||||||
|
}
|
||||||
providers[p] = true
|
providers[p] = true
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
|
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
|
||||||
|
// A provider that waits for the operator for a part this finding cannot be matched to does not hold it
|
||||||
|
// (novox/hq issue 405): raised as its own, and saying the provider waits, so neither is hidden.
|
||||||
|
if p, waiting := hold.waitingUncovered(node, m, unhealthy[m]); waiting {
|
||||||
|
o.Said += fmt.Sprintf("; %s on %s waits for you, but not for anything %s is known to need, so %s's "+
|
||||||
|
"fault is said on its own", p.Module, p.Node, m, m)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
seen[o.Key()] = true
|
seen[o.Key()] = true
|
||||||
became[m] = kindModuleUnhealthy
|
became[m] = kindModuleUnhealthy
|
||||||
@@ -228,8 +283,11 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
|||||||
if c.Kind == kindUsedAsFound {
|
if c.Kind == kindUsedAsFound {
|
||||||
why = fmt.Sprintf("%s says no directory of %s is used as found any more", node, module)
|
why = fmt.Sprintf("%s says no directory of %s is used as found any more", node, module)
|
||||||
}
|
}
|
||||||
|
if c.Kind == kindNeedsOperator {
|
||||||
|
why = fmt.Sprintf("%s says %s no longer waits for the operator", node, module)
|
||||||
|
}
|
||||||
if on, held := heldOn[key]; held {
|
if on, held := heldOn[key]; held {
|
||||||
why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on)
|
why = fmt.Sprintf("what %s finds on %s waits on %s: held under its condition", module, node, on)
|
||||||
}
|
}
|
||||||
// **A condition that became the other kind** is not "working again" (issue 318 review): its clearing
|
// **A condition that became the other kind** is not "working again" (issue 318 review): its clearing
|
||||||
// line says what it became.
|
// line says what it became.
|
||||||
@@ -238,6 +296,12 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
|||||||
case c.Kind == kindModuleUnhealthy && became[module] == kindReloginNeeded:
|
case c.Kind == kindModuleUnhealthy && became[module] == kindReloginNeeded:
|
||||||
why = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
|
why = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
|
||||||
resolved = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
|
resolved = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
|
||||||
|
case c.Kind == kindModuleUnhealthy && became[module] == kindNeedsOperator:
|
||||||
|
why = fmt.Sprintf("%s on %s now only waits for the operator", module, node)
|
||||||
|
resolved = fmt.Sprintf("%s on %s now only waits for you", module, node)
|
||||||
|
case c.Kind == kindNeedsOperator && became[module] == kindModuleUnhealthy:
|
||||||
|
why = fmt.Sprintf("%s on %s no longer only waits for the operator, and is not healthy", module, node)
|
||||||
|
resolved = fmt.Sprintf("What %s on %s waited for is given, and it still does not work", module, node)
|
||||||
case c.Kind == kindReloginNeeded && became[module] == kindModuleUnhealthy:
|
case c.Kind == kindReloginNeeded && became[module] == kindModuleUnhealthy:
|
||||||
why = fmt.Sprintf("%s on %s no longer waits for a new login, and is not healthy", module, node)
|
why = fmt.Sprintf("%s on %s no longer waits for a new login, and is not healthy", module, node)
|
||||||
resolved = fmt.Sprintf("The new login on %s is done, and %s still does not work", node, module)
|
resolved = fmt.Sprintf("The new login on %s is done, and %s still does not work", node, module)
|
||||||
@@ -262,36 +326,74 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
|||||||
// sayWaiters observes a provider's open condition again, with who waits on it, from its machine's newest
|
// sayWaiters observes a provider's open condition again, with who waits on it, from its machine's newest
|
||||||
// statement. Nothing when its condition is not open: it is raised by its own statements, on its own looks.
|
// statement. Nothing when its condition is not open: it is raised by its own statements, on its own looks.
|
||||||
func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p catalogue.Chosen, now time.Time) error {
|
func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p catalogue.Chosen, now time.Time) error {
|
||||||
var raisedAt *conditions.Condition
|
// Its statement as it was judged, its waits checked (novox/hq issue 450): a wait that failed the check is
|
||||||
for i, c := range hold.open {
|
// unhealthy, so the condition built here is the one its own statement raised, and who waits on it is listed.
|
||||||
if c.Key == moduleUnhealthyKey(p.Module, p.Node) || c.Key == reloginKey(p.Module, p.Node) {
|
|
||||||
raisedAt = &hold.open[i]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if raisedAt == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
var rs []inventory.ResourceHealth
|
var rs []inventory.ResourceHealth
|
||||||
for _, r := range hold.healths[p.Node].Resources {
|
for _, r := range hold.checked(p.Node) {
|
||||||
if r.Module == p.Module && r.State == link.StateUnhealthy {
|
if r.Module == p.Module && (r.State == link.StateUnhealthy || r.State == link.StateWaiting) {
|
||||||
rs = append(rs, r)
|
rs = append(rs, r)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(rs) == 0 {
|
if len(rs) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
o := moduleUnhealthyObservation(p.Module, p.Node, rs)
|
// The condition its own statement says it under: needs-operator while it only waits for the operator (novox/hq
|
||||||
if said, waits := personWait(p.Module, p.Node, rs); waits {
|
// issue 405), else that for the rest of it, a wait beside it said on its own.
|
||||||
o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rs)
|
var o conditions.Observation
|
||||||
|
if waits, waiting := operatorWait(p.Module, rs); waiting {
|
||||||
|
o = needsOperatorObservation(p.Module, p.Node, waits, rs)
|
||||||
|
} else {
|
||||||
|
_, rest := besideAWait(p.Module, rs)
|
||||||
|
o = moduleUnhealthyObservation(p.Module, p.Node, rest)
|
||||||
|
if said, waits := personWait(p.Module, p.Node, rest); waits {
|
||||||
|
o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rest)
|
||||||
}
|
}
|
||||||
if o.Key() != raisedAt.Key {
|
}
|
||||||
return nil // its own statement says it next
|
raised := false
|
||||||
|
for _, c := range hold.open {
|
||||||
|
raised = raised || c.Key == o.Key()
|
||||||
|
}
|
||||||
|
if !raised {
|
||||||
|
return nil // not open yet, or open as another kind: its own statement says it next
|
||||||
}
|
}
|
||||||
sayWaitingOn(&o, hold.waitersOn(p))
|
sayWaitingOn(&o, hold.waitersOn(p))
|
||||||
_, err := k.Observe(ctx, o)
|
_, err := k.Observe(ctx, o)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// besideAWait is, for a module with something not healthy beside a part waiting for the operator, the waits (checked
|
||||||
|
// already) and the rest without the waiting parts (novox/hq issue 405); no waits when nothing waits, or when the
|
||||||
|
// module only waits (operatorWait says that whole). Pure.
|
||||||
|
func besideAWait(module string, rs []inventory.ResourceHealth) ([]inventory.Wait, []inventory.ResourceHealth) {
|
||||||
|
if _, only := operatorWait(module, rs); only {
|
||||||
|
return nil, rs
|
||||||
|
}
|
||||||
|
var waits []inventory.Wait
|
||||||
|
var rest []inventory.ResourceHealth
|
||||||
|
for _, r := range rs {
|
||||||
|
if r.Module == module && r.State == link.StateWaiting {
|
||||||
|
waits = append(waits, r.Waits...)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
rest = append(rest, r)
|
||||||
|
}
|
||||||
|
if len(waits) == 0 {
|
||||||
|
return nil, rs
|
||||||
|
}
|
||||||
|
return waits, rest
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitingOf is a module's waiting resources: the evidence of its wait.
|
||||||
|
func waitingOf(module string, rs []inventory.ResourceHealth) []inventory.ResourceHealth {
|
||||||
|
var out []inventory.ResourceHealth
|
||||||
|
for _, r := range rs {
|
||||||
|
if r.Module == module && r.State == link.StateWaiting {
|
||||||
|
out = append(out, r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// reloginKey is a module's relogin-needed condition on a machine.
|
// reloginKey is a module's relogin-needed condition on a machine.
|
||||||
func reloginKey(module, node string) string {
|
func reloginKey(module, node string) string {
|
||||||
return conditions.Key(conditions.ScopeModule, module+"."+node, kindReloginNeeded)
|
return conditions.Key(conditions.ScopeModule, module+"."+node, kindReloginNeeded)
|
||||||
@@ -363,12 +465,15 @@ func waitingAccounts(module string, rs []inventory.ResourceHealth) []string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// sayWaitingOn adds to a module's condition the consumers held under it (to-be 48 §6): urgent while anyone
|
// sayWaitingOn adds to a module's condition the consumers held under it (to-be 48 §6): urgent while anyone
|
||||||
// waits on it, whether it is not working or waits for a new login.
|
// waits on it, whether it is not working or waits for a new login. **A wait for the operator's secret or setting
|
||||||
|
// stays a warning** (ADR 0283 decision 5, novox/hq issue 405): who waits on it is listed, and nothing escalates it.
|
||||||
func sayWaitingOn(o *conditions.Observation, waiters []string) {
|
func sayWaitingOn(o *conditions.Observation, waiters []string) {
|
||||||
if len(waiters) == 0 {
|
if len(waiters) == 0 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if o.Kind != kindNeedsOperator {
|
||||||
o.Severity = conditions.Urgent
|
o.Severity = conditions.Urgent
|
||||||
|
}
|
||||||
o.Said += "; " + waitingWords(waiters)
|
o.Said += "; " + waitingWords(waiters)
|
||||||
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
|
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
|
||||||
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
|
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
|
||||||
@@ -420,6 +525,11 @@ func reasonWords(r inventory.ResourceHealth) string {
|
|||||||
case "":
|
case "":
|
||||||
return "is unhealthy"
|
return "is unhealthy"
|
||||||
}
|
}
|
||||||
|
// A wait for the operator that did not check out is said as the controller found it (ADR 0283): names of
|
||||||
|
// secrets and settings only, never what the check itself said.
|
||||||
|
if strings.HasPrefix(r.Reason, waitRefusedPrefix) {
|
||||||
|
return r.Reason
|
||||||
|
}
|
||||||
// What a declared check found says an endpoint, a path or an address: evidence, never the summary the
|
// What a declared check found says an endpoint, a path or an address: evidence, never the summary the
|
||||||
// operator's channel carries (ADR 0234 §6). The summary names the check.
|
// operator's channel carries (ADR 0234 §6). The summary names the check.
|
||||||
if r.Check != "" {
|
if r.Check != "" {
|
||||||
@@ -511,7 +621,9 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
|
|||||||
if h.HeardAt.Before(since) {
|
if h.HeardAt.Before(since) {
|
||||||
return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module)
|
return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module)
|
||||||
}
|
}
|
||||||
wait, waits := personWait(module, machine, h.Resources)
|
// **A wait for the operator is checked first** (novox/hq ADR 0283): one that does not check out is unhealthy.
|
||||||
|
resources := checkWaiting(machine, h.Resources, f.waits)
|
||||||
|
wait, waits := personWait(module, machine, resources)
|
||||||
// **Only a build whose own send put the account in a new group is excused** (issue 318 review): read from
|
// **Only a build whose own send put the account in a new group is excused** (issue 318 review): read from
|
||||||
// what the controller sent, never from when the machine says the wait began — that time is the engine's
|
// what the controller sent, never from when the machine says the wait began — that time is the engine's
|
||||||
// memory, reset by its restart and moved by a change of words. A build that adds no account group cannot
|
// memory, reset by its restart and moved by a change of words. A build that adds no account group cannot
|
||||||
@@ -519,11 +631,18 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
|
|||||||
if waits && !f.groupsAdded[module] {
|
if waits && !f.groupsAdded[module] {
|
||||||
waits = false
|
waits = false
|
||||||
}
|
}
|
||||||
var found []string
|
var found, onWaiting []string
|
||||||
for _, r := range h.Resources {
|
var forOperator []inventory.Wait
|
||||||
|
for _, r := range resources {
|
||||||
if r.Module != module {
|
if r.Module != module {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
// **Any build is excused while its wait for the operator checks out** (ADR 0283 decision 4): a secret not
|
||||||
|
// given is owed by every build alike, so it is no fault of this one, and the verdict carries it.
|
||||||
|
if r.State == link.StateWaiting {
|
||||||
|
forOperator = append(forOperator, r.Waits...)
|
||||||
|
continue
|
||||||
|
}
|
||||||
if waits && r.State == link.StateUnhealthy {
|
if waits && r.State == link.StateUnhealthy {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -541,8 +660,16 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
|
|||||||
return healthNotYet, fmt.Sprintf("its %s %s on %s is still starting", r.Kind, r.Resource, machine)
|
return healthNotYet, fmt.Sprintf("its %s %s on %s is still starting", r.Kind, r.Resource, machine)
|
||||||
case link.StateUnhealthy:
|
case link.StateUnhealthy:
|
||||||
if on, held := f.heldOn[module+"@"+machine]; held {
|
if on, held := f.heldOn[module+"@"+machine]; held {
|
||||||
|
// **Held under a provider that only waits for the operator** (novox/hq issue 405): a wait for a
|
||||||
|
// person, a pass carrying the wait (ADR 0254, ADR 0283 decision 4) — the walk never waits for the
|
||||||
|
// operator's secret, whichever module owes it.
|
||||||
|
if on.waits != "" {
|
||||||
|
onWaiting = append(onWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which waits for you: %s",
|
||||||
|
r.Kind, r.Resource, machine, on.on, on.waits))
|
||||||
|
continue
|
||||||
|
}
|
||||||
return healthWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which is unhealthy", r.Kind,
|
return healthWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which is unhealthy", r.Kind,
|
||||||
r.Resource, machine, on)
|
r.Resource, machine, on.on)
|
||||||
}
|
}
|
||||||
return healthNotYet, fmt.Sprintf("its %s %s on %s %s", r.Kind, r.Resource, machine, reasonWords(r))
|
return healthNotYet, fmt.Sprintf("its %s %s on %s %s", r.Kind, r.Resource, machine, reasonWords(r))
|
||||||
default:
|
default:
|
||||||
@@ -550,14 +677,17 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
|
|||||||
reasonAfter(r.Reason))
|
reasonAfter(r.Reason))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if waits || len(found) > 0 {
|
if waits || len(found) > 0 || len(forOperator) > 0 || len(onWaiting) > 0 {
|
||||||
var said []string
|
said := onWaiting
|
||||||
if waits {
|
if waits {
|
||||||
said = append(said, wait)
|
said = append(said, wait)
|
||||||
}
|
}
|
||||||
|
if len(forOperator) > 0 {
|
||||||
|
said = append(said, operatorWaitSaid(module, machine, forOperator))
|
||||||
|
}
|
||||||
if len(found) > 0 {
|
if len(found) > 0 {
|
||||||
said = append(said, fmt.Sprintf("on %s, %s uses %s as found and waits for a person to hand it over "+
|
said = append(said, fmt.Sprintf("on %s, %s uses %s as found and waits for the operator to hand it over "+
|
||||||
"(`mesh-host hand-over <directory>` at the machine)", machine, module, strings.Join(found, ", ")))
|
"(`nox node hand-over %s <directory>` on the control-node)", machine, module, strings.Join(found, ", "), machine))
|
||||||
}
|
}
|
||||||
return healthPerson, strings.Join(said, "; ")
|
return healthPerson, strings.Join(said, "; ")
|
||||||
}
|
}
|
||||||
@@ -678,7 +808,9 @@ func usedAsFoundObservation(module, node, said string, rs []inventory.ResourceHe
|
|||||||
o.Explanation = fmt.Sprintf("A directory of %s was already on %s, with another owner or mode than %s declares. "+
|
o.Explanation = fmt.Sprintf("A directory of %s was already on %s, with another owner or mode than %s declares. "+
|
||||||
"The mesh left it as it was rather than hand it to an account, so %s may not be able to use it.",
|
"The mesh left it as it was rather than hand it to an account, so %s may not be able to use it.",
|
||||||
module, node, module, module)
|
module, node, module, module)
|
||||||
o.Needs = fmt.Sprintf("on %s, run mesh-host hand-over with the directory's path as root.", node)
|
// Plain words (ADR 0253): the line itself — `nox node hand-over <node> <path>` on the control-node (ADR 0272,
|
||||||
|
// issue 356) — is in the summary and the evidence, which name the directory; a path is never in these.
|
||||||
|
o.Needs = "hand the directory over from the control-node, as the operator; the details name it and the line to type."
|
||||||
o.Resolved = fmt.Sprintf("%s's directory on %s is the mesh's", module, node)
|
o.Resolved = fmt.Sprintf("%s's directory on %s is the mesh's", module, node)
|
||||||
o.Actions = nil
|
o.Actions = nil
|
||||||
return o
|
return o
|
||||||
|
|||||||
@@ -10,10 +10,12 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
"github.com/novox/mesh-controller/internal/overlay"
|
"github.com/novox/mesh-controller/internal/overlay"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -397,8 +399,9 @@ func assignCommand(ctx context.Context, verb string, args []string) error {
|
|||||||
func settingsCommand(ctx context.Context, args []string) error {
|
func settingsCommand(ctx context.Context, args []string) error {
|
||||||
if len(args) == 0 {
|
if len(args) == 0 {
|
||||||
return errors.New("settings show <module> [--node <node>] [--history], settings set <module> <file> " +
|
return errors.New("settings show <module> [--node <node>] [--history], settings set <module> <file> " +
|
||||||
"[--node <node>] [--replace], settings clear <module> [--node <node>], or settings preferences " +
|
"[--node <node>] [--replace], settings clear <module> [--node <node>], settings preferences " +
|
||||||
"[<module>] [--node <node>]")
|
"[<module>] [--node <node>], settings propose <module> <file | --clear> [--node <node>] [--replace], " +
|
||||||
|
"or settings proposals [<id>]")
|
||||||
}
|
}
|
||||||
open, err := openStores(ctx)
|
open, err := openStores(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -412,12 +415,38 @@ func settingsCommand(ctx context.Context, args []string) error {
|
|||||||
// **What a set removes is refused unless meant** (novox/hq ADR 0217). A layer is replaced whole,
|
// **What a set removes is refused unless meant** (novox/hq ADR 0217). A layer is replaced whole,
|
||||||
// and on 2026-10-05 setting one placement dropped a machine's whole layer for a module without a
|
// and on 2026-10-05 setting one placement dropped a machine's whole layer for a module without a
|
||||||
// word (novox/hq issue 304). Adding and changing keys needs nothing; removing one needs this.
|
// word (novox/hq issue 304). Adding and changing keys needs nothing; removing one needs this.
|
||||||
replace := set.Bool("replace", false, "for set: remove the keys the new layer does not name")
|
replace := set.Bool("replace", false, "for set and propose: remove the keys the new layer does not name")
|
||||||
history := set.Bool("history", false, "for show: the layers this one replaced, the latest first")
|
history := set.Bool("history", false, "for show: the layers this one replaced, the latest first")
|
||||||
|
clear := set.Bool("clear", false, "for propose: propose that the layer be removed")
|
||||||
positionals, err := parseAround(set, args[1:])
|
positionals, err := parseAround(set, args[1:])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
switch args[0] {
|
||||||
|
case "propose":
|
||||||
|
// A trusted setting, proposed by anyone and set only on the operator's warrant (novox/hq ADR 0277):
|
||||||
|
// the stores are opened there, so the proposal and the verb's refusals below never meet.
|
||||||
|
if len(positionals) < 1 || len(positionals) > 2 {
|
||||||
|
return errors.New("settings propose <module> <settings.json | {…}> [--node <node>] [--replace], or settings propose <module> --clear [--node <node>]")
|
||||||
|
}
|
||||||
|
values := ""
|
||||||
|
if len(positionals) == 2 {
|
||||||
|
values = positionals[1]
|
||||||
|
}
|
||||||
|
return proposeCommand(ctx, positionals[0], *node, values, *clear, *replace)
|
||||||
|
case "proposals":
|
||||||
|
if len(positionals) > 1 || *node != "" || *clear || *replace || *history {
|
||||||
|
return errors.New("settings proposals [<id>]")
|
||||||
|
}
|
||||||
|
id := ""
|
||||||
|
if len(positionals) == 1 {
|
||||||
|
id = positionals[0]
|
||||||
|
}
|
||||||
|
return proposalsCommand(ctx, id)
|
||||||
|
}
|
||||||
|
if *clear {
|
||||||
|
return errors.New("--clear is for settings propose; a layer is cleared with settings clear")
|
||||||
|
}
|
||||||
|
|
||||||
where := "the whole mesh"
|
where := "the whole mesh"
|
||||||
if *node != "" {
|
if *node != "" {
|
||||||
@@ -455,7 +484,7 @@ func settingsCommand(ctx context.Context, args []string) error {
|
|||||||
"removal is meant (novox/hq ADR 0217). Nothing was changed",
|
"removal is meant (novox/hq ADR 0217). Nothing was changed",
|
||||||
positionals[0], where, strings.Join(removed, ", "), positionals[0], nodeFlag(*node))
|
positionals[0], where, strings.Join(removed, ", "), positionals[0], nodeFlag(*node))
|
||||||
}
|
}
|
||||||
if err := inv.SetSettings(ctx, *node, positionals[0], values); err != nil {
|
if err := inv.SetSettingsBy(ctx, *node, positionals[0], values, setByWords()); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
fmt.Printf("%s on %s:\n", positionals[0], where)
|
fmt.Printf("%s on %s:\n", positionals[0], where)
|
||||||
@@ -496,8 +525,12 @@ func settingsCommand(ctx context.Context, args []string) error {
|
|||||||
}
|
}
|
||||||
for _, p := range past {
|
for _, p := range past {
|
||||||
shown, _ := json.MarshalIndent(p.Values, " ", " ")
|
shown, _ := json.MarshalIndent(p.Values, " ", " ")
|
||||||
fmt.Printf("%s on %s, until %s (%s):\n %s\n", positionals[0], where,
|
by := ""
|
||||||
p.ReplacedAt.Local().Format("2006-01-02 15:04:05"), p.ReplacedBy, shown)
|
if p.SetBy != "" {
|
||||||
|
by = "; " + p.SetBy
|
||||||
|
}
|
||||||
|
fmt.Printf("%s on %s, until %s (%s%s):\n %s\n", positionals[0], where,
|
||||||
|
p.ReplacedAt.Local().Format("2006-01-02 15:04:05"), p.ReplacedBy, by, shown)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -513,6 +546,14 @@ func settingsCommand(ctx context.Context, args []string) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
fmt.Println(string(shown))
|
fmt.Println(string(shown))
|
||||||
|
// And who set it (novox/hq ADR 0277): a layer the operator approved on their phone says so.
|
||||||
|
if setBy, setAt, has, err := inv.LayerOrigin(ctx, *node, positionals[0]); err != nil {
|
||||||
|
return err
|
||||||
|
} else if has && setBy != "" {
|
||||||
|
fmt.Printf(" %s\n", setBy)
|
||||||
|
} else if has {
|
||||||
|
fmt.Printf(" set at %s; who set it was not kept\n", setAt.Local().Format("2006-01-02 15:04"))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// Every value the module gives a default or a layer sets, and where it came from (novox/hq
|
// Every value the module gives a default or a layer sets, and where it came from (novox/hq
|
||||||
// ADR 0262): the default, the mesh's layer, or this node's. Said after the layer, which stays
|
// ADR 0262): the default, the mesh's layer, or this node's. Said after the layer, which stays
|
||||||
@@ -606,17 +647,26 @@ func settingsCommand(ctx context.Context, args []string) error {
|
|||||||
if err := refuseTerminalSettingsThroughAVerb(ctx, inv, before, nil, positionals[0], where); err != nil {
|
if err := refuseTerminalSettingsThroughAVerb(ctx, inv, before, nil, positionals[0], where); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := inv.ClearSettings(ctx, *node, positionals[0]); err != nil {
|
if err := inv.ClearSettingsBy(ctx, *node, positionals[0], setByWords()); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
fmt.Printf("%s on %s is back to what the module says\n", positionals[0], where)
|
fmt.Printf("%s on %s is back to what the module says\n", positionals[0], where)
|
||||||
return nil
|
return nil
|
||||||
|
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("settings has no %q; it has show, set, clear and preferences", args[0])
|
return fmt.Errorf("settings has no %q; it has show, set, clear, preferences, propose and proposals", args[0])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// setByWords is who sets a layer from this process, as the layer keeps it (novox/hq ADR 0277): the caller at the
|
||||||
|
// controller's terminal, or through which verb.
|
||||||
|
func setByWords() string {
|
||||||
|
if verb, through := throughAVerb(); through {
|
||||||
|
return "set by " + link.Caller() + " through " + verb + " at " + time.Now().Local().Format("2006-01-02 15:04")
|
||||||
|
}
|
||||||
|
return "set at the controller's terminal by " + link.Caller() + " at " + time.Now().Local().Format("2006-01-02 15:04")
|
||||||
|
}
|
||||||
|
|
||||||
// describeEffective says each setting's value on a machine or the whole mesh, where it came from, and
|
// describeEffective says each setting's value on a machine or the whole mesh, where it came from, and
|
||||||
// the module's default when a layer overrides it.
|
// the module's default when a layer overrides it.
|
||||||
func describeEffective(module, where string, values []catalogue.SettingSource) string {
|
func describeEffective(module, where string, values []catalogue.SettingSource) string {
|
||||||
@@ -1107,10 +1157,13 @@ func refuseTerminalSettingsThroughAVerb(ctx context.Context, inv *inventory.Inve
|
|||||||
}
|
}
|
||||||
// A trusted mergeable file takes any key, so its module's whole layer is the terminal's (novox/hq issue 340).
|
// A trusted mergeable file takes any key, so its module's whole layer is the terminal's (novox/hq issue 340).
|
||||||
if files := catalogue.TrustedMergeable(shelf[module]); len(files) > 0 && !sameLayer(before, after) {
|
if files := catalogue.TrustedMergeable(shelf[module]); len(files) > 0 && !sameLayer(before, after) {
|
||||||
return fmt.Errorf("the settings of %s on %s are set at the controller's terminal only (`mesh-cli` on the control-node), never through a verb (this "+
|
return fmt.Errorf("the settings of %s on %s are set at the controller's terminal (`mesh-cli` on the control-node) or on "+
|
||||||
"line came through %q): %s merges whatever key a layer sets into a file root or a consumer trusts, so "+
|
"the operator's warrant, never through a verb alone (this line came through %q): %s merges whatever key a "+
|
||||||
"any key could point the module at a listener of the caller's, and whoever may call a verb includes "+
|
"layer sets into a file root or a consumer trusts, so any key could point the module at a listener of the "+
|
||||||
"agents (novox/hq issue 340; a file nothing trusts says \"trusted\": false). Nothing was changed",
|
"caller's, and whoever may call a verb includes agents (novox/hq issue 340; a file nothing trusts says "+
|
||||||
|
"\"trusted\": false). Propose it instead: the settings verb with propose puts the exact values to the "+
|
||||||
|
"operator on a channel that proves who answers, and the layer is set on their Approve (novox/hq ADR 0277). "+
|
||||||
|
"Nothing was changed",
|
||||||
module, where, verb, strings.Join(files, ", "))
|
module, where, verb, strings.Join(files, ", "))
|
||||||
}
|
}
|
||||||
for _, key := range catalogue.TerminalKeys(shelf[module]) {
|
for _, key := range catalogue.TerminalKeys(shelf[module]) {
|
||||||
@@ -1119,11 +1172,13 @@ func refuseTerminalSettingsThroughAVerb(ctx context.Context, inv *inventory.Inve
|
|||||||
if string(was) == string(now) {
|
if string(was) == string(now) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
return fmt.Errorf("%s of %s on %s is set at the controller's terminal only (`mesh-cli` on the control-node), never through a verb (this "+
|
return fmt.Errorf("%s of %s on %s is set at the controller's terminal (`mesh-cli` on the control-node) or on the "+
|
||||||
"line came through %q): it says where root creates and owns a module's directories, which of "+
|
"operator's warrant, never through a verb alone (this line came through %q): it says where root creates and "+
|
||||||
"the machine's paths are mounted into its container, what the mesh's consumers trust, or what a file "+
|
"owns a module's directories, which of the machine's paths are mounted into its container, what the mesh's "+
|
||||||
"root or a person's session obeys takes, and whoever may call a verb includes agents (novox/hq issue 339; "+
|
"consumers trust, or what a file root or a person's session obeys takes, and whoever may call a verb "+
|
||||||
"issue 340 for a mergeable file's own keys). Nothing was changed", key, module, where, verb)
|
"includes agents (novox/hq issue 339; issue 340 for a mergeable file's own keys). Propose it instead: the "+
|
||||||
|
"settings verb with propose puts the exact values to the operator on a channel that proves who answers, and "+
|
||||||
|
"the layer is set on their Approve (novox/hq ADR 0277). Nothing was changed", key, module, where, verb)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -426,10 +426,10 @@ func overlayShow(ctx context.Context, open *stores) error {
|
|||||||
tunnel.Interface, tunnel.Range, tunnel.Port)
|
tunnel.Interface, tunnel.Range, tunnel.Port)
|
||||||
case n.Hub && hubName == n.Name && tunnel.Interface != "":
|
case n.Hub && hubName == n.Name && tunnel.Interface != "":
|
||||||
fmt.Printf(" hub — found a tunnel on %s and did NOT take it over: its key is not the tunnel's; "+
|
fmt.Printf(" hub — found a tunnel on %s and did NOT take it over: its key is not the tunnel's; "+
|
||||||
"`mesh-host overlay take --tunnel %s` on the machine takes it", tunnel.Interface, tunnel.Interface)
|
"`nox-mesh-host overlay take --tunnel %s` on the machine takes it", tunnel.Interface, tunnel.Interface)
|
||||||
case n.Hub:
|
case n.Hub:
|
||||||
fmt.Print(" hub — found no tunnel; if the machine runs the predecessor's, " +
|
fmt.Print(" hub — found no tunnel; if the machine runs the predecessor's, " +
|
||||||
"`mesh-host overlay take --tunnel <iface>` there adopts it (novox/hq ADR 0105)")
|
"`nox-mesh-host overlay take --tunnel <iface>` there adopts it (novox/hq ADR 0105)")
|
||||||
case !n.Reachable():
|
case !n.Reachable():
|
||||||
fmt.Print(" not dialable")
|
fmt.Print(" not dialable")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,10 +3,12 @@ package main
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"reflect"
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/beside"
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/overlay"
|
"github.com/novox/mesh-controller/internal/overlay"
|
||||||
@@ -239,13 +241,12 @@ func TestATakeoverIsNotComposedForAHubPlacedOffItsTunnel(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// theResolver is the catalogue's dnsmasq module as it is, or the test is skipped where the
|
// theResolver is the catalogue's dnsmasq module as it is (internal/beside).
|
||||||
// catalogue is not beside this checkout.
|
|
||||||
func theResolver(t *testing.T) catalogue.Manifest {
|
func theResolver(t *testing.T) catalogue.Manifest {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
raw, err := os.ReadFile("../../../mesh-catalog/modules/dnsmasq/module.json")
|
raw, err := os.ReadFile(filepath.Join(beside.Catalogue(t), "dnsmasq", "module.json"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Skipf("the catalogue is not beside this checkout: %v", err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
m, err := catalogue.ParseManifest(raw)
|
m, err := catalogue.ParseManifest(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"net"
|
"net"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -17,6 +18,7 @@ import (
|
|||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
"github.com/novox/mesh-controller/internal/token"
|
"github.com/novox/mesh-controller/internal/token"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -28,7 +30,7 @@ import (
|
|||||||
|
|
||||||
func nodeCommand(ctx context.Context, args []string) error {
|
func nodeCommand(ctx context.Context, args []string) error {
|
||||||
if len(args) == 0 {
|
if len(args) == 0 {
|
||||||
return errors.New("node add <name>, node list, node show <name>, or " + publicDomainUsage)
|
return errors.New("node add <name>, node list, node show <name>, " + publicDomainUsage + ", or " + handOverUsage)
|
||||||
}
|
}
|
||||||
open, err := openStores(ctx)
|
open, err := openStores(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -112,11 +114,163 @@ func nodeCommand(ctx context.Context, args []string) error {
|
|||||||
// an optional second argument is the home when it is not /home/<account>.
|
// an optional second argument is the home when it is not /home/<account>.
|
||||||
return nodeAccount(ctx, inv, args[1:])
|
return nodeAccount(ctx, inv, args[1:])
|
||||||
|
|
||||||
|
case "hand-over":
|
||||||
|
// A directory the node-engine uses as found, handed to the mesh (novox/hq issue 356, issue 339). Here, at
|
||||||
|
// the controller's terminal, and nowhere else: at the next apply root gives the directory to the account
|
||||||
|
// the module declares, and whoever may call a verb includes agents.
|
||||||
|
return nodeHandOver(ctx, open, args[1:])
|
||||||
|
|
||||||
|
case "setuid-search":
|
||||||
|
// A fresh search for setuid programs on a node (novox/hq issue 361), after the operator changed by hand
|
||||||
|
// what the last one found. Here, at the controller's terminal, and nowhere else: a search never makes a
|
||||||
|
// machine free wrongly, but asked again and again it would keep the machine unjudged and its disks busy,
|
||||||
|
// and whoever may call a verb includes agents.
|
||||||
|
return nodeSetuidSearch(ctx, open, args[1:])
|
||||||
|
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("node has no %q; it has add, list, show, public-domain, account and agent-account", args[0])
|
return fmt.Errorf("node has no %q; it has add, list, show, public-domain, account, agent-account, hand-over "+
|
||||||
|
"and setuid-search", args[0])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const handOverUsage = "node hand-over <node> <directory> — hand a directory the node-engine on <node> uses as found " +
|
||||||
|
"to the mesh: its next apply gives it the declared owner and mode. The directory's absolute path, as the module's " +
|
||||||
|
"condition names it"
|
||||||
|
|
||||||
|
// handOverLine reads a hand-over's line: the node and the directory's absolute path, exactly as the engine states
|
||||||
|
// it. Judged before anything is asked, and judged again by the engine, which is the one that acts.
|
||||||
|
func handOverLine(args []string) (node, path string, err error) {
|
||||||
|
if len(args) != 2 {
|
||||||
|
return "", "", errors.New(handOverUsage)
|
||||||
|
}
|
||||||
|
node, path = args[0], args[1]
|
||||||
|
if node == "" || strings.HasPrefix(node, "-") {
|
||||||
|
return "", "", fmt.Errorf("%q is not a node's name; %s", node, handOverUsage)
|
||||||
|
}
|
||||||
|
if !filepath.IsAbs(path) {
|
||||||
|
return "", "", fmt.Errorf("%q is not an absolute path; %s", path, handOverUsage)
|
||||||
|
}
|
||||||
|
if filepath.Clean(path) != path {
|
||||||
|
return "", "", fmt.Errorf("%q is not the directory's path as the engine states it (no `..`, no doubled or "+
|
||||||
|
"trailing separator); %s", path, handOverUsage)
|
||||||
|
}
|
||||||
|
return node, path, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// handOverBy is who hands the directory over, in the words a verb's caller is recorded in: the operator through
|
||||||
|
// mesh-cli on the control-node, or whoever runs this controller's binary at its terminal.
|
||||||
|
func handOverBy() string {
|
||||||
|
if by := strings.TrimSpace(os.Getenv(link.CallerVar)); by != "" {
|
||||||
|
return by
|
||||||
|
}
|
||||||
|
return "the controller's terminal"
|
||||||
|
}
|
||||||
|
|
||||||
|
// nodeHandOver asks the node's engine to take a directory it uses as found as the mesh's, and says what came of
|
||||||
|
// it. The engine records the hand-over or refuses; nothing is recorded here, because the directory is the
|
||||||
|
// machine's and the engine is the one that reads it. The ask is signed with the mesh's key (issue 356's review).
|
||||||
|
func nodeHandOver(ctx context.Context, open *stores, args []string) error {
|
||||||
|
known := func(node string) error {
|
||||||
|
_, err := open.inventory.NodeByName(ctx, node)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
ask := func(node, path, by string) (link.HandOverAnswer, error) {
|
||||||
|
ident, err := open.Identity(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return link.HandOverAnswer{}, fmt.Errorf("the mesh's signing key cannot be read, so nothing was asked of %s: %w",
|
||||||
|
node, err)
|
||||||
|
}
|
||||||
|
address, err := broker.BusAddress()
|
||||||
|
if err != nil {
|
||||||
|
return link.HandOverAnswer{}, err
|
||||||
|
}
|
||||||
|
js, err := broker.Dial(address)
|
||||||
|
if err != nil {
|
||||||
|
return link.HandOverAnswer{}, fmt.Errorf("cannot reach the bus, so nothing was asked of %s: %w", node, err)
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
return link.AskHandOver(ctx, js.Conn(), ident, node, path, by, link.HandOverWithin)
|
||||||
|
}
|
||||||
|
return handOverAsked(args, known, ask, os.Stdout)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handOverAsked is the hand-over's line with its two acts given: whether the mesh knows the node, and the ask.
|
||||||
|
// Nothing is asked of a line or a node that is refused, and the engine's refusal is this command's failure —
|
||||||
|
// never a success with the refusal printed.
|
||||||
|
func handOverAsked(args []string, known func(node string) error,
|
||||||
|
ask func(node, path, by string) (link.HandOverAnswer, error), out io.Writer) error {
|
||||||
|
node, path, err := handOverLine(args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := known(node); err != nil {
|
||||||
|
return fmt.Errorf("nothing was asked: %w", err)
|
||||||
|
}
|
||||||
|
answer, err := ask(node, path, handOverBy())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if answer.Refused != "" {
|
||||||
|
return fmt.Errorf("%s refused: %s", node, answer.Refused)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(out, answer.Said)
|
||||||
|
fmt.Fprintf(out, " the module's condition clears once %s applies; `nox push %s` applies it now\n", node, node)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const setuidSearchUsage = "node setuid-search <node> — throw away the node-engine's last search for setuid " +
|
||||||
|
"programs on <node> and start a full one: after a setuid-root program it found was removed by hand. Until it " +
|
||||||
|
"completes, root-free says the node is not judged yet"
|
||||||
|
|
||||||
|
// nodeSetuidSearch asks the node's engine for a fresh search, signed with the mesh's key as a hand-over is.
|
||||||
|
func nodeSetuidSearch(ctx context.Context, open *stores, args []string) error {
|
||||||
|
known := func(node string) error {
|
||||||
|
_, err := open.inventory.NodeByName(ctx, node)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
ask := func(node, by string) (link.HandOverAnswer, error) {
|
||||||
|
ident, err := open.Identity(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return link.HandOverAnswer{}, fmt.Errorf("the mesh's signing key cannot be read, so nothing was asked of %s: %w",
|
||||||
|
node, err)
|
||||||
|
}
|
||||||
|
address, err := broker.BusAddress()
|
||||||
|
if err != nil {
|
||||||
|
return link.HandOverAnswer{}, err
|
||||||
|
}
|
||||||
|
js, err := broker.Dial(address)
|
||||||
|
if err != nil {
|
||||||
|
return link.HandOverAnswer{}, fmt.Errorf("cannot reach the bus, so nothing was asked of %s: %w", node, err)
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
return link.AskSetuidSearch(ctx, js.Conn(), ident, node, by, link.HandOverWithin)
|
||||||
|
}
|
||||||
|
return setuidSearchAsked(args, known, ask, os.Stdout)
|
||||||
|
}
|
||||||
|
|
||||||
|
// setuidSearchAsked is the line with its two acts given: whether the mesh knows the node, and the ask. The
|
||||||
|
// engine's refusal is this command's failure.
|
||||||
|
func setuidSearchAsked(args []string, known func(node string) error,
|
||||||
|
ask func(node, by string) (link.HandOverAnswer, error), out io.Writer) error {
|
||||||
|
if len(args) != 1 || args[0] == "" || strings.HasPrefix(args[0], "-") {
|
||||||
|
return errors.New(setuidSearchUsage)
|
||||||
|
}
|
||||||
|
node := args[0]
|
||||||
|
if err := known(node); err != nil {
|
||||||
|
return fmt.Errorf("nothing was asked: %w", err)
|
||||||
|
}
|
||||||
|
answer, err := ask(node, handOverBy())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if answer.Refused != "" {
|
||||||
|
return fmt.Errorf("%s refused: %s", node, answer.Refused)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(out, answer.Said)
|
||||||
|
fmt.Fprintf(out, " the controller's root-free verb shows the search's progress until it completes\n")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// addNode creates a node record, adopted when the operator says so (novox/hq ADR 0100).
|
// addNode creates a node record, adopted when the operator says so (novox/hq ADR 0100).
|
||||||
func addNode(ctx context.Context, inv *inventory.Inventory, args []string) error {
|
func addNode(ctx context.Context, inv *inventory.Inventory, args []string) error {
|
||||||
set := flag.NewFlagSet("node add", flag.ContinueOnError)
|
set := flag.NewFlagSet("node add", flag.ContinueOnError)
|
||||||
|
|||||||
@@ -0,0 +1,261 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A part that waits for the operator's secret or setting (novox/hq ADR 0283, issue 386).
|
||||||
|
//
|
||||||
|
// **A module's tool check may say it waits**: nothing of it is wrong but a part that cannot work until the operator
|
||||||
|
// gives one of its own secrets or one of its settings. The node-engine states such a resource `waiting`, with what
|
||||||
|
// it waits for. A module saying so is an assertion, so **the controller checks each wait before it excuses it**:
|
||||||
|
//
|
||||||
|
// - a wait for a secret names an own secret the module's manifest declares — by its name, or as a member of a
|
||||||
|
// secret family — said `"issued-by": "outside"`, and the store holds no value a person gave for it on that
|
||||||
|
// machine;
|
||||||
|
// - a wait for a setting names a setting the manifest declares (checked by name only: the controller cannot tell
|
||||||
|
// whether a free-form value covers a part, and the needs-operator condition is where a false one shows).
|
||||||
|
//
|
||||||
|
// An excused wait is read by the first-node gate as *waits for a person* (ADR 0254), a pass carried in the verdict,
|
||||||
|
// for any build of the module — a secret not given is owed by every build alike. It is said to the operator as
|
||||||
|
// `module.<module>.<machine>.needs-operator`, naming the act. A wait that fails the check is judged unhealthy, saying
|
||||||
|
// why, and raises the module's `unhealthy` condition.
|
||||||
|
|
||||||
|
// kindNeedsOperator is a module's condition while a part of it waits for the operator's secret or setting.
|
||||||
|
const kindNeedsOperator = "needs-operator"
|
||||||
|
|
||||||
|
// needsOperatorKey is a module's needs-operator condition on a machine.
|
||||||
|
func needsOperatorKey(module, node string) string {
|
||||||
|
return conditions.Key(conditions.ScopeModule, module+"."+node, kindNeedsOperator)
|
||||||
|
}
|
||||||
|
|
||||||
|
// operatorWaitFacts is what the controller holds to check a module's waits: the manifest judged per module, and per
|
||||||
|
// "<module>@<machine>" the own secrets a person gave there, with when. A module or a machine absent is not known,
|
||||||
|
// and no wait of it is excused.
|
||||||
|
type operatorWaitFacts struct {
|
||||||
|
manifests map[string]catalogue.Manifest
|
||||||
|
given map[string]map[string]time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkWait is nil when a wait is excused, and otherwise why not, in words. Pure.
|
||||||
|
func checkWait(module, machine string, w inventory.Wait, f operatorWaitFacts) error {
|
||||||
|
m, known := f.manifests[module]
|
||||||
|
if !known {
|
||||||
|
return fmt.Errorf("says it waits for %s, and the mesh holds no manifest of %s to check it against", waitNames(w), module)
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case w.Secret != "" && w.Setting != "", w.Secret == "" && w.Setting == "":
|
||||||
|
return fmt.Errorf("says it waits, naming %s, where a wait names one secret or one setting", waitNames(w))
|
||||||
|
case w.Setting != "":
|
||||||
|
if _, declared := m.Settings[w.Setting]; !declared {
|
||||||
|
return fmt.Errorf("says it waits for the setting %s, which %s does not declare", w.Setting, module)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
own, _, declared := m.OwnSecrets.Lookup(w.Secret)
|
||||||
|
if !declared {
|
||||||
|
return fmt.Errorf("says it waits for the secret %s, which %s does not declare", w.Secret, module)
|
||||||
|
}
|
||||||
|
if own.IssuedBy != catalogue.IssuedOutside {
|
||||||
|
return fmt.Errorf("says it waits for the secret %s, which the mesh makes itself: only a secret issued outside "+
|
||||||
|
"the mesh waits for the operator", w.Secret)
|
||||||
|
}
|
||||||
|
given, readable := f.given[module+"@"+machine]
|
||||||
|
if !readable {
|
||||||
|
return fmt.Errorf("says it waits for the secret %s, and what was given on %s could not be read", w.Secret, machine)
|
||||||
|
}
|
||||||
|
if at, was := given[w.Secret]; was {
|
||||||
|
return fmt.Errorf("says it waits for the secret %s, which was given at %s", w.Secret,
|
||||||
|
at.UTC().Format("2006-01-02 15:04 MST"))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitRefusedPrefix opens every reason checkWait gives, so the words of a refused wait are told from a check's own.
|
||||||
|
const waitRefusedPrefix = "says it waits"
|
||||||
|
|
||||||
|
// waitNames is what a wait names, as "the secret x" or "the setting y".
|
||||||
|
func waitNames(w inventory.Wait) string {
|
||||||
|
switch {
|
||||||
|
case w.Secret != "" && w.Setting != "":
|
||||||
|
return "the secret " + w.Secret + " and the setting " + w.Setting
|
||||||
|
case w.Secret != "":
|
||||||
|
return "the secret " + w.Secret
|
||||||
|
case w.Setting != "":
|
||||||
|
return "the setting " + w.Setting
|
||||||
|
}
|
||||||
|
return "nothing"
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkWaiting reads one machine's resources against the facts: every waiting resource whose waits all check out is
|
||||||
|
// kept as said; one with a wait that does not, or with no wait at all, is answered as unhealthy with why. Pure; the
|
||||||
|
// statement as kept is not changed.
|
||||||
|
func checkWaiting(machine string, rs []inventory.ResourceHealth, f operatorWaitFacts) []inventory.ResourceHealth {
|
||||||
|
out := make([]inventory.ResourceHealth, 0, len(rs))
|
||||||
|
for _, r := range rs {
|
||||||
|
if r.State == link.StateWaiting {
|
||||||
|
var why error
|
||||||
|
if len(r.Waits) == 0 {
|
||||||
|
why = fmt.Errorf("says it waits, and names nothing it waits for")
|
||||||
|
}
|
||||||
|
for _, w := range r.Waits {
|
||||||
|
if why == nil {
|
||||||
|
why = checkWait(r.Module, machine, w, f)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if why != nil {
|
||||||
|
r.State, r.Reason = link.StateUnhealthy, why.Error()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out = append(out, r)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// operatorWait is whether everything not healthy of a module on a machine is waiting with its waits checked
|
||||||
|
// (checkWaiting already applied), and those waits. A module with anything unhealthy, starting or unknown beside it
|
||||||
|
// does not wait: it is judged as before.
|
||||||
|
func operatorWait(module string, rs []inventory.ResourceHealth) ([]inventory.Wait, bool) {
|
||||||
|
var waits []inventory.Wait
|
||||||
|
for _, r := range rs {
|
||||||
|
if r.Module != module {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
switch r.State {
|
||||||
|
case link.StateHealthy:
|
||||||
|
case link.StateWaiting:
|
||||||
|
waits = append(waits, r.Waits...)
|
||||||
|
default:
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return waits, len(waits) > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// operatorWaitSaid is a module's wait for the operator in one sentence, for the gate's verdict and the condition's
|
||||||
|
// summary: what the operator gives and what it names, and for a secret the line that opens the desk prompt.
|
||||||
|
func operatorWaitSaid(module, machine string, waits []inventory.Wait) string {
|
||||||
|
var parts []string
|
||||||
|
for _, w := range waits {
|
||||||
|
part := fmt.Sprintf("%s (%s", w.What, waitNames(w))
|
||||||
|
if w.Secret != "" {
|
||||||
|
part += fmt.Sprintf(", given with `nox secret ask %s %s %s`", machine, module, w.Secret)
|
||||||
|
}
|
||||||
|
parts = append(parts, part+")")
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s on %s waits for the operator: %s", module, machine, strings.Join(parts, "; "))
|
||||||
|
}
|
||||||
|
|
||||||
|
// needsOperatorObservation is a module whose only parts not healthy wait for the operator (ADR 0283): the operator's,
|
||||||
|
// a warning however long it stands, its plain words naming the act and never saying there is nothing to do.
|
||||||
|
func needsOperatorObservation(module, node string, waits []inventory.Wait, rs []inventory.ResourceHealth) conditions.Observation {
|
||||||
|
o := moduleUnhealthyObservation(module, node, rs)
|
||||||
|
o.Token, o.Kind, o.Resolver, o.Severity = kindNeedsOperator, kindNeedsOperator, conditions.ResolverOperator, conditions.Warning
|
||||||
|
o.Summary = operatorWaitSaid(module, node, waits)
|
||||||
|
w := needsOperatorWords(module, node, waits)
|
||||||
|
o.Headline, o.Explanation, o.Needs, o.Resolved, o.Actions = w.Headline, w.Explanation, w.Needs, w.Resolved, nil
|
||||||
|
return o
|
||||||
|
}
|
||||||
|
|
||||||
|
// needsOperatorWords is what the operator reads of a module waiting for them (ADR 0253, ADR 0283): the act, for a
|
||||||
|
// secret typed at the machine's desk prompt and for a setting approved when an agent proposes it. The secret's and
|
||||||
|
// the setting's names, and the line, are in the summary for whoever looks closer.
|
||||||
|
func needsOperatorWords(module, node string, waits []inventory.Wait) words {
|
||||||
|
var acts []string
|
||||||
|
seen := map[string]bool{}
|
||||||
|
secret := false
|
||||||
|
for _, w := range waits {
|
||||||
|
var act string
|
||||||
|
switch {
|
||||||
|
case w.Secret != "":
|
||||||
|
act, secret = fmt.Sprintf("type %s at %s's desk prompt", w.What, node), true
|
||||||
|
case w.Setting != "":
|
||||||
|
act = fmt.Sprintf("approve %s of %s on %s when it is proposed to you", w.Setting, module, node)
|
||||||
|
}
|
||||||
|
if act != "" && !seen[act] {
|
||||||
|
seen[act] = true
|
||||||
|
acts = append(acts, act)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
needs := strings.Join(acts, "; and ") + "."
|
||||||
|
// Plain words hold one sentence of at most conditions.NeedsMax characters: several acts are named in the
|
||||||
|
// summary instead.
|
||||||
|
if len(acts) == 0 || len(needs) > conditions.NeedsMax {
|
||||||
|
needs = fmt.Sprintf("give what %s waits for on %s; the details name each secret and setting.", module, node)
|
||||||
|
}
|
||||||
|
explanation := fmt.Sprintf("Part of %s on %s cannot work until you give what it waits for.", module, node)
|
||||||
|
if secret {
|
||||||
|
explanation += " A hidden prompt opens at the desk when the secret is asked for, and what you type there " +
|
||||||
|
"is sealed to the machine."
|
||||||
|
}
|
||||||
|
explanation += " Its update is in place and nothing was undone; it carries on by itself once it is given."
|
||||||
|
return words{
|
||||||
|
Headline: fmt.Sprintf("%s waits for you on %s", module, node),
|
||||||
|
Needs: needs,
|
||||||
|
Explanation: explanation,
|
||||||
|
Resolved: fmt.Sprintf("%s on %s no longer waits for you", module, node),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// readWaitFacts reads what the controller holds to check the waits of the modules named on one machine: the
|
||||||
|
// manifests (the catalogue's, or those given) and the secrets given there. A read that fails leaves that module
|
||||||
|
// unknown, so none of its waits is excused.
|
||||||
|
func readWaitFacts(ctx context.Context, inv *inventory.Inventory, machine string, modules []string,
|
||||||
|
manifests map[string]catalogue.Manifest, f *operatorWaitFacts) {
|
||||||
|
if f.manifests == nil {
|
||||||
|
f.manifests = map[string]catalogue.Manifest{}
|
||||||
|
}
|
||||||
|
if f.given == nil {
|
||||||
|
f.given = map[string]map[string]time.Time{}
|
||||||
|
}
|
||||||
|
if inv == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var shelf map[string]catalogue.Manifest
|
||||||
|
sort.Strings(modules)
|
||||||
|
for _, module := range modules {
|
||||||
|
if _, has := f.manifests[module]; !has {
|
||||||
|
if m, given := manifests[module]; given {
|
||||||
|
f.manifests[module] = m
|
||||||
|
} else {
|
||||||
|
if shelf == nil {
|
||||||
|
var err error
|
||||||
|
if shelf, err = inv.Catalogue(ctx); err != nil {
|
||||||
|
shelf = map[string]catalogue.Manifest{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if m, known := shelf[module]; known {
|
||||||
|
f.manifests[module] = m
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, read := f.given[module+"@"+machine]; read {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if given, err := inv.GivenOwnSecrets(ctx, machine, module); err == nil {
|
||||||
|
f.given[module+"@"+machine] = given
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitingModules is every module with a waiting resource in a statement.
|
||||||
|
func waitingModules(rs []inventory.ResourceHealth) []string {
|
||||||
|
seen := map[string]bool{}
|
||||||
|
var out []string
|
||||||
|
for _, r := range rs {
|
||||||
|
if r.State == link.StateWaiting && r.Module != "" && !seen[r.Module] {
|
||||||
|
seen[r.Module] = true
|
||||||
|
out = append(out, r.Module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,264 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A part that waits for the operator's secret or setting (novox/hq ADR 0283, issue 386).
|
||||||
|
|
||||||
|
var mountsManifest = catalogue.Manifest{Module: "mounts", Version: "1",
|
||||||
|
Settings: map[string]catalogue.SettingDeclaration{"smb-users": {}, "sources": {}},
|
||||||
|
OwnSecrets: catalogue.OwnSecrets{
|
||||||
|
"smb-password-*": {Path: "/s/smb-password-*.secret", IssuedBy: catalogue.IssuedOutside},
|
||||||
|
"broker": {Path: "/s/broker"},
|
||||||
|
"made": {Path: "/s/made", Taken: catalogue.TakenAtStart},
|
||||||
|
"licence": {Path: "/s/licence", IssuedBy: catalogue.IssuedOutside},
|
||||||
|
}}
|
||||||
|
|
||||||
|
var passwordWait = inventory.Wait{Part: "the source games", Secret: "smb-password-games",
|
||||||
|
What: "the password of the source games"}
|
||||||
|
|
||||||
|
var usernameWait = inventory.Wait{Part: "the source games", Setting: "smb-users", What: "the username of the source games"}
|
||||||
|
|
||||||
|
func waitingResource(waits ...inventory.Wait) inventory.ResourceHealth {
|
||||||
|
return inventory.ResourceHealth{Module: "mounts", Resource: "mounts.watch", Kind: "process",
|
||||||
|
Target: "mesh-mounts-watch.service", State: link.StateWaiting, Check: "tool",
|
||||||
|
Reason: "the source games waits for its password", Waits: waits}
|
||||||
|
}
|
||||||
|
|
||||||
|
func factsGiven(given map[string]time.Time) operatorWaitFacts {
|
||||||
|
return operatorWaitFacts{manifests: map[string]catalogue.Manifest{"mounts": mountsManifest},
|
||||||
|
given: map[string]map[string]time.Time{"mounts@workstation": given}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rule 3: a wait is excused only when what it names is the module's, issued outside the mesh, and not given there.
|
||||||
|
func TestAWaitIsExcusedOnlyWhenItChecksOut(t *testing.T) {
|
||||||
|
at := time.Date(2026, 10, 10, 15, 8, 0, 0, time.UTC)
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
w inventory.Wait
|
||||||
|
f operatorWaitFacts
|
||||||
|
says string // "" when excused
|
||||||
|
}{
|
||||||
|
{"a member of an outside family, not given", passwordWait, factsGiven(nil), ""},
|
||||||
|
{"an outside secret by name, not given", inventory.Wait{Part: "p", Secret: "licence", What: "w"}, factsGiven(nil), ""},
|
||||||
|
{"a declared setting", usernameWait, factsGiven(nil), ""},
|
||||||
|
{"a member given", passwordWait, factsGiven(map[string]time.Time{"smb-password-games": at}), "was given at 2026-10-10 15:08"},
|
||||||
|
{"a secret not declared", inventory.Wait{Part: "p", Secret: "smb-credentials", What: "w"}, factsGiven(nil), "does not declare"},
|
||||||
|
{"a secret the mesh makes", inventory.Wait{Part: "p", Secret: "made", What: "w"}, factsGiven(nil), "mesh makes itself"},
|
||||||
|
{"the bus account", inventory.Wait{Part: "p", Secret: "broker", What: "w"}, factsGiven(nil), "mesh makes itself"},
|
||||||
|
{"a setting not declared", inventory.Wait{Part: "p", Setting: "logins", What: "w"}, factsGiven(nil), "does not declare"},
|
||||||
|
{"both", inventory.Wait{Part: "p", Secret: "licence", Setting: "smb-users", What: "w"}, factsGiven(nil), "one secret or one setting"},
|
||||||
|
{"neither", inventory.Wait{Part: "p", What: "w"}, factsGiven(nil), "one secret or one setting"},
|
||||||
|
{"no manifest known", passwordWait, operatorWaitFacts{}, "no manifest"},
|
||||||
|
{"what was given cannot be read", passwordWait,
|
||||||
|
operatorWaitFacts{manifests: map[string]catalogue.Manifest{"mounts": mountsManifest}}, "could not be read"},
|
||||||
|
} {
|
||||||
|
err := checkWait("mounts", "workstation", c.w, c.f)
|
||||||
|
switch {
|
||||||
|
case c.says == "" && err != nil:
|
||||||
|
t.Errorf("%s: refused: %v", c.name, err)
|
||||||
|
case c.says != "" && (err == nil || !strings.Contains(err.Error(), c.says)):
|
||||||
|
t.Errorf("%s: %v; want a refusal saying %q", c.name, err, c.says)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A waiting resource whose wait does not check out, or that names nothing, is judged unhealthy, saying why.
|
||||||
|
func TestAWaitThatFailsItsCheckIsUnhealthy(t *testing.T) {
|
||||||
|
given := factsGiven(map[string]time.Time{"smb-password-games": time.Now()})
|
||||||
|
got := checkWaiting("workstation", []inventory.ResourceHealth{waitingResource(passwordWait)}, given)
|
||||||
|
if got[0].State != link.StateUnhealthy || !strings.Contains(got[0].Reason, "was given") {
|
||||||
|
t.Fatalf("a wait for a secret given: %+v", got[0])
|
||||||
|
}
|
||||||
|
got = checkWaiting("workstation", []inventory.ResourceHealth{waitingResource()}, factsGiven(nil))
|
||||||
|
if got[0].State != link.StateUnhealthy || !strings.Contains(got[0].Reason, "names nothing") {
|
||||||
|
t.Fatalf("a wait naming nothing: %+v", got[0])
|
||||||
|
}
|
||||||
|
got = checkWaiting("workstation", []inventory.ResourceHealth{waitingResource(passwordWait, usernameWait)}, factsGiven(nil))
|
||||||
|
if got[0].State != link.StateWaiting {
|
||||||
|
t.Fatalf("two waits that check out: %+v", got[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rule 4: the gate passes a module whose only parts not healthy wait for the operator, carrying the wait; anything
|
||||||
|
// else beside it is judged as before; and any build is excused, not only one that added something.
|
||||||
|
func TestTheGatePassesAWaitForTheOperatorCarriedAlong(t *testing.T) {
|
||||||
|
now := time.Now()
|
||||||
|
since := now.Add(-time.Minute)
|
||||||
|
healthy := inventory.ResourceHealth{Module: "mounts", Resource: "mounts.apply", Kind: "process",
|
||||||
|
Target: "mesh-mounts-apply.service", State: link.StateHealthy}
|
||||||
|
f := gateFacts{now: now, waits: factsGiven(nil), groupsAdded: map[string]bool{"mounts": false},
|
||||||
|
health: map[string]inventory.NodeHealth{"workstation": {Node: "workstation", HeardAt: now,
|
||||||
|
Resources: []inventory.ResourceHealth{healthy, waitingResource(passwordWait)}}}}
|
||||||
|
h, why := moduleHealthWord("mounts", "workstation", since, f)
|
||||||
|
if h != healthPerson || !strings.Contains(why, "waits for the operator: the password of the source games") ||
|
||||||
|
!strings.Contains(why, "nox secret ask workstation mounts smb-password-games") {
|
||||||
|
t.Fatalf("an excused wait reads %v %q; want a wait for a person naming the act", h, why)
|
||||||
|
}
|
||||||
|
// A second resource unhealthy beside it: not yet, as before.
|
||||||
|
down := healthy
|
||||||
|
down.State, down.Reason = link.StateUnhealthy, "down"
|
||||||
|
f.health["workstation"] = inventory.NodeHealth{Node: "workstation", HeardAt: now,
|
||||||
|
Resources: []inventory.ResourceHealth{down, waitingResource(passwordWait)}}
|
||||||
|
if h, why := moduleHealthWord("mounts", "workstation", since, f); h != healthNotYet {
|
||||||
|
t.Fatalf("a resource down beside the wait reads %v %q", h, why)
|
||||||
|
}
|
||||||
|
// The password given and the module still saying it waits: not excused.
|
||||||
|
f.waits = factsGiven(map[string]time.Time{"smb-password-games": now})
|
||||||
|
f.health["workstation"] = inventory.NodeHealth{Node: "workstation", HeardAt: now,
|
||||||
|
Resources: []inventory.ResourceHealth{healthy, waitingResource(passwordWait)}}
|
||||||
|
if h, why := moduleHealthWord("mounts", "workstation", since, f); h != healthNotYet || !strings.Contains(why, "was given") {
|
||||||
|
t.Fatalf("a wait for a secret given reads %v %q", h, why)
|
||||||
|
}
|
||||||
|
// Facts never read (no manifest): never excused.
|
||||||
|
f.waits = operatorWaitFacts{}
|
||||||
|
if h, _ := moduleHealthWord("mounts", "workstation", since, f); h != healthNotYet {
|
||||||
|
t.Fatalf("a wait nothing could check reads %v", h)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func needsOperatorOpen(t *testing.T, k *conditions.Keeper) (*conditions.Condition, []conditions.Condition) {
|
||||||
|
t.Helper()
|
||||||
|
open, err := k.Open(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for i, c := range open {
|
||||||
|
if c.Key == needsOperatorKey("mounts", "workstation") {
|
||||||
|
return &open[i], open
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil, open
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rule 5: two statements of an excused wait raise needs-operator, the operator's, a warning however long, naming the
|
||||||
|
// act; a statement without it clears it.
|
||||||
|
func TestTheNeedsOperatorConditionNamesTheAct(t *testing.T) {
|
||||||
|
k, _ := withConditionsInMemory(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
rs := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}}
|
||||||
|
if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 1}, time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got, _ := needsOperatorOpen(t, k); got != nil {
|
||||||
|
t.Fatal("raised on one statement")
|
||||||
|
}
|
||||||
|
if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 2}, time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, open := needsOperatorOpen(t, k)
|
||||||
|
if got == nil {
|
||||||
|
t.Fatalf("not raised on two statements: %+v", open)
|
||||||
|
}
|
||||||
|
for _, c := range open {
|
||||||
|
if c.Kind == kindModuleUnhealthy {
|
||||||
|
t.Fatalf("raised as a fault too: %+v", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if got.Kind != kindNeedsOperator || got.Resolver != conditions.ResolverOperator || got.Severity != conditions.Warning {
|
||||||
|
t.Fatalf("the condition: %+v", got)
|
||||||
|
}
|
||||||
|
if !strings.Contains(got.Needs, "type the password of the source games at workstation's desk prompt") ||
|
||||||
|
!strings.Contains(got.Explanation, "hidden prompt opens at the desk") {
|
||||||
|
t.Fatalf("its needs do not name the act: %q", got.Needs)
|
||||||
|
}
|
||||||
|
if strings.Contains(strings.ToLower(got.Explanation), "nothing for you") || !strings.Contains(got.Explanation, "nothing was undone") {
|
||||||
|
t.Fatalf("its explanation: %q", got.Explanation)
|
||||||
|
}
|
||||||
|
if !strings.Contains(got.Summary, "smb-password-games") || !strings.Contains(got.Summary, "nox secret ask workstation mounts smb-password-games") {
|
||||||
|
t.Fatalf("its summary does not name the secret and the line: %q", got.Summary)
|
||||||
|
}
|
||||||
|
// Long open is still a warning: only the operator can end it.
|
||||||
|
if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 3}, time.Now().Add(48*time.Hour)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got, _ := needsOperatorOpen(t, k); got == nil || got.Severity == conditions.Urgent {
|
||||||
|
t.Fatalf("after two days: %+v", got)
|
||||||
|
}
|
||||||
|
// Given: the next statement does not say it, and it clears.
|
||||||
|
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got, _ := needsOperatorOpen(t, k); got != nil {
|
||||||
|
t.Fatal("not cleared once given")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestASettingsWaitAsksForTheApproval(t *testing.T) {
|
||||||
|
k, _ := withConditionsInMemory(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
rs := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(usernameWait)}}
|
||||||
|
for i := 1; i <= 2; i++ {
|
||||||
|
if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": i}, time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
got, _ := needsOperatorOpen(t, k)
|
||||||
|
if got == nil || !strings.Contains(got.Needs, "approve smb-users of mounts on workstation when it is proposed to you") {
|
||||||
|
t.Fatalf("the condition: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A wait that fails its check is the module's own fault: unhealthy, with why, and no needs-operator.
|
||||||
|
func TestAWaitThatFailsItsCheckRaisesUnhealthy(t *testing.T) {
|
||||||
|
k, _ := withConditionsInMemory(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
checked := checkWaiting("workstation", []inventory.ResourceHealth{waitingResource(passwordWait)},
|
||||||
|
factsGiven(map[string]time.Time{"smb-password-games": time.Now()}))
|
||||||
|
rs := map[string][]inventory.ResourceHealth{"mounts": checked}
|
||||||
|
for i := 1; i <= 2; i++ {
|
||||||
|
if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": i}, time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
got, open := needsOperatorOpen(t, k)
|
||||||
|
if got != nil {
|
||||||
|
t.Fatalf("a wait for a secret given raised needs-operator: %+v", got)
|
||||||
|
}
|
||||||
|
unhealthy := false
|
||||||
|
for _, c := range open {
|
||||||
|
unhealthy = unhealthy || c.Key == moduleUnhealthyKey("mounts", "workstation")
|
||||||
|
}
|
||||||
|
if !unhealthy {
|
||||||
|
t.Fatalf("not raised as unhealthy: %+v", open)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A module that waited and is then broken says so when the wait clears, and the other way round.
|
||||||
|
func TestANeedsOperatorThatBecameUnhealthySaysSo(t *testing.T) {
|
||||||
|
k, _ := withConditionsInMemory(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
waiting := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}}
|
||||||
|
for i := 1; i <= 2; i++ {
|
||||||
|
if err := judgeModuleHealth(ctx, nil, k, "workstation", waiting, map[string]int{"mounts": i}, time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
broken := waitingResource()
|
||||||
|
broken.State, broken.Reason, broken.Waits = link.StateUnhealthy, "the source games refused its login", nil
|
||||||
|
for i := 3; i <= 4; i++ {
|
||||||
|
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": {broken}},
|
||||||
|
map[string]int{"mounts": i}, time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
got, open := needsOperatorOpen(t, k)
|
||||||
|
if got != nil {
|
||||||
|
t.Fatal("needs-operator still open after it became a fault")
|
||||||
|
}
|
||||||
|
found := false
|
||||||
|
for _, c := range open {
|
||||||
|
found = found || c.Key == moduleUnhealthyKey("mounts", "workstation")
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatalf("the fault is not raised: %+v", open)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -157,7 +157,7 @@ func TestAMergeRebuildsTheModulesItChanged(t *testing.T) {
|
|||||||
{"a file directly among the modules", merge([]string{"modules/README.md"}, false), ""},
|
{"a file directly among the modules", merge([]string{"modules/README.md"}, false), ""},
|
||||||
{"a root file beside a module's", merge([]string{"merge-check.sh", "modules/keycloak/x.ts"}, false), "keycloak"},
|
{"a root file beside a module's", merge([]string{"merge-check.sh", "modules/keycloak/x.ts"}, false), "keycloak"},
|
||||||
} {
|
} {
|
||||||
if got := named(whatTheMergeTouched(candidates, known, c.m)); got != c.want {
|
if got := named(whatTheMergeTouched(candidates, known, c.m, nil)); got != c.want {
|
||||||
t.Errorf("%s: rebuilt %q, wanted %q", c.what, got, c.want)
|
t.Errorf("%s: rebuilt %q, wanted %q", c.what, got, c.want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -285,7 +285,7 @@ func TestAChangeInsideAModuleIsThatModulesHeldOrNot(t *testing.T) {
|
|||||||
{"an old announcer saying nothing", merge(showcase, nil, false), ""},
|
{"an old announcer saying nothing", merge(showcase, nil, false), ""},
|
||||||
{"a manifest the merge removed, said or not", merge([]string{"modules/gone/module.json", "modules/gone/x.ts"}, nil, true), ""},
|
{"a manifest the merge removed, said or not", merge([]string{"modules/gone/module.json", "modules/gone/x.ts"}, nil, true), ""},
|
||||||
} {
|
} {
|
||||||
if got := named(whatTheMergeTouched(candidates, known, c.m)); got != c.want {
|
if got := named(whatTheMergeTouched(candidates, known, c.m, nil)); got != c.want {
|
||||||
t.Errorf("%s: rebuilt %q, wanted %q", c.what, got, c.want)
|
t.Errorf("%s: rebuilt %q, wanted %q", c.what, got, c.want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,124 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A delivery over its budget is loud (novox/hq ADR 0282 decision 7, issue 382): the self-check reads
|
||||||
|
// mesh-delivery's `times` and raises the warning `delivery.<class>.over-budget` when the newest delivery of a
|
||||||
|
// class whose delivery time is known took longer than its class's budget — five minutes for a leaf module, ten
|
||||||
|
// for a core module — naming the delivery and its longest phase. It clears when the next delivery of that class
|
||||||
|
// lands within its budget. Measurement only: the condition says, it never holds or acts on a delivery.
|
||||||
|
|
||||||
|
// probeBudgetsID is the probe that reads the delivery times.
|
||||||
|
const probeBudgetsID = "D16"
|
||||||
|
|
||||||
|
// kindOverBudget is what a delivery over its class's budget raises.
|
||||||
|
const kindOverBudget = "over-budget"
|
||||||
|
|
||||||
|
// deliveryBudgets are the budgets by class (ADR 0282 decision 1); the probe raises nothing for another class.
|
||||||
|
var deliveryBudgets = map[string]time.Duration{inventory.ClassLeaf: 5 * time.Minute, inventory.ClassCore: 10 * time.Minute}
|
||||||
|
|
||||||
|
// timesAnswer is what mesh-delivery's `times` answers, as far as the probe reads it.
|
||||||
|
type timesAnswer struct {
|
||||||
|
Classes []timesClass `json:"classes"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// timesClass is one class's line of `times`.
|
||||||
|
type timesClass struct {
|
||||||
|
Class string `json:"class"`
|
||||||
|
Latest *timesLatest `json:"latest,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// timesLatest is the newest delivery of a class whose delivery time is known.
|
||||||
|
type timesLatest struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
TookMS int64 `json:"took_ms"`
|
||||||
|
Longest string `json:"longest,omitempty"`
|
||||||
|
Landed string `json:"landed,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// deliveryTimes is what the delivery's owner says of its delivery times; nothing when no holder is on record or
|
||||||
|
// none answers (D3 says that one).
|
||||||
|
func deliveryTimes(ctx context.Context, conn *nats.Conn, held bool) (*timesAnswer, error) {
|
||||||
|
if !held {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
raw, err := askDeliveryOwner(ctx, conn, "times", map[string]any{})
|
||||||
|
if errors.Is(err, link.ErrNothingServes) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var a timesAnswer
|
||||||
|
if err := json.Unmarshal(raw, &a); err != nil {
|
||||||
|
return nil, fmt.Errorf("%s.times answered something unreadable: %w", catalogue.DeliverySeat, err)
|
||||||
|
}
|
||||||
|
return &a, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// overBudgetObservations are the conditions of the classes whose newest delivery took longer than its budget:
|
||||||
|
// strictly longer, so a delivery of exactly its budget is within it.
|
||||||
|
func overBudgetObservations(a *timesAnswer) []conditions.Observation {
|
||||||
|
if a == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, c := range a.Classes {
|
||||||
|
budget, ok := deliveryBudgets[c.Class]
|
||||||
|
if !ok || c.Latest == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
took := time.Duration(c.Latest.TookMS) * time.Millisecond
|
||||||
|
if took <= budget {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
longest := c.Latest.Longest
|
||||||
|
if longest == "" {
|
||||||
|
longest = "not known"
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeDelivery, ID: c.Class, Kind: kindOverBudget,
|
||||||
|
Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("the %s delivery %s took %s from its merge to running everywhere, over its budget of %s; "+
|
||||||
|
"its longest phase: %s — `mesh-delivery.times`", c.Class, c.Latest.ID, humanDuration(took),
|
||||||
|
humanDuration(budget), longest),
|
||||||
|
Said: fmt.Sprintf("%s took %s (budget %s), longest phase %s", c.Latest.ID, took.Round(time.Second), budget, longest),
|
||||||
|
Headline: fmt.Sprintf("A %s delivery took %s, over its %s budget", c.Class, humanDuration(took),
|
||||||
|
humanDuration(budget)),
|
||||||
|
Explanation: fmt.Sprintf("The delivery %s took %s from its merge until every machine ran it; a %s module is "+
|
||||||
|
"held to %s (ADR 0282). Most of the time went to %s. Nothing was held or changed because of this: it is "+
|
||||||
|
"a measurement.", c.Latest.ID, humanDuration(took), c.Class, humanDuration(budget), longest),
|
||||||
|
Resolved: fmt.Sprintf("the next %s delivery lands within %s", c.Class, humanDuration(budget)),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeBudgets is D16: the newest delivery of each class lands within its class's budget.
|
||||||
|
func probeBudgets(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
entries, err := d.open.inventory.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var conn *nats.Conn
|
||||||
|
if d.js != nil {
|
||||||
|
conn = d.js.Conn()
|
||||||
|
}
|
||||||
|
a, err := deliveryTimes(ctx, conn, deliverySeatHeld(entries))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return overBudgetObservations(a), nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,111 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A leaf delivery of 5 minutes 10 seconds raises delivery.leaf.over-budget naming its longest phase; one of
|
||||||
|
// exactly five minutes, a core one of nine and a class without a budget raise nothing (ADR 0282 decision 7).
|
||||||
|
func TestADeliveryOverItsBudgetIsLoud(t *testing.T) {
|
||||||
|
a := ×Answer{Classes: []timesClass{
|
||||||
|
{Class: inventory.ClassLeaf, Latest: ×Latest{ID: "novox/mesh-catalog@abc", TookMS: (5*time.Minute + 10*time.Second).Milliseconds(),
|
||||||
|
Longest: "judgement 2m40s"}},
|
||||||
|
{Class: inventory.ClassCore, Latest: ×Latest{ID: "novox/mesh-controller@def", TookMS: (9 * time.Minute).Milliseconds(),
|
||||||
|
Longest: "build 1m"}},
|
||||||
|
{Class: "unclassed", Latest: ×Latest{ID: "x@y", TookMS: time.Hour.Milliseconds()}},
|
||||||
|
}}
|
||||||
|
obs := overBudgetObservations(a)
|
||||||
|
if len(obs) != 1 {
|
||||||
|
t.Fatalf("one class over its budget, got %d: %+v", len(obs), obs)
|
||||||
|
}
|
||||||
|
o := obs[0]
|
||||||
|
if o.Key() != "delivery.leaf.over-budget" || !strings.Contains(o.Summary, "judgement 2m40s") ||
|
||||||
|
!strings.Contains(o.Summary, "novox/mesh-catalog@abc") {
|
||||||
|
t.Fatalf("the condition names its delivery and longest phase: %s — %s", o.Key(), o.Summary)
|
||||||
|
}
|
||||||
|
// The next leaf delivery within its budget clears it: the probe raises nothing for the class.
|
||||||
|
a.Classes[0].Latest = ×Latest{ID: "novox/mesh-catalog@ghi", TookMS: (5 * time.Minute).Milliseconds()}
|
||||||
|
if obs := overBudgetObservations(a); len(obs) != 0 {
|
||||||
|
t.Fatalf("a delivery of exactly its budget is within it: %+v", obs)
|
||||||
|
}
|
||||||
|
a.Classes[1].Latest.TookMS = (10*time.Minute + time.Second).Milliseconds()
|
||||||
|
if obs := overBudgetObservations(a); len(obs) != 1 || obs[0].Key() != "delivery.core.over-budget" {
|
||||||
|
t.Fatalf("a core delivery over ten minutes: %+v", obs)
|
||||||
|
}
|
||||||
|
if obs := overBudgetObservations(nil); obs != nil {
|
||||||
|
t.Fatal("no answer raises nothing")
|
||||||
|
}
|
||||||
|
// No delivery of a class with a known time yet: nothing said of it.
|
||||||
|
if obs := overBudgetObservations(×Answer{Classes: []timesClass{{Class: inventory.ClassLeaf}}}); len(obs) != 0 {
|
||||||
|
t.Fatalf("a class with no delivery: %+v", obs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The probe's question is one the controller's grant names: a question the bus refuses checks nothing.
|
||||||
|
func TestTheControllerMayAskForTheDeliveryTimes(t *testing.T) {
|
||||||
|
found := false
|
||||||
|
for _, v := range broker.VerbsTheControllerAsksTheDeliveryOwner {
|
||||||
|
found = found || (v.Seat == catalogue.DeliverySeat && v.Verb == "times")
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatal("the grant does not name mesh-delivery.times")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A walk's class is core when it walks a module of the controller's own path or one holding the mesh's resolver,
|
||||||
|
// leaf otherwise; its window closed at its batch's window, or its maximum, never after its cut.
|
||||||
|
func TestAWalksClassAndWindowAreReadAtItsCut(t *testing.T) {
|
||||||
|
entries := []inventory.Entry{
|
||||||
|
{Manifest: catalogue.Manifest{Module: "dnsmasq", Claims: []catalogue.Claim{{Name: resolverSeat}}}},
|
||||||
|
{Manifest: catalogue.Manifest{Module: "gitea"}},
|
||||||
|
}
|
||||||
|
walk := func(modules ...string) inventory.Plan {
|
||||||
|
p := inventory.Plan{Modules: map[string]*inventory.PlanModule{}}
|
||||||
|
for _, m := range modules {
|
||||||
|
p.Modules[m] = &inventory.PlanModule{}
|
||||||
|
}
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
for want, w := range map[string]inventory.Plan{
|
||||||
|
inventory.ClassLeaf: walk("gitea"), inventory.ClassCore: walk("gitea", "mesh-controller"),
|
||||||
|
} {
|
||||||
|
if got := classOf(w, entries); got != want {
|
||||||
|
t.Errorf("%v: %s, want %s", w.Modules, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if got := classOf(walk("dnsmasq"), entries); got != inventory.ClassCore {
|
||||||
|
t.Errorf("the resolver's holder is core: %s", got)
|
||||||
|
}
|
||||||
|
now := time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC)
|
||||||
|
batch := inventory.Plan{Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{
|
||||||
|
ClosesAt: now.Add(-20 * time.Second), AtMost: now.Add(5 * time.Minute)}}}
|
||||||
|
times := walkTimesAtCut(batch, walk("gitea"), entries, now)
|
||||||
|
if times.Cut == nil || !times.Cut.Equal(now) || times.WindowClosed == nil || !times.WindowClosed.Equal(now.Add(-20*time.Second)) ||
|
||||||
|
times.Class != inventory.ClassLeaf {
|
||||||
|
t.Fatalf("times at the cut: %+v", times)
|
||||||
|
}
|
||||||
|
batch.Delivery.Batch.AtMost = now.Add(-time.Minute)
|
||||||
|
if times := walkTimesAtCut(batch, walk("gitea"), entries, now); !times.WindowClosed.Equal(now.Add(-time.Minute)) {
|
||||||
|
t.Fatalf("a window closed at its maximum: %v", times.WindowClosed)
|
||||||
|
}
|
||||||
|
if times := walkTimesAtCut(inventory.Plan{Delivery: &inventory.PlanDelivery{Alone: true}}, walk("gitea"), entries, now); times.WindowClosed != nil {
|
||||||
|
t.Fatalf("a merge walked alone had no window: %v", times.WindowClosed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// What each machine of the rest was sent is kept only for the machines the send reached.
|
||||||
|
func TestTheRestIsKeptForTheMachinesTheSendReached(t *testing.T) {
|
||||||
|
got := restOf([]string{"ace", "g14"}, []string{"ace", "shanks"}, map[string]inventory.SentDeclaration{"ace": {Digest: "d1"}})
|
||||||
|
if len(got) != 1 || got["ace"].Digest != "d1" {
|
||||||
|
t.Fatalf("rest: %+v", got)
|
||||||
|
}
|
||||||
|
if restOf([]string{"g14"}, []string{"ace"}, nil) != nil {
|
||||||
|
t.Fatal("no machine reached: nothing kept")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -221,6 +221,16 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
|||||||
w := usedAsFoundObservation(orModule(module), machineOr(o, "a machine"), o.Summary, nil)
|
w := usedAsFoundObservation(orModule(module), machineOr(o, "a machine"), o.Summary, nil)
|
||||||
return words{Headline: w.Headline, Explanation: w.Explanation, Needs: w.Needs, Resolved: w.Resolved}
|
return words{Headline: w.Headline, Explanation: w.Explanation, Needs: w.Needs, Resolved: w.Resolved}
|
||||||
}),
|
}),
|
||||||
|
kindNeedsOperator: worded(func(o conditions.Observation) words {
|
||||||
|
// The observation carries the act itself (ADR 0283); these are its words when only the kind is known.
|
||||||
|
module := ""
|
||||||
|
if o.Scope == conditions.ScopeModule && o.Machine != "" {
|
||||||
|
module = strings.TrimSuffix(o.ID, "."+o.Machine)
|
||||||
|
}
|
||||||
|
node := machineOr(o, "a machine")
|
||||||
|
w := needsOperatorWords(orModule(module), node, nil)
|
||||||
|
return w
|
||||||
|
}),
|
||||||
kindProviderFailing: worded(func(o conditions.Observation) words {
|
kindProviderFailing: worded(func(o conditions.Observation) words {
|
||||||
thing, consumer := conditions.ThingWords(o), idPart(o, 2)
|
thing, consumer := conditions.ThingWords(o), idPart(o, 2)
|
||||||
if consumer == "" {
|
if consumer == "" {
|
||||||
@@ -362,6 +372,17 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
|||||||
Explanation: "Its walk across the machines has not moved for longer than usual. Nothing is lost.",
|
Explanation: "Its walk across the machines has not moved for longer than usual. Nothing is lost.",
|
||||||
Resolved: "Resolved: the delivery moves again"}
|
Resolved: "Resolved: the delivery moves again"}
|
||||||
}),
|
}),
|
||||||
|
kindBatchNotCut: worded(func(o conditions.Observation) words {
|
||||||
|
return words{Headline: "Merged changes are not being delivered",
|
||||||
|
Explanation: "Merges collected for one delivery should have been planned and were not. Nothing is lost.",
|
||||||
|
Resolved: "Resolved: the merged changes are being delivered"}
|
||||||
|
}),
|
||||||
|
kindBatchBehindWalk: worded(func(o conditions.Observation) words {
|
||||||
|
return words{Headline: "Merged changes wait behind a slow delivery",
|
||||||
|
Explanation: "Merges collected for the next delivery wait for the delivery before them, which is taking " +
|
||||||
|
"longer than it should. Nothing is lost.",
|
||||||
|
Resolved: "Resolved: the merged changes no longer wait"}
|
||||||
|
}),
|
||||||
kindWalkWaiting: worded(func(o conditions.Observation) words {
|
kindWalkWaiting: worded(func(o conditions.Observation) words {
|
||||||
return words{Headline: "A delivery is waiting to start",
|
return words{Headline: "A delivery is waiting to start",
|
||||||
Explanation: "A merged change is built, and mesh-delivery (the module that decides when a delivery goes " +
|
Explanation: "A merged change is built, and mesh-delivery (the module that decides when a delivery goes " +
|
||||||
@@ -585,6 +606,19 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
|||||||
Explanation: "The bus refused messages from a part of the mesh, so what they carried did not happen.",
|
Explanation: "The bus refused messages from a part of the mesh, so what they carried did not happen.",
|
||||||
Resolved: "Resolved: the bus takes the messages again"}
|
Resolved: "Resolved: the bus takes the messages again"}
|
||||||
}),
|
}),
|
||||||
|
kindBucketOrLogFilling: worded(func(o conditions.Observation) words {
|
||||||
|
return words{Headline: "A module's bucket or log on the bus is filling up",
|
||||||
|
Needs: "decide whether to raise its cap or have the module keep less.",
|
||||||
|
Explanation: "A bucket or log a module keeps on the bus holds three quarters of its cap or more. When it " +
|
||||||
|
"is full, the bus refuses what the module writes there.",
|
||||||
|
Resolved: "It has room again"}
|
||||||
|
}),
|
||||||
|
kindBucketOrLogUnread: worded(func(o conditions.Observation) words {
|
||||||
|
return words{Headline: "A module's bucket or log could not be read",
|
||||||
|
Explanation: "The controller could not read how full a bucket or log a module keeps on the bus is, so it " +
|
||||||
|
"cannot say whether it is filling up. It asks again at its next check.",
|
||||||
|
Resolved: "It can be read again"}
|
||||||
|
}),
|
||||||
"stream-wrong": worded(func(o conditions.Observation) words {
|
"stream-wrong": worded(func(o conditions.Observation) words {
|
||||||
return words{Headline: "Part of the bus's storage is wrong",
|
return words{Headline: "Part of the bus's storage is wrong",
|
||||||
Needs: "check the machine the bus runs on; the details say what is missing.",
|
Needs: "check the machine the bus runs on; the details say what is missing.",
|
||||||
@@ -807,6 +841,9 @@ func stalledWords(l stalledLine, o conditions.Observation) (headline, explanatio
|
|||||||
"to check it: the forge never announced it. It cannot merge until it is checked.", pull, long),
|
"to check it: the forge never announced it. It cannot merge until it is checked.", pull, long),
|
||||||
fmt.Sprintf("%s has a merge check now, or is closed", pull), needs, nil
|
fmt.Sprintf("%s has a merge check now, or is closed", pull), needs, nil
|
||||||
}
|
}
|
||||||
|
if l.State == "unanswered" {
|
||||||
|
return unansweredWords(l, o)
|
||||||
|
}
|
||||||
held := l.State
|
held := l.State
|
||||||
if held == "" {
|
if held == "" {
|
||||||
held = "held"
|
held = "held"
|
||||||
@@ -836,6 +873,74 @@ func stalledWords(l stalledLine, o conditions.Observation) (headline, explanatio
|
|||||||
fmt.Sprintf("Delivery of %s is no longer %s", name, held), needs, actions
|
fmt.Sprintf("Delivery of %s is no longer %s", name, held), needs, actions
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// unansweredWords are the plain words of a pull request's head whose merge check was asked and has not answered
|
||||||
|
// within its bound (novox/hq issue 438): mesh-delivery holds no delivery of it, so there is nothing to stop, release
|
||||||
|
// or close, and no action is offered. The operator's acts are a new commit, which asks the check again, or a look
|
||||||
|
// at the build queue, where a check that never ran may still wait.
|
||||||
|
func unansweredWords(l stalledLine, o conditions.Observation) (headline, explanation, resolved, needs string,
|
||||||
|
actions []conditions.Action) {
|
||||||
|
repository, _, _ := strings.Cut(l.ID, "@")
|
||||||
|
pull := "A pull request of " + repoName(repository)
|
||||||
|
if l.Number > 0 {
|
||||||
|
pull = fmt.Sprintf("Pull request %s #%d", repoName(repository), l.Number)
|
||||||
|
}
|
||||||
|
long, limit := "for too long", ""
|
||||||
|
if d, err := time.ParseDuration(l.For); err == nil {
|
||||||
|
long = "for " + humanDuration(d)
|
||||||
|
}
|
||||||
|
if d, err := time.ParseDuration(l.Bound); err == nil {
|
||||||
|
limit = ", past its limit of " + humanDuration(d)
|
||||||
|
}
|
||||||
|
which := "Its merge check"
|
||||||
|
if said := uncheckedWaitWords(l); len(said) > 0 {
|
||||||
|
which = "Its merge check (" + strings.Join(said, ", ") + ")"
|
||||||
|
}
|
||||||
|
if o.Resolver == conditions.ResolverOperator {
|
||||||
|
needs = "push a new commit to its branch, which asks the check again, or see whether its check still waits " +
|
||||||
|
"in the build queue: mesh-controller.queue."
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s's merge check has not answered %s", pull, long),
|
||||||
|
fmt.Sprintf("%s is open on a branch that requires the merge check. %s was asked and has not answered %s%s. "+
|
||||||
|
"It cannot merge until its check answers.", pull, which, long, limit),
|
||||||
|
fmt.Sprintf("%s has an answer from its merge check, or is closed", pull), needs, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// uncheckedWaitWords are the merge checks an unanswered line waits on, as the operator reads them: from the checks
|
||||||
|
// given as data, each time in the controller's local zone, as every other time in a message — "mesh/merge-gate
|
||||||
|
// pending since 02:11" — never the UTC time inside mesh-delivery's finished words, which cannot be said again in
|
||||||
|
// another zone (novox/hq issue 443). A line from a mesh-delivery that gives no such data is said by its words.
|
||||||
|
func uncheckedWaitWords(l stalledLine) []string {
|
||||||
|
if len(l.Checks) == 0 {
|
||||||
|
return l.Waiting
|
||||||
|
}
|
||||||
|
out := make([]string, 0, len(l.Checks))
|
||||||
|
for _, c := range l.Checks {
|
||||||
|
switch c.State {
|
||||||
|
case "never-set":
|
||||||
|
out = append(out, c.Context+" never set")
|
||||||
|
case "pending":
|
||||||
|
out = append(out, c.Context+" pending"+sinceWords(c.Since))
|
||||||
|
default:
|
||||||
|
out = append(out, c.Context+" not answered")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// sinceWords is " since 02:11" in the controller's local zone, with its day when that is not today ("since 23:05 on
|
||||||
|
// 9 Oct"), so the time stays absolute; a time not given, or not readable, is said so and never made up.
|
||||||
|
func sinceWords(since string) string {
|
||||||
|
at, err := time.Parse(time.RFC3339, since)
|
||||||
|
if err != nil {
|
||||||
|
return ", since a time the forge did not say"
|
||||||
|
}
|
||||||
|
local, today := at.In(wordsZone()), wordsNow().In(wordsZone())
|
||||||
|
if local.YearDay() == today.YearDay() && local.Year() == today.Year() {
|
||||||
|
return " since " + local.Format("15:04")
|
||||||
|
}
|
||||||
|
return " since " + local.Format("15:04 on 2 Jan")
|
||||||
|
}
|
||||||
|
|
||||||
// causeWords is a hand-act's cause as a person says it.
|
// causeWords is a hand-act's cause as a person says it.
|
||||||
func causeWords(cause string) string {
|
func causeWords(cause string) string {
|
||||||
return strings.NewReplacer(".", " ", "_", " ").Replace(cause)
|
return strings.NewReplacer(".", " ", "_", " ").Replace(cause)
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/json"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -347,3 +348,107 @@ func TestAnUnannouncedPullRequestSaysToPushANewCommit(t *testing.T) {
|
|||||||
t.Fatalf("it reads %q / %q", o.Headline, o.Explanation)
|
t.Fatalf("it reads %q / %q", o.Headline, o.Explanation)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **A pull request whose merge check never answered says which check, since when, and what to do** (novox/hq issue
|
||||||
|
// 438): mesh-delivery says one as a stalled line in state `unanswered`, the line below exactly as its test makes it.
|
||||||
|
// No delivery of the head is held, so there is nothing to stop, release or close: no action is offered, and the
|
||||||
|
// operator's acts are a new commit, which asks the check again, or a look at the build queue.
|
||||||
|
func TestAnUnansweredMergeCheckSaysWhichCheckAndWhatToDo(t *testing.T) {
|
||||||
|
var l stalledLine
|
||||||
|
if err := json.Unmarshal([]byte(`{"id":"novox/mesh-controller@c11222026a3b","number":212,"state":"unanswered",`+
|
||||||
|
`"waiting":["mesh/merge-gate pending since 2026-10-11T00:11:39Z","mesh/repo-check never set"],"for":"1h1m0s",`+
|
||||||
|
`"bound":"1h0m0s","h2":"none: no delivery of it is held here, so there is none to close — the operator's",`+
|
||||||
|
`"says":"novox/mesh-controller#212 is open on main, which requires the merge check, and its head's check `+
|
||||||
|
`started and never answered: mesh/merge-gate pending since 2026-10-11T00:11:39Z, mesh/repo-check never set. `+
|
||||||
|
`No delivery of it is held here, so nothing asks it again. A new commit on its branch announces it and asks `+
|
||||||
|
`its check"}`), &l); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
obs := stalledObservations([]stalledLine{l})
|
||||||
|
if len(obs) != 1 || obs[0].Resolver != conditions.ResolverOperator {
|
||||||
|
t.Fatalf("an unanswered merge check is not the operator's: %+v", obs)
|
||||||
|
}
|
||||||
|
o := obs[0]
|
||||||
|
t.Logf("headline: %s\nexplanation: %s\nneeds: %s\nresolved: %s", o.Headline, o.Explanation, o.Needs, o.Resolved)
|
||||||
|
if len(o.Actions) != 0 {
|
||||||
|
t.Fatalf("it offers an action a head with no delivery cannot take: %+v", o.Actions)
|
||||||
|
}
|
||||||
|
if strings.Contains(o.Needs, "stop") || strings.Contains(o.Needs, "release") || !strings.Contains(o.Needs, "commit") ||
|
||||||
|
!strings.Contains(o.Needs, "in the build queue: mesh-controller.queue") || strings.Contains(o.Needs, "build seat") {
|
||||||
|
t.Fatalf("it says to %q", o.Needs)
|
||||||
|
}
|
||||||
|
if o.Headline != "Pull request mesh-controller #212's merge check has not answered for 61 minutes" {
|
||||||
|
t.Fatalf("its headline reads %q", o.Headline)
|
||||||
|
}
|
||||||
|
for _, want := range []string{"mesh/merge-gate pending since 2026-10-11T00:11:39Z", "mesh/repo-check never set",
|
||||||
|
"past its limit of 60 minutes"} {
|
||||||
|
if !strings.Contains(o.Explanation, want) {
|
||||||
|
t.Fatalf("its explanation does not say %q: %q", want, o.Explanation)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(o.Explanation, "never asked") || strings.Contains(o.Explanation, "never announced") {
|
||||||
|
t.Fatalf("it says the mesh was never asked, of a head whose check started: %q", o.Explanation)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A line from a mesh-delivery that names no checks still says the check did not answer, and offers nothing.
|
||||||
|
var bare stalledLine
|
||||||
|
if err := json.Unmarshal([]byte(`{"id":"novox/mesh-controller@c11222026a3b","number":212,"state":"unanswered",`+
|
||||||
|
`"for":"1h1m0s","bound":"1h0m0s","h2":"none: no delivery of it is held here, so there is none to close — the operator's"}`),
|
||||||
|
&bare); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
o = stalledObservations([]stalledLine{bare})[0]
|
||||||
|
if len(o.Actions) != 0 || !strings.Contains(o.Explanation, "has not answered") || strings.Contains(o.Headline, "Delivery of") {
|
||||||
|
t.Fatalf("a line naming no checks reads %q / %q, actions %+v", o.Headline, o.Explanation, o.Actions)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **An unanswered merge check's time reaches the operator in their own time, never as raw UTC** (novox/hq issue
|
||||||
|
// 443): mesh-delivery says each check waited on as data — its context, its state and since when, in RFC 3339 — beside
|
||||||
|
// the finished words it gave before, and the controller words it in its local zone, as every other time in a message.
|
||||||
|
// The line is the one mesh-delivery says, as in the report of issue 443, with the checks it now carries.
|
||||||
|
func TestAnUnansweredMergeChecksTimeIsSaidInLocalTime(t *testing.T) {
|
||||||
|
// The operator's zone given through the seam, never by writing time.Local: other tests' goroutines read it, and
|
||||||
|
// the build seat runs the suite under the race detector.
|
||||||
|
wasZone, wasNow := wordsZone, wordsNow
|
||||||
|
wordsZone = func() *time.Location { return time.FixedZone("CEST", 2*60*60) }
|
||||||
|
wordsNow = func() time.Time { return time.Date(2026, 10, 11, 1, 12, 39, 0, time.UTC) }
|
||||||
|
t.Cleanup(func() { wordsZone, wordsNow = wasZone, wasNow })
|
||||||
|
|
||||||
|
var l stalledLine
|
||||||
|
if err := json.Unmarshal([]byte(`{"id":"novox/mesh-controller@c11222026a3b","number":212,"state":"unanswered",`+
|
||||||
|
`"waiting":["mesh/merge-gate pending since 2026-10-11T00:11:39Z","mesh/repo-check never set"],`+
|
||||||
|
`"checks":[{"context":"mesh/merge-gate","state":"pending","since":"2026-10-11T00:11:39Z"},`+
|
||||||
|
`{"context":"mesh/repo-check","state":"never-set"}],"for":"1h1m0s",`+
|
||||||
|
`"bound":"1h0m0s","h2":"none: no delivery of it is held here, so there is none to close — the operator's",`+
|
||||||
|
`"says":"novox/mesh-controller#212 is open on main, which requires the merge check, and its head's check `+
|
||||||
|
`started and never answered: mesh/merge-gate pending since 2026-10-11T00:11:39Z, mesh/repo-check never set. `+
|
||||||
|
`No delivery of it is held here, so nothing asks it again. A new commit on its branch announces it and asks `+
|
||||||
|
`its check"}`), &l); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
o := stalledObservations([]stalledLine{l})[0]
|
||||||
|
t.Logf("explanation: %s", o.Explanation)
|
||||||
|
for _, raw := range []string{"2026-10-11T00:11:39Z", "00:11", "UTC"} {
|
||||||
|
if strings.Contains(o.Explanation, raw) || strings.Contains(o.Headline, raw) {
|
||||||
|
t.Fatalf("the operator reads %q: %q / %q", raw, o.Headline, o.Explanation)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, want := range []string{"mesh/merge-gate pending since 02:11", "mesh/repo-check never set"} {
|
||||||
|
if !strings.Contains(o.Explanation, want) {
|
||||||
|
t.Fatalf("its explanation does not say %q: %q", want, o.Explanation)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A check pending since another day than today says which day, so the time stays absolute.
|
||||||
|
l.Checks[0].Since = "2026-10-09T21:05:00Z"
|
||||||
|
if o = stalledObservations([]stalledLine{l})[0]; !strings.Contains(o.Explanation, "mesh/merge-gate pending since 23:05 on 9 Oct") {
|
||||||
|
t.Fatalf("a check pending since an earlier day reads %q", o.Explanation)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A pending check whose time the forge did not say is said so, with no time made up.
|
||||||
|
l.Checks[0].Since = ""
|
||||||
|
if o = stalledObservations([]stalledLine{l})[0]; !strings.Contains(o.Explanation, "mesh/merge-gate pending, since a time the forge did not say") {
|
||||||
|
t.Fatalf("a pending check without its time reads %q", o.Explanation)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -420,7 +420,8 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
|||||||
out := sendable{Resources: composed.Resources, Adoption: adoption,
|
out := sendable{Resources: composed.Resources, Adoption: adoption,
|
||||||
Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
|
Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
|
||||||
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld,
|
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld,
|
||||||
unbound: with.Unbound, foreseen: composed.Foreseen, unplaced: composed.Unplaced}
|
unbound: with.Unbound, foreseen: composed.Foreseen, unplaced: composed.Unplaced,
|
||||||
|
modules: namedModules(plan, composed.LeftOut)}
|
||||||
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
|
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
|
||||||
// 0221). Read only on the send path: a question about what would be sent records nothing.
|
// 0221). Read only on the send path: a question about what would be sent records nothing.
|
||||||
if choosing == Allocating {
|
if choosing == Allocating {
|
||||||
@@ -664,7 +665,33 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
|||||||
needed := map[string]map[string]string{}
|
needed := map[string]map[string]string{}
|
||||||
foreseen := map[string]map[string]bool{}
|
foreseen := map[string]map[string]bool{}
|
||||||
for _, m := range plan.Modules {
|
for _, m := range plan.Modules {
|
||||||
for name := range m.OwnSecrets {
|
// **A secret family is never made** (novox/hq ADR 0283): each member a person gave on this machine is
|
||||||
|
// placed, and one not given is nothing — the module says it waits for it.
|
||||||
|
for _, family := range m.OwnSecrets.Families() {
|
||||||
|
members, err := inv.GivenMembers(ctx, node, m.Module, family)
|
||||||
|
if err != nil {
|
||||||
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
|
}
|
||||||
|
for _, g := range members {
|
||||||
|
if _, fam, ok := m.OwnSecrets.Lookup(g.Name); !ok || fam != family {
|
||||||
|
continue // a longer family's member, or a name no longer of this family
|
||||||
|
}
|
||||||
|
if !g.Current {
|
||||||
|
if choosing == Allocating {
|
||||||
|
return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf(
|
||||||
|
"%s on %s holds %q, which was given to the mesh rather than made by it, and %s has "+
|
||||||
|
"since generated a new sealing key. The mesh cannot make another; give it again",
|
||||||
|
m.Module, node, g.Name, node)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if needed[m.Module] == nil {
|
||||||
|
needed[m.Module] = map[string]string{}
|
||||||
|
}
|
||||||
|
needed[m.Module][g.Name] = g.Sealed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for name := range m.OwnSecrets.Plain() {
|
||||||
// Minted on the send path and only read on every other. Making one is an insert, and
|
// Minted on the send path and only read on every other. Making one is an insert, and
|
||||||
// a question that writes is a question that can block against the machine it is about.
|
// a question that writes is a question that can block against the machine it is about.
|
||||||
var sealed string
|
var sealed string
|
||||||
@@ -1264,6 +1291,11 @@ func planCommand(ctx context.Context, args []string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// What the machine was last told, by whom and from which assignment generation (novox/hq issue 234),
|
||||||
|
// beside what it would be told now. A record that cannot be read is said, not taken for none.
|
||||||
|
if err := writeLastSend(ctx, os.Stdout, open.inventory, args[0]); err != nil {
|
||||||
|
fmt.Printf("what %s was last sent could not be read: %v\n", args[0], err)
|
||||||
|
}
|
||||||
// Which modules a push would leave out, and why — said before the plan, since the plan is of
|
// Which modules a push would leave out, and why — said before the plan, since the plan is of
|
||||||
// what the machine would be told (novox/hq ADR 0163, rule 6). Judged, never composed: `plan`
|
// what the machine would be told (novox/hq ADR 0163, rule 6). Judged, never composed: `plan`
|
||||||
// without --json allocates nothing.
|
// without --json allocates nothing.
|
||||||
|
|||||||
@@ -164,12 +164,13 @@ func retryRefusal(p inventory.Plan, plans []inventory.Plan) error {
|
|||||||
case p.Open():
|
case p.Open():
|
||||||
return fmt.Errorf("%s is still %s; nothing in it failed to retry — `rebuild <module>` asks one module again", p.ID, p.State)
|
return fmt.Errorf("%s is still %s; nothing in it failed to retry — `rebuild <module>` asks one module again", p.ID, p.State)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(failedIn(p)) > 0 {
|
if len(failedIn(p)) > 0 {
|
||||||
if q, found := newerOpenPlan(p, plans); found {
|
if q, found := newerOpenPlan(p, plans); found {
|
||||||
return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and retrying %s would build "+
|
return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and retrying %s would build "+
|
||||||
"what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID)
|
"what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID)
|
||||||
}
|
}
|
||||||
return nil
|
return oneWalkAtATime(p, plans)
|
||||||
}
|
}
|
||||||
stopped := stoppedRollouts(p)
|
stopped := stoppedRollouts(p)
|
||||||
if len(stopped) == 0 {
|
if len(stopped) == 0 {
|
||||||
@@ -193,6 +194,17 @@ func retryRefusal(p inventory.Plan, plans []inventory.Plan) error {
|
|||||||
"the older build back", m, q.ID, q.State, q.Repository, short(q.Commit), p.ID)
|
"the older build back", m, q.ID, q.State, q.Repository, short(q.Commit), p.ID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return oneWalkAtATime(p, plans)
|
||||||
|
}
|
||||||
|
|
||||||
|
// oneWalkAtATime refuses a retry while another walk is open, started or waiting for its word (novox/hq ADR
|
||||||
|
// 0276): a walk retried beside it would be two walks at once.
|
||||||
|
func oneWalkAtATime(p inventory.Plan, plans []inventory.Plan) error {
|
||||||
|
for _, q := range plans {
|
||||||
|
if q.ID != p.ID && q.Open() && q.Release == nil {
|
||||||
|
return fmt.Errorf("%s is open (%s): one walk at a time — retry %s once it ended", q.ID, q.Named(), p.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -264,13 +276,36 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
|||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
plans = append(plans, recent...)
|
plans = append(plans, recent...)
|
||||||
|
// **A failed walk whose earlier merges are walked alone is not retried** (novox/hq ADR 0276): the search for
|
||||||
|
// the merge that brought the failure answers them now, and a retried walk would name them twice.
|
||||||
|
if p.Delivery != nil && len(p.Delivery.Merges) > 0 {
|
||||||
|
kept, err := inv.MergesOf(ctx, p.ID)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if len(kept) < len(p.Delivery.Merges) {
|
||||||
|
return "", fmt.Errorf("%s's earlier merges are walked alone, to find which one brought its failure: "+
|
||||||
|
"those walks answer them; a newer merge, or `rebuild <module>`, builds again", p.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Settled from the build records before anything is judged (novox/hq issue 457): a build of the tier
|
||||||
|
// that failed after the plan did is as failed as the one that failed it. What toRetry says is the
|
||||||
|
// failed set every step below works from.
|
||||||
|
var failed []string
|
||||||
|
if p.Tier < len(p.Tiers) {
|
||||||
|
recorded, byID, err := recordsOfAsked(ctx, inv, &p, p.Tiers[p.Tier])
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
failed = toRetry(&p, recorded, byID)
|
||||||
|
}
|
||||||
if err := retryRefusal(p, plans); err != nil {
|
if err := retryRefusal(p, plans); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
if again := unjudgedAtGate(p); len(failedIn(p)) == 0 && len(again) > 0 {
|
if again := unjudgedAtGate(p); len(failed) == 0 && len(again) > 0 {
|
||||||
return retryTierWhole(ctx, open, &p, again)
|
return retryTierWhole(ctx, open, &p, again)
|
||||||
}
|
}
|
||||||
if len(failedIn(p)) == 0 {
|
if len(failed) == 0 {
|
||||||
return retryRollouts(ctx, open, &p)
|
return retryRollouts(ctx, open, &p)
|
||||||
}
|
}
|
||||||
entries, err := inv.Catalogued(ctx)
|
entries, err := inv.Catalogued(ctx)
|
||||||
@@ -281,7 +316,6 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
|||||||
for _, e := range entries {
|
for _, e := range entries {
|
||||||
byName[e.Manifest.Module] = e
|
byName[e.Manifest.Module] = e
|
||||||
}
|
}
|
||||||
failed := failedIn(p)
|
|
||||||
var asked []string
|
var asked []string
|
||||||
for _, m := range failed {
|
for _, m := range failed {
|
||||||
askModule(ctx, &p, m, byName)
|
askModule(ctx, &p, m, byName)
|
||||||
@@ -501,3 +535,16 @@ func retryTierWhole(ctx context.Context, open *stores, p *inventory.Plan, again
|
|||||||
func sendAgain(s *inventory.PlanModule) {
|
func sendAgain(s *inventory.PlanModule) {
|
||||||
s.First, s.FirstAt, s.Gate, s.GatedBy, s.Previous, s.Why = nil, nil, nil, "", "", ""
|
s.First, s.FirstAt, s.Gate, s.GatedBy, s.Previous, s.Why = nil, nil, nil, "", "", ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// toRetry is the modules a retry asks again: every one of the tier that failed, the plan's state
|
||||||
|
// settled from the build records first (novox/hq issue 457). A plan fails on the first failure in its
|
||||||
|
// tier, and an outcome arriving after that finds no open plan to answer — it is kept only in the build
|
||||||
|
// records. Read from the plan alone, a retry asked only the build that failed first, and the records
|
||||||
|
// then failed the plan again on the next: each failed build of a tier took a retry of its own. What
|
||||||
|
// still runs is left asked, and its outcome is the plan's once the retry sets it building.
|
||||||
|
func toRetry(p *inventory.Plan, recorded map[string][]inventory.Build, byID map[string]inventory.Build) []string {
|
||||||
|
if p.Tier < len(p.Tiers) {
|
||||||
|
settleFromRecords(p, p.Tiers[p.Tier], recorded, byID)
|
||||||
|
}
|
||||||
|
return failedIn(*p)
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,57 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A retry asks every failed build of the tier, not one (novox/hq issue 457). Seen 2026-10-11: gitea
|
||||||
|
// and plex both failed in tier 0 while the registry was held still; gitea's failure failed the plan,
|
||||||
|
// and plex's, arriving after, found no open plan and was kept only in the build records. The first
|
||||||
|
// retry asked gitea alone, the records then failed the plan again on plex, and a second retry asked
|
||||||
|
// plex.
|
||||||
|
func TestARetryAsksEveryFailedBuildOfTheTier(t *testing.T) {
|
||||||
|
asked := time.Date(2026, 10, 11, 1, 29, 0, 0, time.UTC)
|
||||||
|
failedAt := asked.Add(2 * time.Minute)
|
||||||
|
p := inventory.Plan{ID: "plan-457", State: inventory.PlanFailed, Tier: 0,
|
||||||
|
Tiers: [][]string{{"gitea", "plex"}}, Note: "gitea failed to build in tier 0",
|
||||||
|
Modules: map[string]*inventory.PlanModule{
|
||||||
|
"gitea": {State: "failed", AskedAt: &asked, Build: "build-gitea", Why: "cannot reach the registry"},
|
||||||
|
"plex": {State: "asked", AskedAt: &asked, Build: "build-plex"},
|
||||||
|
}}
|
||||||
|
byID := map[string]inventory.Build{
|
||||||
|
"build-plex": {ID: "build-plex", Module: "plex", At: failedAt, Failed: "cannot reach the registry"},
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := toRetry(&p, nil, byID); !reflect.DeepEqual(got, []string{"gitea", "plex"}) {
|
||||||
|
t.Fatalf("a retry of a tier where gitea and plex failed asks %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A build of the tier still running when the plan failed is not asked again: its outcome is the plan's
|
||||||
|
// once the retry sets it building, and one that is recorded built is taken as built.
|
||||||
|
func TestARetryLeavesABuildThatRunsOrWorked(t *testing.T) {
|
||||||
|
asked := time.Date(2026, 10, 11, 1, 29, 0, 0, time.UTC)
|
||||||
|
builtAt := asked.Add(3 * time.Minute)
|
||||||
|
p := inventory.Plan{ID: "plan-457", State: inventory.PlanFailed, Tier: 0,
|
||||||
|
Tiers: [][]string{{"a", "b", "c"}},
|
||||||
|
Modules: map[string]*inventory.PlanModule{
|
||||||
|
"a": {State: "failed", AskedAt: &asked, Build: "build-a", Why: "broken"},
|
||||||
|
"b": {State: "asked", AskedAt: &asked, Build: "build-b"},
|
||||||
|
"c": {State: "asked", AskedAt: &asked, Build: "build-c"},
|
||||||
|
}}
|
||||||
|
byID := map[string]inventory.Build{"build-c": {ID: "build-c", Module: "c", At: builtAt, Commit: "c0ffee"}}
|
||||||
|
|
||||||
|
if got := toRetry(&p, nil, byID); !reflect.DeepEqual(got, []string{"a"}) {
|
||||||
|
t.Fatalf("asks %v, want a alone", got)
|
||||||
|
}
|
||||||
|
if s := p.Modules["c"]; s.State != "built" || s.Commit != "c0ffee" {
|
||||||
|
t.Errorf("c, recorded built, is %+v", s)
|
||||||
|
}
|
||||||
|
if s := p.Modules["b"]; s.State != "asked" {
|
||||||
|
t.Errorf("b, still building, is %+v", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -15,16 +15,16 @@ import (
|
|||||||
// inventory.Dependencies (dependenciesOf over the records), and the merge planned by reachOfMerge — the
|
// inventory.Dependencies (dependenciesOf over the records), and the merge planned by reachOfMerge — the
|
||||||
// path a real merge takes, short of the bus.
|
// path a real merge takes, short of the bus.
|
||||||
//
|
//
|
||||||
// **The repository rows are CURRENT BEHAVIOUR, documented — not the rule the operator states**
|
// **A shared repository moves only what a change's files are in the build source of** (novox/hq ADR 0267,
|
||||||
// (novox/hq issue 338, and the decision pending on it): a build that read a repository gives its module a
|
// issues 338 and 363): each build records the build source it said, and a merge is mapped onto those of the
|
||||||
// packages edge to every module built from that repository, and mergeCandidates moves it on any merge to
|
// newest builds. A build that said none (P below, as every build before ADR 0267) is read as before: P moves
|
||||||
// that repository, whatever the files. So a change to C alone, or to a README, moves the module that
|
// on any merge to the repository it packages, though through no edge. The rows tagged 338 held the opposite
|
||||||
// packages C's repository. ADR 0238 §3 records exactly that today ("a repository a recipe names"); the
|
// until ADR 0267 was built.
|
||||||
// expectations marked 338 change with that decision.
|
func TestASharedRepositoryIsPlannedFromTheRecordedBuildSources(t *testing.T) {
|
||||||
func TestASharedRepositoryIsPlannedFromTheRecordsAsItIsToday(t *testing.T) {
|
|
||||||
inv := inventory.ForTest(t)
|
inv := inventory.ForTest(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
asked := time.Now().Add(-time.Hour)
|
asked := time.Now().Add(-time.Hour)
|
||||||
|
sourcesOf := map[string][]inventory.BuildSource{}
|
||||||
register := func(m catalogue.Manifest, repository, path string, against []string, read []inventory.ReadRepository) {
|
register := func(m catalogue.Manifest, repository, path string, against []string, read []inventory.ReadRepository) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: repository, Seat: "git", Path: path,
|
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: repository, Seat: "git", Path: path,
|
||||||
@@ -32,7 +32,8 @@ func TestASharedRepositoryIsPlannedFromTheRecordsAsItIsToday(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + m.Module, Repository: repository, Ref: "main",
|
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + m.Module, Repository: repository, Ref: "main",
|
||||||
Module: m.Module, Commit: "old", On: "builder", Path: path, Against: against, Read: read, Asked: asked}); err != nil {
|
Module: m.Module, Commit: "old", On: "builder", Path: path, Against: against, Read: read, Asked: asked,
|
||||||
|
Sources: sourcesOf[m.Module]}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -40,7 +41,16 @@ func TestASharedRepositoryIsPlannedFromTheRecordsAsItIsToday(t *testing.T) {
|
|||||||
agent := catalogue.Manifest{Module: "build-agent", Version: "1",
|
agent := catalogue.Manifest{Module: "build-agent", Version: "1",
|
||||||
Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}}
|
Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}}
|
||||||
|
|
||||||
// The shape of issue 338.
|
// The shape of issue 338, each build saying its build source (ADR 0267).
|
||||||
|
gomod := []string{"go.mod", "go.sum"}
|
||||||
|
sourcesOf["mesh-controller"] = []inventory.BuildSource{{Paths: append([]string{"module.json", "cmd/mesh-controller/",
|
||||||
|
"internal/conditions/", "internal/broker/"}, gomod...)}}
|
||||||
|
sourcesOf["build-agent"] = []inventory.BuildSource{
|
||||||
|
{Paths: []string{"modules/build-agent/Dockerfile", "modules/build-agent/module.json"}},
|
||||||
|
{Repository: "novox/mesh-controller", Ref: "main", Paths: append([]string{"cmd/mesh-builder/", "internal/broker/"}, gomod...)}}
|
||||||
|
sourcesOf["route-proxy"] = []inventory.BuildSource{
|
||||||
|
{Paths: []string{"modules/route-proxy/Dockerfile", "modules/route-proxy/module.json"}},
|
||||||
|
{Repository: "novox/mesh-controller", Ref: "main", Paths: append([]string{"examples/route-proxy/", "internal/broker/"}, gomod...)}}
|
||||||
register(catalogue.Manifest{Module: "mesh-controller", Version: "1"}, "novox/mesh-controller", "", nil, nil)
|
register(catalogue.Manifest{Module: "mesh-controller", Version: "1"}, "novox/mesh-controller", "", nil, nil)
|
||||||
register(agent, "novox/mesh-catalog", "modules/build-agent", nil, controllerRead)
|
register(agent, "novox/mesh-catalog", "modules/build-agent", nil, controllerRead)
|
||||||
register(catalogue.Manifest{Module: "route-proxy", Version: "1"}, "novox/mesh-catalog", "modules/route-proxy", nil, controllerRead)
|
register(catalogue.Manifest{Module: "route-proxy", Version: "1"}, "novox/mesh-catalog", "modules/route-proxy", nil, controllerRead)
|
||||||
@@ -81,13 +91,15 @@ func TestASharedRepositoryIsPlannedFromTheRecordsAsItIsToday(t *testing.T) {
|
|||||||
if !reflect.DeepEqual(shared, sharedRepositoryEdges) {
|
if !reflect.DeepEqual(shared, sharedRepositoryEdges) {
|
||||||
t.Errorf("derived %v\nthe hand-written rows use %v", shared, sharedRepositoryEdges)
|
t.Errorf("derived %v\nthe hand-written rows use %v", shared, sharedRepositoryEdges)
|
||||||
}
|
}
|
||||||
// Each kind derived from its record: built against (stands-on), build.on (declared), read (packages).
|
// Each kind derived from its record: built against (stands-on), build.on (declared); a read draws none.
|
||||||
|
for _, e := range edges {
|
||||||
|
if e.Kind == inventory.EdgePackages {
|
||||||
|
t.Errorf("a packages edge was drawn (ADR 0267 rule 4): %v", e)
|
||||||
|
}
|
||||||
|
}
|
||||||
for _, want := range []inventory.Edge{
|
for _, want := range []inventory.Edge{
|
||||||
dep("d", inventory.EdgeStandsOn, "a"),
|
dep("d", inventory.EdgeStandsOn, "a"),
|
||||||
dep("e", inventory.EdgeDeclared, "b"),
|
dep("e", inventory.EdgeDeclared, "b"),
|
||||||
dep("p", inventory.EdgePackages, "a"),
|
|
||||||
dep("p", inventory.EdgePackages, "b"),
|
|
||||||
dep("p", inventory.EdgePackages, "c"),
|
|
||||||
dep("d", inventory.EdgeBuiltBy, "build-agent"),
|
dep("d", inventory.EdgeBuiltBy, "build-agent"),
|
||||||
} {
|
} {
|
||||||
found := false
|
found := false
|
||||||
@@ -105,15 +117,23 @@ func TestASharedRepositoryIsPlannedFromTheRecordsAsItIsToday(t *testing.T) {
|
|||||||
want string
|
want string
|
||||||
issue338 bool
|
issue338 bool
|
||||||
}{
|
}{
|
||||||
{"A and B changed, C untouched: D after A, E after B; P packages their repository", "one",
|
{"A and B changed, C untouched: D after A, E after B; P, which said no build source, reads all", "one",
|
||||||
[]string{"modules/a/x.go", "modules/b/x.go"}, "a,b,p | d,e", false},
|
[]string{"modules/a/x.go", "modules/b/x.go"}, "a,b,p | d,e", false},
|
||||||
{"C alone: C, and P, which packages C's repository", "one",
|
{"C alone: C, and P, read whole as before; P after nothing", "one",
|
||||||
[]string{"modules/c/x.go"}, "c,p", true},
|
[]string{"modules/c/x.go"}, "c,p", false},
|
||||||
{"a README of the repository P packages: P moves, nothing built from it does", "one",
|
{"a README of the repository P packages: P, read whole as before", "one",
|
||||||
[]string{"README.md"}, "p", true},
|
[]string{"README.md"}, "p", false},
|
||||||
{"the dependent's repository: D alone", "two", []string{"d/main.go"}, "d", false},
|
{"the dependent's repository: D alone", "two", []string{"d/main.go"}, "d", false},
|
||||||
{"a README of the controller's repository: all three, three tiers", "mesh-controller",
|
{"a README of the controller's repository: no module", "mesh-controller",
|
||||||
[]string{"README.md"}, "mesh-controller | build-agent | route-proxy", true},
|
[]string{"README.md"}, "", true},
|
||||||
|
{"the controller's own command: the controller alone", "mesh-controller",
|
||||||
|
[]string{"cmd/mesh-controller/main.go"}, "mesh-controller", true},
|
||||||
|
{"a package only the controller builds from: the controller alone", "mesh-controller",
|
||||||
|
[]string{"internal/conditions/condition.go"}, "mesh-controller", true},
|
||||||
|
{"the route proxy's program: the route proxy alone", "mesh-controller",
|
||||||
|
[]string{"examples/route-proxy/main.go"}, "route-proxy", true},
|
||||||
|
{"a package all three build from: all three", "mesh-controller",
|
||||||
|
[]string{"internal/broker/broker.go"}, "mesh-controller | build-agent | route-proxy", false},
|
||||||
{"the route proxy's directory in the catalogue: it alone", "mesh-catalog",
|
{"the route proxy's directory in the catalogue: it alone", "mesh-catalog",
|
||||||
[]string{"modules/route-proxy/module.json"}, "route-proxy", false},
|
[]string{"modules/route-proxy/module.json"}, "route-proxy", false},
|
||||||
{"the build agent's directory: it alone, nothing it builds", "mesh-catalog",
|
{"the build agent's directory: it alone, nothing it builds", "mesh-catalog",
|
||||||
@@ -123,7 +143,7 @@ func TestASharedRepositoryIsPlannedFromTheRecordsAsItIsToday(t *testing.T) {
|
|||||||
if got != c.want {
|
if got != c.want {
|
||||||
tag := ""
|
tag := ""
|
||||||
if c.issue338 {
|
if c.issue338 {
|
||||||
tag = " (current behaviour, issue 338)"
|
tag = " (issue 338, flipped by ADR 0267)"
|
||||||
}
|
}
|
||||||
t.Errorf("%s: planned %q, wanted %q%s", c.what, got, c.want, tag)
|
t.Errorf("%s: planned %q, wanted %q%s", c.what, got, c.want, tag)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,12 +1,15 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"math/rand/v2"
|
"math/rand/v2"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
snapshot "github.com/novox/mesh-controller/internal/facts"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
)
|
)
|
||||||
@@ -21,7 +24,7 @@ import (
|
|||||||
// kind widens the plan orders the tiers
|
// kind widens the plan orders the tiers
|
||||||
// stands-on yes yes, after its base is built
|
// stands-on yes yes, after its base is built
|
||||||
// declared yes yes, after its base is built
|
// declared yes yes, after its base is built
|
||||||
// packages yes no, the same tier (a code dependency)
|
// packages no no — retired by novox/hq ADR 0267; one recorded before is read and ignored
|
||||||
// built-by no yes, after the build machine — except for what the build machine stands
|
// built-by no yes, after the build machine — except for what the build machine stands
|
||||||
// on, and for the controller whose worker it binds
|
// on, and for the controller whose worker it binds
|
||||||
// worker-of no yes, the build seat's holder after the controller (hq issue 206)
|
// worker-of no yes, the build seat's holder after the controller (hq issue 206)
|
||||||
@@ -123,9 +126,9 @@ func TestAPlanIsWhatTheChangeTouchedAndWhatIsBuiltOnIt(t *testing.T) {
|
|||||||
{what: "transitive: F on D on A, A changed",
|
{what: "transitive: F on D on A, A changed",
|
||||||
edges: []inventory.Edge{dep("f", standsOn, "d"), dep("d", standsOn, "a")},
|
edges: []inventory.Edge{dep("f", standsOn, "d"), dep("d", standsOn, "a")},
|
||||||
repo: "one", paths: []string{"modules/a/x"}, want: "a | d | f"},
|
repo: "one", paths: []string{"modules/a/x"}, want: "a | d | f"},
|
||||||
{what: "transitive across kinds: F declared on D, D packages A",
|
{what: "transitive across kinds stops at a packages edge: F declared on D, D packages A (ADR 0267)",
|
||||||
edges: []inventory.Edge{dep("f", declared, "d"), dep("d", packages, "a")},
|
edges: []inventory.Edge{dep("f", declared, "d"), dep("d", packages, "a")},
|
||||||
repo: "one", paths: []string{"modules/a/x"}, want: "a,d | f"},
|
repo: "one", paths: []string{"modules/a/x"}, want: "a"},
|
||||||
|
|
||||||
// Each kind alone: X depends on A, A changed (widening), then both changed (ordering).
|
// Each kind alone: X depends on A, A changed (widening), then both changed (ordering).
|
||||||
{what: "stands-on (built against A's artifact) widens", edges: []inventory.Edge{dep("x", standsOn, "a")},
|
{what: "stands-on (built against A's artifact) widens", edges: []inventory.Edge{dep("x", standsOn, "a")},
|
||||||
@@ -136,8 +139,8 @@ func TestAPlanIsWhatTheChangeTouchedAndWhatIsBuiltOnIt(t *testing.T) {
|
|||||||
repo: "one", paths: []string{"modules/a/x"}, want: "a | x"},
|
repo: "one", paths: []string{"modules/a/x"}, want: "a | x"},
|
||||||
{what: "declared orders", edges: []inventory.Edge{dep("x", declared, "a")},
|
{what: "declared orders", edges: []inventory.Edge{dep("x", declared, "a")},
|
||||||
repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a | x"},
|
repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a | x"},
|
||||||
{what: "packages widens, into the same tier", edges: []inventory.Edge{dep("x", packages, "a")},
|
{what: "packages, recorded before ADR 0267, widens nothing", edges: []inventory.Edge{dep("x", packages, "a")},
|
||||||
repo: "one", paths: []string{"modules/a/x"}, want: "a,x"},
|
repo: "one", paths: []string{"modules/a/x"}, want: "a"},
|
||||||
{what: "packages does not order", edges: []inventory.Edge{dep("x", packages, "a")},
|
{what: "packages does not order", edges: []inventory.Edge{dep("x", packages, "a")},
|
||||||
repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a,x"},
|
repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a,x"},
|
||||||
{what: "built-by never widens", edges: []inventory.Edge{dep("x", builtBy, "a")},
|
{what: "built-by never widens", edges: []inventory.Edge{dep("x", builtBy, "a")},
|
||||||
@@ -188,7 +191,7 @@ func TestAPlanIsWhatTheChangeTouchedAndWhatIsBuiltOnIt(t *testing.T) {
|
|||||||
repo: "one", paths: []string{"modules/z/x"}, want: "z | a,b | d"},
|
repo: "one", paths: []string{"modules/z/x"}, want: "z | a,b | d"},
|
||||||
{what: "a diamond of mixed kinds orders on the ordering side only",
|
{what: "a diamond of mixed kinds orders on the ordering side only",
|
||||||
edges: []inventory.Edge{dep("d", standsOn, "a"), dep("d", packages, "b")},
|
edges: []inventory.Edge{dep("d", standsOn, "a"), dep("d", packages, "b")},
|
||||||
repo: "one", paths: []string{"modules/b/x"}, want: "b,d"},
|
repo: "one", paths: []string{"modules/b/x"}, want: "b"},
|
||||||
|
|
||||||
// A cycle the catalogue should never produce: what remains is one last tier, and said.
|
// A cycle the catalogue should never produce: what remains is one last tier, and said.
|
||||||
{what: "a cycle is one last tier, not lost", edges: []inventory.Edge{dep("a", standsOn, "b"), dep("b", standsOn, "a"),
|
{what: "a cycle is one last tier, not lost", edges: []inventory.Edge{dep("a", standsOn, "b"), dep("b", standsOn, "a"),
|
||||||
@@ -225,22 +228,16 @@ func TestAPlanIsWhatTheChangeTouchedAndWhatIsBuiltOnIt(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// **CURRENT BEHAVIOUR, documented — not the rule the operator states.** novox/hq issue 338 (a module
|
// **A shared repository moves only what a change's files are in the build source of** (novox/hq ADR 0267,
|
||||||
// built from a shared repository moves on every merge to it) and the decision pending on it would change
|
// issue 338, issue 363). The controller is built from its repository's root as a Go bundle; the route proxy
|
||||||
// every row here. Today:
|
// and the build seat's holder build images whose context is that repository and which name the package they
|
||||||
|
// compile. Each newest trunk build said its build source — the import closure of its program — and a merge
|
||||||
|
// is mapped onto those. The rows tagged 338 held the opposite until ADR 0267 was built: every merge to the
|
||||||
|
// controller's repository planned all three, in three tiers.
|
||||||
//
|
//
|
||||||
// - mesh-controller is built from its repository's root, so every file of that repository touches it;
|
// The build sources are this repository's own programs as GoBuildSource reads them (held to that by
|
||||||
// - route-proxy and build-agent package the whole of that repository (a build context), so the build
|
// TestThisRepositorysProgramsHaveBuildSourcesOfTheirOwn in internal/builder), cut to what the rows need.
|
||||||
// record's `read` makes them move on any merge to it, whatever the files, and dependenciesOf gives
|
func TestASharedRepositoryMovesOnlyWhatItsBuildSourceHolds(t *testing.T) {
|
||||||
// each a packages edge to every module built from it;
|
|
||||||
// - built-by (route-proxy on build-agent) and worker-of (build-agent on the controller) make it three
|
|
||||||
// tiers.
|
|
||||||
//
|
|
||||||
// These follow ADR 0238 §3 as written ("the whole repository for a module built from its root, and a
|
|
||||||
// repository a recipe names"), so they are not failures; when the decision on issue 338 lands, these
|
|
||||||
// expectations change with it. The edges are the ones dependenciesOf derives from this catalogue — held
|
|
||||||
// to that by TestASharedRepositoryIsPlannedFromTheRecordsAsItIsToday, which derives them from the store.
|
|
||||||
func TestASharedRepositoryMovesWhatPackagesItAsItDoesToday(t *testing.T) {
|
|
||||||
const catalogueRepo = "http://forge.internal:20000/novox/mesh-catalog.git"
|
const catalogueRepo = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||||
const controllerRepo = "http://forge.internal:20000/novox/mesh-controller.git"
|
const controllerRepo = "http://forge.internal:20000/novox/mesh-controller.git"
|
||||||
entries := []inventory.Entry{
|
entries := []inventory.Entry{
|
||||||
@@ -249,7 +246,26 @@ func TestASharedRepositoryMovesWhatPackagesItAsItDoesToday(t *testing.T) {
|
|||||||
fromRepo("route-proxy", catalogueRepo, "modules/route-proxy"),
|
fromRepo("route-proxy", catalogueRepo, "modules/route-proxy"),
|
||||||
fromRepo("gitea", catalogueRepo, "modules/gitea"),
|
fromRepo("gitea", catalogueRepo, "modules/gitea"),
|
||||||
}
|
}
|
||||||
|
gomod := []string{"go.mod", "go.sum", "vendor/modules.txt"}
|
||||||
|
with := func(paths ...string) []string { return append(append([]string{}, gomod...), paths...) }
|
||||||
read := map[string][]inventory.ReadRepository{
|
read := map[string][]inventory.ReadRepository{
|
||||||
|
"mesh-controller": {{Own: true, Paths: with("module.json", "cmd/mesh-controller/", "internal/conditions/",
|
||||||
|
"internal/broker/", "internal/builder/", "internal/inventory/", "internal/inventory/migrations/**",
|
||||||
|
"vendor/github.com/nats-io/nats.go/")}},
|
||||||
|
"build-agent": {
|
||||||
|
{Repository: "novox/mesh-controller", Ref: "main", Paths: with("cmd/mesh-builder/", "internal/broker/",
|
||||||
|
"internal/builder/", "internal/inventory/", "internal/inventory/migrations/**", "vendor/github.com/nats-io/nats.go/")},
|
||||||
|
{Own: true, Paths: []string{"modules/build-agent/Dockerfile", "modules/build-agent/module.json"}},
|
||||||
|
},
|
||||||
|
"route-proxy": {
|
||||||
|
{Repository: "novox/mesh-controller", Ref: "main", Paths: with("examples/route-proxy/", "internal/broker/",
|
||||||
|
"vendor/github.com/nats-io/nats.go/")},
|
||||||
|
{Own: true, Paths: []string{"modules/route-proxy/Dockerfile", "modules/route-proxy/module.json"}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
// With no build source said — before each module's first trunk build under ADR 0267, or while an
|
||||||
|
// earlier merge's build of it is pending — a module is read as before.
|
||||||
|
unsaid := map[string][]inventory.ReadRepository{
|
||||||
"build-agent": {{Repository: "novox/mesh-controller", Ref: "main"}},
|
"build-agent": {{Repository: "novox/mesh-controller", Ref: "main"}},
|
||||||
"route-proxy": {{Repository: "novox/mesh-controller", Ref: "main"}},
|
"route-proxy": {{Repository: "novox/mesh-controller", Ref: "main"}},
|
||||||
}
|
}
|
||||||
@@ -257,51 +273,176 @@ func TestASharedRepositoryMovesWhatPackagesItAsItDoesToday(t *testing.T) {
|
|||||||
for _, c := range []struct {
|
for _, c := range []struct {
|
||||||
what, repo string
|
what, repo string
|
||||||
paths []string
|
paths []string
|
||||||
|
read map[string][]inventory.ReadRepository
|
||||||
want string
|
want string
|
||||||
|
unread string
|
||||||
}{
|
}{
|
||||||
// The live three-tier plan of 2026-10-08 (issue 338), in the worker-of order (issue 206) that
|
// The operator's acceptance: a merge of the controller's own code plans the controller alone.
|
||||||
// TestAMergeIsPlannedInTiersAlongTheThreeKindsOfDependency's controller case holds too.
|
{"the controller's own command: the controller alone (338)", "mesh-controller",
|
||||||
{"a README of the controller's repository moves all three, in three tiers", "mesh-controller",
|
[]string{"cmd/mesh-controller/main.go"}, read, "mesh-controller", ""},
|
||||||
[]string{"README.md"}, "mesh-controller | build-agent | route-proxy"},
|
{"a package only the controller builds from: the controller alone (338)", "mesh-controller",
|
||||||
{"the controller's own code: the same", "mesh-controller",
|
[]string{"internal/conditions/condition.go", "internal/conditions/bus.go"}, read, "mesh-controller", ""},
|
||||||
[]string{"cmd/mesh-controller/main.go"}, "mesh-controller | build-agent | route-proxy"},
|
{"a test beside the controller's command: nothing is built from it", "mesh-controller",
|
||||||
{"the route proxy's program alone: the same, the controller with it", "mesh-controller",
|
[]string{"cmd/mesh-controller/main_test.go"}, read, "", "cmd/mesh-controller/main_test.go"},
|
||||||
[]string{"examples/route-proxy/main.go"}, "mesh-controller | build-agent | route-proxy"},
|
{"a README of the controller's repository: no module (338)", "mesh-controller",
|
||||||
// In the catalogue, where they live, the rule is path-precise.
|
[]string{"README.md"}, read, "", "README.md"},
|
||||||
{"the route proxy's directory in the catalogue: it alone", "mesh-catalog",
|
{"the route proxy's program alone: the route proxy alone (338)", "mesh-controller",
|
||||||
[]string{"modules/route-proxy/module.json"}, "route-proxy"},
|
[]string{"examples/route-proxy/main.go"}, read, "route-proxy", ""},
|
||||||
{"the build agent's directory: it alone, nothing it builds", "mesh-catalog",
|
{"the build seat's program alone: its holder alone", "mesh-controller",
|
||||||
[]string{"modules/build-agent/module.json"}, "build-agent"},
|
[]string{"cmd/mesh-builder/main.go"}, read, "build-agent", ""},
|
||||||
|
{"a package the build seat's program and the controller build from: both", "mesh-controller",
|
||||||
|
[]string{"internal/builder/builder.go"}, read, "mesh-controller | build-agent", ""},
|
||||||
|
{"a migration the controller and the build seat's program embed: both", "mesh-controller",
|
||||||
|
[]string{"internal/inventory/migrations/0088-a-build-says-its-build-source.sql"}, read,
|
||||||
|
"mesh-controller | build-agent", ""},
|
||||||
|
// A package all three build from: all three; the build seat's holder after the controller whose worker
|
||||||
|
// it binds (worker-of, issue 206), the proxy after the holder that builds it (built-by).
|
||||||
|
{"a package all three build from: all three", "mesh-controller",
|
||||||
|
[]string{"internal/broker/broker.go"}, read, "mesh-controller | build-agent | route-proxy", ""},
|
||||||
|
{"a vendored package all three build from: all three", "mesh-controller",
|
||||||
|
[]string{"vendor/github.com/nats-io/nats.go/nats.go"}, read, "mesh-controller | build-agent | route-proxy", ""},
|
||||||
|
{"go.sum: all three", "mesh-controller",
|
||||||
|
[]string{"go.sum"}, read, "mesh-controller | build-agent | route-proxy", ""},
|
||||||
|
{"a file added to the proxy's package: the proxy", "mesh-controller",
|
||||||
|
[]string{"examples/route-proxy/new.go"}, read, "route-proxy", ""},
|
||||||
|
// Before any build source is said: as before.
|
||||||
|
{"no build source said: a README moves all three, as before", "mesh-controller",
|
||||||
|
[]string{"README.md"}, unsaid, "mesh-controller | build-agent | route-proxy", ""},
|
||||||
|
// In the catalogue, where they live, each by its own build source.
|
||||||
|
{"the route proxy's recipe: it alone", "mesh-catalog",
|
||||||
|
[]string{"modules/route-proxy/Dockerfile"}, read, "route-proxy", ""},
|
||||||
|
{"the route proxy's manifest: it alone", "mesh-catalog",
|
||||||
|
[]string{"modules/route-proxy/module.json"}, read, "route-proxy", ""},
|
||||||
|
{"the route proxy's README: nothing", "mesh-catalog",
|
||||||
|
[]string{"modules/route-proxy/README.md"}, read, "", "modules/route-proxy/README.md"},
|
||||||
|
{"the build agent's manifest: it alone, nothing it builds", "mesh-catalog",
|
||||||
|
[]string{"modules/build-agent/module.json"}, read, "build-agent", ""},
|
||||||
{"another module of the catalogue: neither", "mesh-catalog",
|
{"another module of the catalogue: neither", "mesh-catalog",
|
||||||
[]string{"modules/gitea/index.ts"}, "gitea"},
|
[]string{"modules/gitea/index.ts"}, read, "gitea", ""},
|
||||||
} {
|
} {
|
||||||
r, got := planMerge(t, c.repo, c.paths, entries, read, edges)
|
r, got := planMerge(t, c.repo, c.paths, entries, c.read, edges)
|
||||||
if got != c.want {
|
if got != c.want {
|
||||||
t.Errorf("%s: planned %q, wanted %q (as today; issue 338)", c.what, got, c.want)
|
t.Errorf("%s: planned %q, wanted %q", c.what, got, c.want)
|
||||||
}
|
}
|
||||||
if c.repo == "mesh-controller" && strings.Join(r.Unread, ",") != "" {
|
if u := strings.Join(r.Unread, ","); u != c.unread {
|
||||||
t.Errorf("%s: a root-built module reads every file, and %v were said unread", c.what, r.Unread)
|
t.Errorf("%s: unread %q, wanted %q", c.what, u, c.unread)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Files not all said: everything the repository builds, as before.
|
||||||
|
m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main", Commit: "head",
|
||||||
|
Paths: []string{"README.md"}, PathsTruncated: true}
|
||||||
|
if got := tiered(reachOfMerge(m, entries, read, edges).Plan.Tiers); got != "mesh-controller | build-agent | route-proxy" {
|
||||||
|
t.Errorf("a merge whose files were not all said: planned %q, wanted all three", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A module whose recorded build source a plan has overtaken is read whole** (novox/hq ADR 0267): a merge
|
||||||
|
// that added an import to the route proxy is planned; before a build of it works — still building, failed,
|
||||||
|
// or its plan closed before reaching it — a merge changing only the newly imported package must still move
|
||||||
|
// the proxy, since the build source its last build said does not hold that package.
|
||||||
|
func TestAModuleAPlanOvertookIsReadWhole(t *testing.T) {
|
||||||
|
built := time.Date(2026, 10, 10, 1, 0, 0, 0, time.UTC)
|
||||||
|
read := map[string][]inventory.ReadRepository{
|
||||||
|
"route-proxy": {
|
||||||
|
{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"examples/route-proxy/", "go.mod"}, Built: built},
|
||||||
|
{Own: true, Paths: []string{"modules/route-proxy/module.json"}, Built: built},
|
||||||
|
},
|
||||||
|
"mesh-controller": {{Own: true, Paths: []string{"module.json", "cmd/mesh-controller/"}, Built: built}},
|
||||||
|
}
|
||||||
|
m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main", Paths: []string{"internal/newly/imported.go"}}
|
||||||
|
if readsFrom(read["route-proxy"], m) {
|
||||||
|
t.Fatal("the said build source holds the new package: the fixture is wrong")
|
||||||
|
}
|
||||||
|
proxy := func(state string) map[string]*inventory.PlanModule {
|
||||||
|
return map[string]*inventory.PlanModule{"route-proxy": {State: state}, "gitea": {State: "built"}}
|
||||||
|
}
|
||||||
|
for _, c := range []struct {
|
||||||
|
what string
|
||||||
|
plans []inventory.Plan
|
||||||
|
whole bool
|
||||||
|
}{
|
||||||
|
{"no plan", nil, false},
|
||||||
|
{"a plan still building it", []inventory.Plan{{State: inventory.PlanBuilding, Created: built.Add(-time.Hour),
|
||||||
|
Modules: proxy("building")}}, true},
|
||||||
|
{"a plan made after its build that failed before building it", []inventory.Plan{{State: "failed",
|
||||||
|
Created: built.Add(time.Minute), Modules: proxy("waiting")}}, true},
|
||||||
|
{"a plan made after its build that built it", []inventory.Plan{{State: inventory.PlanDone,
|
||||||
|
Created: built.Add(time.Minute), Modules: proxy("built")}}, false},
|
||||||
|
{"a plan closed before its build", []inventory.Plan{{State: "failed", Created: built.Add(-time.Hour),
|
||||||
|
Modules: proxy("waiting")}}, false},
|
||||||
|
} {
|
||||||
|
view := planningView(read, c.plans)
|
||||||
|
if readsFrom(view["route-proxy"], m) != c.whole {
|
||||||
|
t.Errorf("%s: read whole %v, wanted %v", c.what, !c.whole, c.whole)
|
||||||
|
}
|
||||||
|
if (ownSource(view["route-proxy"]) == nil) != c.whole {
|
||||||
|
t.Errorf("%s: its own build source kept %v", c.what, ownSource(view["route-proxy"]) != nil)
|
||||||
|
}
|
||||||
|
if ownSource(view["mesh-controller"]) == nil {
|
||||||
|
t.Errorf("%s: a module no plan holds lost its build source", c.what)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **A missed merge that moves only a module packaging the repository is acted on** (novox/hq ADR 0267,
|
||||||
|
// issue 266): the catch-up asks wouldMove, which counts it; once a plan or build of it is made after the
|
||||||
|
// merge, the merge is history for it, and the catch-up leaves it.
|
||||||
|
func TestAMissedMergeMovingOnlyAPackagingModuleIsActedOnOnce(t *testing.T) {
|
||||||
|
merged := time.Date(2026, 10, 10, 1, 0, 0, 0, time.UTC)
|
||||||
|
entries := []inventory.Entry{
|
||||||
|
fromRepo("mesh-controller", "http://forge.internal:20000/novox/mesh-controller.git", ""),
|
||||||
|
fromRepo("route-proxy", "http://forge.internal:20000/novox/mesh-catalog.git", "modules/route-proxy"),
|
||||||
|
}
|
||||||
|
read := map[string][]inventory.ReadRepository{
|
||||||
|
"mesh-controller": {{Own: true, Paths: []string{"module.json", "cmd/mesh-controller/"}, Built: merged.Add(-time.Hour)}},
|
||||||
|
"route-proxy": {{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"examples/route-proxy/"},
|
||||||
|
Built: merged.Add(-time.Hour), Looked: merged.Add(-time.Hour)}},
|
||||||
|
}
|
||||||
|
m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main", Commit: "c1",
|
||||||
|
MergedAt: merged.Format(time.RFC3339), Paths: []string{"examples/route-proxy/main.go"}}
|
||||||
|
if got := wouldMove(m, entries, planningView(read, nil)); len(got) != 1 || got[0].Manifest.Module != "route-proxy" {
|
||||||
|
t.Fatalf("a missed merge of the proxy's program would move %v", got)
|
||||||
|
}
|
||||||
|
// Acted on at once: the plan answering this very merge is a look, however close the clocks.
|
||||||
|
atOnce := []inventory.Plan{{State: inventory.PlanBuilding, Commit: "c1", Created: merged.Add(2 * time.Second),
|
||||||
|
Modules: map[string]*inventory.PlanModule{"route-proxy": {State: "building"}}}}
|
||||||
|
if got := wouldMove(m, entries, planningView(read, atOnce)); len(got) != 0 {
|
||||||
|
t.Fatalf("a merge whose own plan holds the proxy would move %v again", got)
|
||||||
|
}
|
||||||
|
acted := []inventory.Plan{{State: inventory.PlanBuilding, Created: merged.Add(2 * time.Minute),
|
||||||
|
Modules: map[string]*inventory.PlanModule{"route-proxy": {State: "building"}}}}
|
||||||
|
if got := wouldMove(m, entries, planningView(read, acted)); len(got) != 0 {
|
||||||
|
t.Fatalf("a merge acted on for the proxy would move %v again", got)
|
||||||
|
}
|
||||||
|
// A plan that closed without building it looked at nothing: the merge is still news for it.
|
||||||
|
closed := []inventory.Plan{{State: "failed", Created: merged.Add(2 * time.Minute),
|
||||||
|
Modules: map[string]*inventory.PlanModule{"route-proxy": {State: "waiting"}}}}
|
||||||
|
if got := wouldMove(m, entries, planningView(read, closed)); len(got) != 1 {
|
||||||
|
t.Fatalf("a plan that never built the proxy hid the merge from it: %v", got)
|
||||||
|
}
|
||||||
|
// A look just before the merge, on clocks a little apart, is no look after it.
|
||||||
|
skewed := []inventory.Plan{{State: inventory.PlanBuilding, Created: merged.Add(30 * time.Second),
|
||||||
|
Modules: map[string]*inventory.PlanModule{"route-proxy": {State: "building"}}}}
|
||||||
|
if got := wouldMove(m, entries, planningView(read, skewed)); len(got) != 1 {
|
||||||
|
t.Fatalf("a look within the clocks' margin made the merge history: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// sharedRepositoryEdges is what dependenciesOf derives for the catalogue of the test above, sorted as it
|
// sharedRepositoryEdges is what dependenciesOf derives for the catalogue of the test above, sorted as it
|
||||||
// sorts them.
|
// sorts them: no packages edge (novox/hq ADR 0267 rule 4).
|
||||||
var sharedRepositoryEdges = []inventory.Edge{
|
var sharedRepositoryEdges = []inventory.Edge{
|
||||||
dep("build-agent", inventory.EdgePackages, "mesh-controller"),
|
|
||||||
dep("build-agent", inventory.EdgeWorkerOf, "mesh-controller"),
|
dep("build-agent", inventory.EdgeWorkerOf, "mesh-controller"),
|
||||||
dep("gitea", inventory.EdgeBuiltBy, "build-agent"),
|
dep("gitea", inventory.EdgeBuiltBy, "build-agent"),
|
||||||
dep("mesh-controller", inventory.EdgeBuiltBy, "build-agent"),
|
dep("mesh-controller", inventory.EdgeBuiltBy, "build-agent"),
|
||||||
dep("route-proxy", inventory.EdgeBuiltBy, "build-agent"),
|
dep("route-proxy", inventory.EdgeBuiltBy, "build-agent"),
|
||||||
dep("route-proxy", inventory.EdgePackages, "mesh-controller"),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// **The planner's invariant, over random catalogues.** For any catalogue whose dependencies form no cycle
|
// **The planner's invariant, over random catalogues.** For any catalogue whose dependencies form no cycle
|
||||||
// and any set of changed files in one repository:
|
// and any set of changed files in one repository:
|
||||||
//
|
//
|
||||||
// - the plan is exactly the modules of that repository whose directory holds a changed file (every file,
|
// - the plan is exactly the modules of that repository whose directory holds a changed file (every file,
|
||||||
// for a module built from the root), and everything reachable from them along stands-on, declared and
|
// for a module built from the root), and everything reachable from them along stands-on and declared —
|
||||||
// packages — never along built-by or worker-of;
|
// never along packages (novox/hq ADR 0267), built-by or worker-of;
|
||||||
// - every stands-on, declared, built-by and worker-of edge with both ends in the plan has the module
|
// - every stands-on, declared, built-by and worker-of edge with both ends in the plan has the module
|
||||||
// depended on in an earlier tier;
|
// depended on in an earlier tier;
|
||||||
// - no cycle is said.
|
// - no cycle is said.
|
||||||
@@ -310,7 +451,7 @@ var sharedRepositoryEdges = []inventory.Edge{
|
|||||||
func TestAPlanIsTheTouchedModulesAndWhatIsReachableAlongTheWideningEdges(t *testing.T) {
|
func TestAPlanIsTheTouchedModulesAndWhatIsReachableAlongTheWideningEdges(t *testing.T) {
|
||||||
kinds := []string{inventory.EdgeStandsOn, inventory.EdgeDeclared, inventory.EdgePackages,
|
kinds := []string{inventory.EdgeStandsOn, inventory.EdgeDeclared, inventory.EdgePackages,
|
||||||
inventory.EdgeBuiltBy, inventory.EdgeWorkerOf}
|
inventory.EdgeBuiltBy, inventory.EdgeWorkerOf}
|
||||||
widens := map[string]bool{inventory.EdgeStandsOn: true, inventory.EdgeDeclared: true, inventory.EdgePackages: true}
|
widens := map[string]bool{inventory.EdgeStandsOn: true, inventory.EdgeDeclared: true}
|
||||||
orders := map[string]bool{inventory.EdgeStandsOn: true, inventory.EdgeDeclared: true,
|
orders := map[string]bool{inventory.EdgeStandsOn: true, inventory.EdgeDeclared: true,
|
||||||
inventory.EdgeBuiltBy: true, inventory.EdgeWorkerOf: true}
|
inventory.EdgeBuiltBy: true, inventory.EdgeWorkerOf: true}
|
||||||
// Directory names drawn from one pool, so two repositories hold directories of the same name, and one
|
// Directory names drawn from one pool, so two repositories hold directories of the same name, and one
|
||||||
@@ -445,3 +586,95 @@ func describe(entries []inventory.Entry) []string {
|
|||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **The merge gate reads the build sources the snapshot carries** (novox/hq ADR 0267): the gate's plan of a
|
||||||
|
// change is the merge handler's, so a snapshot taken by a controller that records build sources narrows the
|
||||||
|
// gate's plan as it narrows the merge's; one without them reads every module as before.
|
||||||
|
func TestTheGatePlansFromTheBuildSourcesTheSnapshotCarries(t *testing.T) {
|
||||||
|
manifest := func(name string) json.RawMessage { return json.RawMessage(`{"module":"` + name + `","version":"1"}`) }
|
||||||
|
facts := snapshot.Facts{Modules: []snapshot.Module{
|
||||||
|
{Name: "mesh-controller", Repository: "novox/mesh-controller", Manifest: manifest("mesh-controller"),
|
||||||
|
Sources: []snapshot.BuildSource{{Own: true, Paths: []string{"module.json", "cmd/mesh-controller/", "internal/broker/"}}}},
|
||||||
|
{Name: "route-proxy", Repository: "novox/mesh-catalog", Path: "modules/route-proxy", Manifest: manifest("route-proxy"),
|
||||||
|
Reads: []string{"novox/mesh-controller"},
|
||||||
|
Sources: []snapshot.BuildSource{{Repository: "novox/mesh-controller", Paths: []string{"examples/route-proxy/", "internal/broker/"}},
|
||||||
|
{Own: true, Paths: []string{"modules/route-proxy/module.json"}}}},
|
||||||
|
}}
|
||||||
|
for paths, want := range map[string]string{
|
||||||
|
"cmd/mesh-controller/main.go": "mesh-controller",
|
||||||
|
"examples/route-proxy/main.go": "route-proxy",
|
||||||
|
"internal/broker/broker.go": "mesh-controller,route-proxy",
|
||||||
|
"README.md": "",
|
||||||
|
} {
|
||||||
|
r, err := reachOfChange(facts, "novox/mesh-controller", []string{paths}, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := tiered(r.Plan.Tiers); got != want {
|
||||||
|
t.Errorf("%s: the gate planned %q, wanted %q", paths, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A snapshot without build sources: as before.
|
||||||
|
for i := range facts.Modules {
|
||||||
|
facts.Modules[i].Sources = nil
|
||||||
|
}
|
||||||
|
r, err := reachOfChange(facts, "novox/mesh-controller", []string{"README.md"}, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := tiered(r.Plan.Tiers); got != "mesh-controller,route-proxy" {
|
||||||
|
t.Errorf("a snapshot without build sources: the gate planned %q for a README", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A plan says why each module is in it** (novox/hq ADR 0267, issue 363): the files of its build source the
|
||||||
|
// merge changed, or why it is read whole — never that it packages a repository as though that moved it.
|
||||||
|
func TestAPlanSaysWhyEachModuleIsInIt(t *testing.T) {
|
||||||
|
const controllerRepo = "http://forge.internal:20000/novox/mesh-controller.git"
|
||||||
|
controller := fromRepo("mesh-controller", controllerRepo, "")
|
||||||
|
agent := fromRepo("build-agent", "http://forge.internal:20000/novox/mesh-catalog.git", "modules/build-agent")
|
||||||
|
gitea := fromRepo("gitea", "http://forge.internal:20000/novox/mesh-catalog.git", "modules/gitea")
|
||||||
|
built := time.Date(2026, 10, 10, 12, 26, 0, 0, time.UTC)
|
||||||
|
read := map[string][]inventory.ReadRepository{
|
||||||
|
"mesh-controller": {{Own: true, Paths: []string{"module.json", "cmd/mesh-controller/", "internal/link/"}, Built: built}},
|
||||||
|
"build-agent": {
|
||||||
|
{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"cmd/mesh-builder/", "internal/link/"}, Built: built},
|
||||||
|
{Own: true, Paths: []string{"modules/build-agent/module.json"}, Built: built},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
merge := func(repo string, paths ...string) link.SourceMoved {
|
||||||
|
return link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Paths: paths}
|
||||||
|
}
|
||||||
|
open := []inventory.Plan{{ID: "plan-1", State: inventory.PlanBuilding, Created: built.Add(time.Minute),
|
||||||
|
Modules: map[string]*inventory.PlanModule{"build-agent": {State: "asked"}}}}
|
||||||
|
for _, c := range []struct {
|
||||||
|
what string
|
||||||
|
e inventory.Entry
|
||||||
|
read map[string][]inventory.ReadRepository
|
||||||
|
m link.SourceMoved
|
||||||
|
says string
|
||||||
|
}{
|
||||||
|
{"the controller's own closure", controller, read, merge("mesh-controller", "README.md", "internal/link/handacts.go"),
|
||||||
|
"its build source changed: 1 changed file(s) in it, e.g. internal/link/handacts.go"},
|
||||||
|
{"the build seat's closure, through its context", agent, read, merge("mesh-controller", "internal/link/handacts.go"),
|
||||||
|
"its build source in novox/mesh-controller changed: 1 changed file(s) in it, e.g. internal/link/handacts.go"},
|
||||||
|
{"an open plan has yet to build it", agent, planningView(read, open), merge("mesh-controller", "cmd/mesh-controller/main.go"),
|
||||||
|
"read whole: plan plan-1 has not built it yet, so every file of novox/mesh-controller, which its build context is, is its build source"},
|
||||||
|
{"nothing recorded, built from its root", controller, nil, merge("mesh-controller", "README.md"),
|
||||||
|
"read whole: no build source recorded, so every file of its repository is its build source"},
|
||||||
|
{"nothing recorded, in its directory", gitea, nil, merge("mesh-catalog", "modules/gitea/index.ts"),
|
||||||
|
"read whole: no build source recorded, so its directory is its build source; e.g. modules/gitea/index.ts"},
|
||||||
|
{"a root manifest is not in a module's directory", gitea, nil, merge("mesh-catalog", "module.json", "modules/gitea/x.ts"),
|
||||||
|
"read whole: no build source recorded, so its directory is its build source; e.g. modules/gitea/x.ts"},
|
||||||
|
{"a context on another branch says nothing of this one", agent, map[string][]inventory.ReadRepository{"build-agent": {
|
||||||
|
{Repository: "novox/mesh-controller", Ref: "release", Paths: []string{"internal/link/"}},
|
||||||
|
{Repository: "novox/mesh-controller", Ref: "main"}}}, merge("mesh-controller", "internal/link/handacts.go"),
|
||||||
|
"read whole: no build source recorded, so every file of novox/mesh-controller, which its build context is, is its build source"},
|
||||||
|
{"files not all said", controller, read, link.SourceMoved{Owner: "novox", Repo: "mesh-controller", PathsTruncated: true,
|
||||||
|
Paths: []string{"x"}}, "read whole: the merge's changed files were not all said"},
|
||||||
|
} {
|
||||||
|
if got := whyMoved(c.e, c.read[c.e.Manifest.Module], c.m); got != c.says {
|
||||||
|
t.Errorf("%s:\n said %q\n wanted %q", c.what, got, c.says)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -25,7 +25,9 @@ import (
|
|||||||
// 1. the machine names an account agents run as (novox/hq ADR 0266), so no agent runs as the operator's
|
// 1. the machine names an account agents run as (novox/hq ADR 0266), so no agent runs as the operator's
|
||||||
// account, which may become root;
|
// account, which may become root;
|
||||||
// 2. its node-engine — running as root, which no agent controls — judged that account unable to become root
|
// 2. its node-engine — running as root, which no agent controls — judged that account unable to become root
|
||||||
// without a person, in a statement heard within the last 15 minutes (agentConfined, judgedConfined);
|
// without a person, in a statement heard within the last 15 minutes (agentConfined, judgedConfined). The
|
||||||
|
// engine gives that verdict only from a complete search for setuid programs younger than mesh-host's
|
||||||
|
// rootsearch.FreshFor; before one, it says "not judged yet", which is no pass (novox/hq issue 361);
|
||||||
// 3. the login shell's `execute` is not served there (novox/hq ADR 0268): its holder's setting withholds it
|
// 3. the login shell's `execute` is not served there (novox/hq ADR 0268): its holder's setting withholds it
|
||||||
// **and** the bus was asked and heard no `execute` answered there. `execute` runs commands as the machine's
|
// **and** the bus was asked and heard no `execute` answered there. `execute` runs commands as the machine's
|
||||||
// runtime account, which the mesh's acting tools give passwordless sudo; that account is taken to become
|
// runtime account, which the mesh's acting tools give passwordless sudo; that account is taken to become
|
||||||
@@ -95,8 +97,8 @@ type rootFacts struct {
|
|||||||
// Execute is whether the login shell's execute is served there; ExecuteWhy why, in words.
|
// Execute is whether the login shell's execute is served there; ExecuteWhy why, in words.
|
||||||
Execute bool
|
Execute bool
|
||||||
ExecuteWhy string
|
ExecuteWhy string
|
||||||
// SearchPending is the agent account unjudged only because the node-engine's first setuid search runs,
|
// SearchPending is the agent account unjudged only because the node-engine's setuid search runs, within
|
||||||
// within its bound (ADR 0266's quiet window).
|
// the quiet the controller gives it (searchQuietFor; ADR 0266's quiet window, issue 361).
|
||||||
SearchPending bool
|
SearchPending bool
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -106,7 +108,7 @@ type rootVerdict struct {
|
|||||||
Free bool `json:"free"`
|
Free bool `json:"free"`
|
||||||
Why string `json:"why"`
|
Why string `json:"why"`
|
||||||
Judged time.Time `json:"judged"`
|
Judged time.Time `json:"judged"`
|
||||||
// Quiet is a machine not free only because its first setuid search still runs, within its bound: the
|
// Quiet is a machine not free only because its setuid search still runs, within searchQuietFor: the
|
||||||
// self-check raises nothing for it then (ADR 0266's quiet window). It is never free for it.
|
// self-check raises nothing for it then (ADR 0266's quiet window). It is never free for it.
|
||||||
Quiet bool `json:"quiet,omitempty"`
|
Quiet bool `json:"quiet,omitempty"`
|
||||||
}
|
}
|
||||||
@@ -136,7 +138,7 @@ func judgeRoot(f rootFacts, now time.Time) rootVerdict {
|
|||||||
}
|
}
|
||||||
if len(not) > 0 {
|
if len(not) > 0 {
|
||||||
v.Why = strings.Join(not, "; ")
|
v.Why = strings.Join(not, "; ")
|
||||||
// The one failure is the agent account not judged yet, because its first search runs.
|
// The one failure is the agent account not judged yet, because its search runs.
|
||||||
v.Quiet = len(not) == 1 && f.SearchPending && f.AgentNamed && !f.Confined && len(f.Unread) == 0 && !f.Execute
|
v.Quiet = len(not) == 1 && f.SearchPending && f.AgentNamed && !f.Confined && len(f.Unread) == 0 && !f.Execute
|
||||||
return v
|
return v
|
||||||
}
|
}
|
||||||
@@ -154,8 +156,8 @@ type rootReader struct {
|
|||||||
asked error
|
asked error
|
||||||
// confined is agentConfined; settings the login shell holder's settings on a machine. Replaceable in a test.
|
// confined is agentConfined; settings the login shell holder's settings on a machine. Replaceable in a test.
|
||||||
confined func(ctx context.Context, node string, now time.Time) (named, confined bool, why string, err error)
|
confined func(ctx context.Context, node string, now time.Time) (named, confined bool, why string, err error)
|
||||||
// quiet says the one thing keeping a machine's agent account unjudged is the node-engine's first setuid
|
// quiet says the one thing keeping a machine's agent account unjudged is the node-engine's setuid
|
||||||
// search, within its bound (ADR 0266, searchStillRunning). Read by the self-check alone, to raise nothing
|
// search, within searchQuietFor (ADR 0266, searchStillRunning). Read by the self-check alone, to raise nothing
|
||||||
// then; nil reads no quiet. It never makes a machine root-free.
|
// then; nil reads no quiet. It never makes a machine root-free.
|
||||||
quiet func(ctx context.Context, node string, now time.Time) bool
|
quiet func(ctx context.Context, node string, now time.Time) bool
|
||||||
settings func(ctx context.Context, node, module string) ([]catalogue.Layer, error)
|
settings func(ctx context.Context, node, module string) ([]catalogue.Layer, error)
|
||||||
@@ -304,8 +306,8 @@ func probeAgentRoot(ctx context.Context, d *doctor) ([]conditions.Observation, e
|
|||||||
if r.read != nil {
|
if r.read != nil {
|
||||||
return nil, r.read
|
return nil, r.read
|
||||||
}
|
}
|
||||||
// The self-check alone reads ADR 0266's quiet window: nothing raised while a machine's first setuid search
|
// The self-check alone reads ADR 0266's quiet window: nothing raised while a machine's setuid search
|
||||||
// runs, within its bound. The root-free verb never reads it, so the machine still answers not free.
|
// runs, within searchQuietFor. The root-free verb never reads it, so the machine still answers not free.
|
||||||
r.quiet = func(ctx context.Context, node string, now time.Time) bool {
|
r.quiet = func(ctx context.Context, node string, now time.Time) bool {
|
||||||
n, err := inv.NodeByName(ctx, node)
|
n, err := inv.NodeByName(ctx, node)
|
||||||
if err != nil || n.AgentAccount == "" {
|
if err != nil || n.AgentAccount == "" {
|
||||||
|
|||||||
@@ -201,7 +201,7 @@ func TestRootFreeIsAnsweredOnlyByTheServingController(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// The confirmation review of 2026-10-09: ADR 0266's quiet window (#175) keeps the self-check from raising
|
// The confirmation review of 2026-10-09: ADR 0266's quiet window (#175) keeps the self-check from raising
|
||||||
// agent-can-become-root while the node-engine's first setuid search runs. It must not make root-free answer free:
|
// agent-can-become-root while the node-engine's setuid search runs and no complete one judges. It must not make root-free answer free:
|
||||||
// root-free needs a complete, fresh verdict. A verdict still waiting for the search is "not judged" to
|
// root-free needs a complete, fresh verdict. A verdict still waiting for the search is "not judged" to
|
||||||
// agentConfined, so the machine is not root-free, whatever the quiet says — and the same statement, complete
|
// agentConfined, so the machine is not root-free, whatever the quiet says — and the same statement, complete
|
||||||
// and healthy, is the control.
|
// and healthy, is the control.
|
||||||
@@ -224,12 +224,12 @@ func TestAMachineWaitingForItsFirstSetuidSearchIsNotRootFree(t *testing.T) {
|
|||||||
t.Fatal("the statement is not one the quiet window counts as waiting for the search")
|
t.Fatal("the statement is not one the quiet window counts as waiting for the search")
|
||||||
}
|
}
|
||||||
if v := judged(pending); v.Free {
|
if v := judged(pending); v.Free {
|
||||||
t.Errorf("a machine whose first setuid search is pending was judged root-free: %+v", v)
|
t.Errorf("a machine whose setuid search is pending was judged root-free: %+v", v)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The confirmation review of 2026-10-09, on #154 beside ADR 0266: D-root keeps ADR 0266's quiet window — nothing
|
// The confirmation review of 2026-10-09, on #154 beside ADR 0266: D-root keeps ADR 0266's quiet window — nothing
|
||||||
// raised while the one thing unjudged is the first setuid search, within its bound — while the root-free verb
|
// raised while the one thing unjudged is the setuid search, within searchQuietFor — while the root-free verb
|
||||||
// still answers the machine not free; and D-root's condition has a key of its own, apart from DA's.
|
// still answers the machine not free; and D-root's condition has a key of its own, apart from DA's.
|
||||||
func TestRootNotFreeIsQuietWhileTheFirstSearchRunsAndKeyedApartFromDA(t *testing.T) {
|
func TestRootNotFreeIsQuietWhileTheFirstSearchRunsAndKeyedApartFromDA(t *testing.T) {
|
||||||
entries := []inventory.Entry{{Manifest: catalogue.Manifest{Module: "telegram", RunsAs: "telegram"}, On: []string{"anchor"}}}
|
entries := []inventory.Entry{{Manifest: catalogue.Manifest{Module: "telegram", RunsAs: "telegram"}, On: []string{"anchor"}}}
|
||||||
@@ -241,21 +241,21 @@ func TestRootNotFreeIsQuietWhileTheFirstSearchRunsAndKeyedApartFromDA(t *testing
|
|||||||
quiet: func(context.Context, string, time.Time) bool { return true }}
|
quiet: func(context.Context, string, time.Time) bool { return true }}
|
||||||
trusted := trustedMachines(entries)
|
trusted := trustedMachines(entries)
|
||||||
if got := rootObservations(context.Background(), r, trusted, rootNow); len(got) != 0 {
|
if got := rootObservations(context.Background(), r, trusted, rootNow); len(got) != 0 {
|
||||||
t.Errorf("raised while the first search runs: %+v", got)
|
t.Errorf("raised while the search runs: %+v", got)
|
||||||
}
|
}
|
||||||
if v := judgeRootFree(context.Background(), r, []string{"anchor"}, rootNow)[0]; v.Free || !v.Quiet {
|
if v := judgeRootFree(context.Background(), r, []string{"anchor"}, rootNow)[0]; v.Free || !v.Quiet {
|
||||||
t.Errorf("root-free while the first search runs: %+v", v)
|
t.Errorf("root-free while the search runs: %+v", v)
|
||||||
}
|
}
|
||||||
// Quiet hides nothing else: the login shell served as well is said.
|
// Quiet hides nothing else: the login shell served as well is said.
|
||||||
r.heard = map[string]map[string]map[string]bool{loginShellSeat: {"execute": {"anchor": true}}}
|
r.heard = map[string]map[string]map[string]bool{loginShellSeat: {"execute": {"anchor": true}}}
|
||||||
if got := rootObservations(context.Background(), r, trusted, rootNow); len(got) != 1 {
|
if got := rootObservations(context.Background(), r, trusted, rootNow); len(got) != 1 {
|
||||||
t.Errorf("a second failure was kept quiet: %+v", got)
|
t.Errorf("a second failure was kept quiet: %+v", got)
|
||||||
}
|
}
|
||||||
// Past its bound, said.
|
// Past searchQuietFor, said.
|
||||||
r.heard, r.quiet = map[string]map[string]map[string]bool{}, func(context.Context, string, time.Time) bool { return false }
|
r.heard, r.quiet = map[string]map[string]map[string]bool{}, func(context.Context, string, time.Time) bool { return false }
|
||||||
got := rootObservations(context.Background(), r, trusted, rootNow)
|
got := rootObservations(context.Background(), r, trusted, rootNow)
|
||||||
if len(got) != 1 {
|
if len(got) != 1 {
|
||||||
t.Fatalf("a search past its bound was not said: %+v", got)
|
t.Fatalf("a search past searchQuietFor was not said: %+v", got)
|
||||||
}
|
}
|
||||||
// One key per judgement: DA's is machine.<m>.agent-root, this one its own.
|
// One key per judgement: DA's is machine.<m>.agent-root, this one its own.
|
||||||
da := conditions.Observation{Scope: conditions.ScopeMachine, ID: "anchor", Token: "agent-root", Machine: "anchor"}
|
da := conditions.Observation{Scope: conditions.ScopeMachine, ID: "anchor", Token: "agent-root", Machine: "anchor"}
|
||||||
|
|||||||
@@ -0,0 +1,793 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// A trusted setting proposed through a verb and set only on the operator's warrant (novox/hq ADR 0277).
|
||||||
|
//
|
||||||
|
// mesh-controller settings propose <module> <values.json | {…}> [--node <node>] [--replace]
|
||||||
|
// mesh-controller settings propose <module> --clear [--node <node>]
|
||||||
|
// mesh-controller settings proposals [<id>]
|
||||||
|
//
|
||||||
|
// Whoever the bus admits may PROPOSE a settings layer — the keys issue 339 made the terminal's (`places`,
|
||||||
|
// `accesses`, what a provider serves, what a trusted file asks for) among them. A proposal changes nothing: it is
|
||||||
|
// kept in the controller's own asks (broker.AskedBucket, written by the controller alone) and asked of the
|
||||||
|
// operator through the operator channel as an ask whose two answers, Approve and Decline, are both at the level
|
||||||
|
// approve, so only a channel that proves who answered (Telegram, today) carries either. The serving controller
|
||||||
|
// acts on the warrant as on any other of its asks (asker.Decided): once, for the ask it holds, the option it
|
||||||
|
// offered, and only when the act about to be performed — the module, the machine, the digest of the exact values,
|
||||||
|
// the layer they replace, whether a removal is meant — is the one the option bound when the operator was shown
|
||||||
|
// it. Then it sets the layer as `settings set` at the terminal does, with the same judgement, keeps who approved
|
||||||
|
// it beside the layer (`settings` says it back), and records the warrant in the hand-act log on the bus, where a
|
||||||
|
// person's decisions are read; the router edits the ask on every channel to its outcome. No seat event of its
|
||||||
|
// own: nothing consumes one, and the installer's first user list in the node-engine's repository names every
|
||||||
|
// controller event, so one would cost a node-engine change for a fact without a reader. The push afterwards is a
|
||||||
|
// separate act, as it is for a layer set at the terminal.
|
||||||
|
//
|
||||||
|
// **What the operator reads** is the module, the machine, each key with its exact new value and, for a changed
|
||||||
|
// key, the value it replaces. A value that may not leave the mesh (an address, a path, a secret's shape: the
|
||||||
|
// router's content rule, outward.Check) is shown with that part replaced by ‹address›, ‹path› or ‹withheld›, the
|
||||||
|
// ask says so, and the whole is read with `settings proposals <id>` — whose fingerprint must be the one on the
|
||||||
|
// phone. Fail closed: a proposal nothing can carry to the operator is refused at once, in words, and never left
|
||||||
|
// waiting for an answer that cannot come.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
"github.com/novox/mesh-controller/internal/outward"
|
||||||
|
)
|
||||||
|
|
||||||
|
// settingsProposal is one proposed change to a module's settings layer, as the controller's ask keeps it
|
||||||
|
// (asked.Proposal). Every field the ask is composed from is here, so the serving controller composes the same ask
|
||||||
|
// the proposer did, and the warrant's digest holds to it.
|
||||||
|
type settingsProposal struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
// Node is the machine, or empty for the whole mesh.
|
||||||
|
Node string `json:"node,omitempty"`
|
||||||
|
// Values is the layer proposed, whole; Clear says the layer is removed instead.
|
||||||
|
Values map[string]any `json:"values,omitempty"`
|
||||||
|
Clear bool `json:"clear,omitempty"`
|
||||||
|
// Replace says the keys the layer had and Values do not name are meant to go (novox/hq ADR 0217).
|
||||||
|
Replace bool `json:"replace,omitempty"`
|
||||||
|
// Before is the layer as it stood when the proposal was made, and HadLayer whether there was one: what the
|
||||||
|
// operator was shown the change against, and what must still stand when the warrant is acted on.
|
||||||
|
Before map[string]any `json:"before,omitempty"`
|
||||||
|
HadLayer bool `json:"had-layer"`
|
||||||
|
// From is who proposed it, as the bus named the caller; At is when.
|
||||||
|
From string `json:"from"`
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
// Digest is the digest of Values (layerDigest), BeforeDigest of Before.
|
||||||
|
Digest string `json:"digest"`
|
||||||
|
BeforeDigest string `json:"before-digest"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// proposalVerb is the verb a proposal's answers bind: the controller's own settings, performed by itself.
|
||||||
|
const proposalVerb = askerName + ".settings"
|
||||||
|
|
||||||
|
// The two answers, both at the level approve.
|
||||||
|
const (
|
||||||
|
answerApprove = "approve"
|
||||||
|
answerDecline = "decline"
|
||||||
|
)
|
||||||
|
|
||||||
|
// layerDigest is the digest a proposal binds: SHA-256 over the layer's canonical JSON (Go sorts a map's keys), an
|
||||||
|
// absent layer and an empty one alike.
|
||||||
|
func layerDigest(values map[string]any) string {
|
||||||
|
if values == nil {
|
||||||
|
values = map[string]any{}
|
||||||
|
}
|
||||||
|
raw, _ := json.Marshal(values)
|
||||||
|
sum := sha256.Sum256(raw)
|
||||||
|
return "sha256:" + hex.EncodeToString(sum[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
// fingerprint is a digest as the operator is shown it: its first 24 hexadecimal digits in groups of four, so no
|
||||||
|
// word of it is long enough for the router to take for a secret.
|
||||||
|
func fingerprint(digest string) string {
|
||||||
|
hexed := strings.TrimPrefix(digest, "sha256:")
|
||||||
|
if len(hexed) < 24 {
|
||||||
|
return hexed
|
||||||
|
}
|
||||||
|
var groups []string
|
||||||
|
for i := 0; i < 24; i += 4 {
|
||||||
|
groups = append(groups, hexed[i:i+4])
|
||||||
|
}
|
||||||
|
return strings.Join(groups, " ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// where is the layer in words: "shanks" or "the whole mesh".
|
||||||
|
func (p settingsProposal) where() string {
|
||||||
|
if p.Node == "" {
|
||||||
|
return "the whole mesh"
|
||||||
|
}
|
||||||
|
return p.Node
|
||||||
|
}
|
||||||
|
|
||||||
|
// about is what the ask is about, a key without spaces: the module's layer on the machine, or on the mesh.
|
||||||
|
func (p settingsProposal) about() string {
|
||||||
|
if p.Node == "" {
|
||||||
|
return "settings." + p.Module + ".mesh"
|
||||||
|
}
|
||||||
|
return "settings." + p.Module + "." + p.Node
|
||||||
|
}
|
||||||
|
|
||||||
|
// actions are the proposal's two answers as the controller keeps them (asked.Actions): each binds the exact act
|
||||||
|
// through boundAct — the verb, the machine, the level and every argument, the values' digest among them.
|
||||||
|
func (p settingsProposal) actions(id string) []conditions.Action {
|
||||||
|
args := func(answer string) map[string]string {
|
||||||
|
return map[string]string{"proposal": id, "answer": answer, "module": p.Module, "node": p.Node,
|
||||||
|
"values": p.Digest, "before": p.BeforeDigest, "had-layer": strconv.FormatBool(p.HadLayer),
|
||||||
|
"replace": strconv.FormatBool(p.Replace), "clear": strconv.FormatBool(p.Clear), "from": p.From,
|
||||||
|
"at": p.At.UTC().Format(time.RFC3339Nano)}
|
||||||
|
}
|
||||||
|
return []conditions.Action{
|
||||||
|
{Label: "Approve", Verb: proposalVerb, Machine: p.Node, Level: conditions.LevelApprove, Arguments: args(answerApprove)},
|
||||||
|
{Label: "Decline", Verb: proposalVerb, Machine: p.Node, Level: conditions.LevelApprove, Arguments: args(answerDecline)},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ask is the proposal's ask, composed from it alone, as the router is sent it: the headline, the explanation
|
||||||
|
// with the change shown under the content rule, and the two options with their bound acts.
|
||||||
|
func (p settingsProposal) ask(id string, machines []string) (asks.Ask, map[string]int) {
|
||||||
|
verb := "Set"
|
||||||
|
if p.Clear {
|
||||||
|
verb = "Clear"
|
||||||
|
}
|
||||||
|
headline := fmt.Sprintf("%s %s on %s?", verb, p.Module, p.where())
|
||||||
|
if len([]rune(headline)) > asks.HeadlineLength {
|
||||||
|
headline = fmt.Sprintf("%s settings on %s?", verb, p.where())
|
||||||
|
}
|
||||||
|
if len([]rune(headline)) > asks.HeadlineLength {
|
||||||
|
headline = verb + " settings?"
|
||||||
|
}
|
||||||
|
shown, whole := p.change(machines)
|
||||||
|
var b strings.Builder
|
||||||
|
if p.Clear {
|
||||||
|
fmt.Fprintf(&b, "Clear the settings of %s on %s, back to what the module says?\n\n", p.Module, p.where())
|
||||||
|
} else {
|
||||||
|
fmt.Fprintf(&b, "Set the settings of %s on %s to these values?\n\n", p.Module, p.where())
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&b, "Proposed by %s, at %s. ", sayable(p.From, machines), p.At.Local().Format("15:04 on 2 Jan"))
|
||||||
|
switch {
|
||||||
|
case p.Clear && p.HadLayer:
|
||||||
|
b.WriteString("The layer it removes:\n\n")
|
||||||
|
case p.Clear:
|
||||||
|
b.WriteString("There is no layer to remove; approving changes nothing.")
|
||||||
|
case !p.HadLayer:
|
||||||
|
b.WriteString("There is no layer yet; this is the whole of it:\n\n")
|
||||||
|
default:
|
||||||
|
b.WriteString("The layer is replaced whole; what changes against it:\n\n")
|
||||||
|
}
|
||||||
|
b.WriteString(shown)
|
||||||
|
fmt.Fprintf(&b, "\n\nFingerprint %s.", fingerprint(p.Digest))
|
||||||
|
if !whole {
|
||||||
|
b.WriteString(" Parts shown as ‹address›, ‹path› or ‹withheld› may not leave the mesh: read it whole, with " +
|
||||||
|
"this fingerprint, through the mesh MCP server (mesh-controller.settings, proposal " + id + ") or with " +
|
||||||
|
"mesh-cli settings proposals " + id + ".")
|
||||||
|
}
|
||||||
|
if p.Clear {
|
||||||
|
b.WriteString(" Approved, the layer is removed at once and the machine takes it at its next push.")
|
||||||
|
} else {
|
||||||
|
b.WriteString(" Approved, the layer is set at once and the machine takes it at its next push.")
|
||||||
|
}
|
||||||
|
q := asks.Ask{ID: id, Headline: headline, Explanation: b.String(), Who: asks.Operator,
|
||||||
|
Expires: p.At.Add(askApproveFor), OnExpiry: "the proposal is discarded; nothing changes",
|
||||||
|
About: p.about()}
|
||||||
|
options := map[string]int{}
|
||||||
|
for i, act := range p.actions(id) {
|
||||||
|
binds, _ := asks.ActDigest(boundAct(act))
|
||||||
|
oid := optionID(act.Label)
|
||||||
|
options[oid] = i
|
||||||
|
does := "the settings are set; nothing is pushed yet"
|
||||||
|
if p.Clear {
|
||||||
|
does = "the layer is removed; nothing is pushed yet"
|
||||||
|
}
|
||||||
|
if act.Arguments["answer"] == answerDecline {
|
||||||
|
does = "nothing changes; the proposal is discarded"
|
||||||
|
}
|
||||||
|
q.Options = append(q.Options, asks.Option{ID: oid, Label: act.Label, Does: does, Level: asks.Level(act.Level),
|
||||||
|
Binds: binds})
|
||||||
|
}
|
||||||
|
return q, options
|
||||||
|
}
|
||||||
|
|
||||||
|
// shownMost is the most of a change the phone is shown, in bytes; the rest is read whole with `settings proposals`.
|
||||||
|
const shownMost = 1400
|
||||||
|
|
||||||
|
// change is what changes, line by line, each value shown under the content rule, and whether every value was
|
||||||
|
// shown whole: "+ key: value" added, "~ key: value (was: old)" changed, "- key (was: old)" removed, and the
|
||||||
|
// count of keys unchanged.
|
||||||
|
func (p settingsProposal) change(machines []string) (string, bool) {
|
||||||
|
whole := true
|
||||||
|
say := func(v any) string {
|
||||||
|
s, w := sayableValue(v, machines)
|
||||||
|
whole = whole && w
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
var lines []string
|
||||||
|
if p.Clear {
|
||||||
|
was := leaves(p.Before, "")
|
||||||
|
for _, k := range layerKeys(was) {
|
||||||
|
lines = append(lines, fmt.Sprintf("- %s: %s", k, say(was[k])))
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
added, changed, removed := settingsChange(p.Before, p.Values)
|
||||||
|
was, now := leaves(p.Before, ""), leaves(p.Values, "")
|
||||||
|
for _, k := range added {
|
||||||
|
lines = append(lines, fmt.Sprintf("+ %s: %s", k, say(now[k])))
|
||||||
|
}
|
||||||
|
for _, k := range changed {
|
||||||
|
lines = append(lines, fmt.Sprintf("~ %s: %s (was: %s)", k, say(now[k]), say(was[k])))
|
||||||
|
}
|
||||||
|
for _, k := range removed {
|
||||||
|
lines = append(lines, fmt.Sprintf("- %s (was: %s)", k, say(was[k])))
|
||||||
|
}
|
||||||
|
unchanged := len(now) - len(added) - len(changed)
|
||||||
|
switch {
|
||||||
|
case len(lines) == 0 && unchanged > 0:
|
||||||
|
lines = append(lines, fmt.Sprintf("= nothing changes: the %d key(s) are as they stand", unchanged))
|
||||||
|
case unchanged > 0:
|
||||||
|
lines = append(lines, fmt.Sprintf("= %d key(s) unchanged", unchanged))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out := strings.Join(lines, "\n")
|
||||||
|
if len(out) > shownMost {
|
||||||
|
cut := shownMost
|
||||||
|
for cut > 0 && out[cut] != '\n' {
|
||||||
|
cut--
|
||||||
|
}
|
||||||
|
out = out[:cut] + fmt.Sprintf("\n… NOT SHOWN IN FULL here: %d more bytes", len(strings.Join(lines, "\n"))-cut)
|
||||||
|
whole = false
|
||||||
|
}
|
||||||
|
return out, whole
|
||||||
|
}
|
||||||
|
|
||||||
|
func layerKeys(m map[string]any) []string {
|
||||||
|
keys := make([]string, 0, len(m))
|
||||||
|
for k := range m {
|
||||||
|
keys = append(keys, k)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
return keys
|
||||||
|
}
|
||||||
|
|
||||||
|
// The shapes a value is shown without, in place: an address with what follows it up to a separator, and a
|
||||||
|
// path. Replaced in place rather than word by word, so "recalbox=smb://host/share@/mnt/recalbox" is shown as
|
||||||
|
// "recalbox=‹address›@‹path›" and keeps its shape.
|
||||||
|
var (
|
||||||
|
shownURL = regexp.MustCompile(`(?i)\b[a-z][a-z0-9+.-]*://[^\s@"',;)\]}]*`)
|
||||||
|
shownIPv4 = regexp.MustCompile(`\b\d{1,3}(\.\d{1,3}){3}(:\d+)?\b`)
|
||||||
|
shownEmail = regexp.MustCompile(`[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}`)
|
||||||
|
shownPath = regexp.MustCompile(`(^|[\s=@,;:"'(\[{])((?:~|\.{1,2})?/[^\s"',;:)\]}]*)`)
|
||||||
|
)
|
||||||
|
|
||||||
|
// Markers for what is not shown.
|
||||||
|
const (
|
||||||
|
markAddress = "‹address›"
|
||||||
|
markPath = "‹path›"
|
||||||
|
markWithheld = "‹withheld›"
|
||||||
|
)
|
||||||
|
|
||||||
|
// sayableValue is a value as the phone is shown it, and whether it was shown whole. A string is shown bare; any
|
||||||
|
// other value as JSON.
|
||||||
|
func sayableValue(v any, machines []string) (string, bool) {
|
||||||
|
text, ok := v.(string)
|
||||||
|
if !ok {
|
||||||
|
raw, err := json.Marshal(v)
|
||||||
|
if err != nil {
|
||||||
|
return markWithheld, false
|
||||||
|
}
|
||||||
|
text = string(raw)
|
||||||
|
}
|
||||||
|
if text == "" {
|
||||||
|
return `""`, true
|
||||||
|
}
|
||||||
|
out := sayable(text, machines)
|
||||||
|
return out, out == text
|
||||||
|
}
|
||||||
|
|
||||||
|
// sayable is a text with what may not leave the mesh replaced in place by a marker; what the markers cannot make
|
||||||
|
// pass is withheld whole.
|
||||||
|
func sayable(text string, machines []string) string {
|
||||||
|
if _, ok := outward.Check(text, machines...); ok {
|
||||||
|
return text
|
||||||
|
}
|
||||||
|
out := shownURL.ReplaceAllString(text, markAddress)
|
||||||
|
out = shownEmail.ReplaceAllString(out, markAddress)
|
||||||
|
out = shownIPv4.ReplaceAllString(out, markAddress)
|
||||||
|
out = shownPath.ReplaceAllString(out, "${1}"+markPath)
|
||||||
|
// Then word by word, as the router reads them: a host name, a path the shapes above missed, a secret's shape.
|
||||||
|
words := strings.FieldsFunc(out, func(r rune) bool {
|
||||||
|
return r == ' ' || r == '\t' || r == '\n' || strings.ContainsRune("\"'`()[]{}<>,;|", r)
|
||||||
|
})
|
||||||
|
for _, w := range words {
|
||||||
|
if refusal, ok := outward.Check(w, machines...); !ok {
|
||||||
|
mark := markWithheld
|
||||||
|
switch refusal.Class {
|
||||||
|
case "address":
|
||||||
|
mark = markAddress
|
||||||
|
case "path":
|
||||||
|
mark = markPath
|
||||||
|
}
|
||||||
|
out = strings.ReplaceAll(out, w, mark)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, ok := outward.Check(out, machines...); ok {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
out = strings.ReplaceAll(outward.Scrub(out, markWithheld, machines...), "(withheld)", markWithheld)
|
||||||
|
if _, ok := outward.Check(out, machines...); ok {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
return markWithheld
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- proposing ---------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// proposer is the propose command's reaches, given so a test needs no store and no bus.
|
||||||
|
type proposer struct {
|
||||||
|
// layer reads a module's layer as it stands.
|
||||||
|
layer func(ctx context.Context, node, module string) (map[string]any, bool, error)
|
||||||
|
// judge judges the layer as SetSettings would, keeping nothing.
|
||||||
|
judge func(ctx context.Context, node, module string, values map[string]any) error
|
||||||
|
// machines are the mesh's machine names: allowed in the ask's words.
|
||||||
|
machines func(ctx context.Context) ([]string, error)
|
||||||
|
store askedStore
|
||||||
|
publish func(ctx context.Context, subject string, body []byte, id string) error
|
||||||
|
// routerHere and grantHeld are the asker's own judgements of whether an ask can be carried; nil is yes.
|
||||||
|
routerHere func(ctx context.Context) (bool, error)
|
||||||
|
grantHeld func(ctx context.Context) (bool, string, error)
|
||||||
|
// routerRecord reads the router's record of an ask: its state, or "" for none.
|
||||||
|
routerRecord func(ctx context.Context, id string) (string, error)
|
||||||
|
now func() time.Time
|
||||||
|
caller string
|
||||||
|
// waitFor and waitEvery bound how long propose waits for the router's word on the new ask.
|
||||||
|
waitFor time.Duration
|
||||||
|
waitEvery time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// proposeInput is what is proposed.
|
||||||
|
type proposeInput struct {
|
||||||
|
module string
|
||||||
|
node string
|
||||||
|
values map[string]any
|
||||||
|
clear bool
|
||||||
|
replace bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// atTheTerminalInstead names the other way, said whenever a proposal is refused for want of a channel.
|
||||||
|
func atTheTerminalInstead(in proposeInput) string {
|
||||||
|
if in.clear {
|
||||||
|
return "the operator may clear it at the controller's terminal instead: mesh-cli settings clear " + in.module + nodeFlag(in.node)
|
||||||
|
}
|
||||||
|
return "the operator may set it at the controller's terminal instead: mesh-cli settings set " + in.module + " '{…}'" + nodeFlag(in.node)
|
||||||
|
}
|
||||||
|
|
||||||
|
// propose keeps a proposal in the controller's asks and asks the operator; it answers the words said to the
|
||||||
|
// caller. Nothing is set here.
|
||||||
|
func (pr proposer) propose(ctx context.Context, in proposeInput) (string, error) {
|
||||||
|
refuse := func(format string, args ...any) (string, error) {
|
||||||
|
return "", fmt.Errorf(format+". Nothing was proposed", args...)
|
||||||
|
}
|
||||||
|
if in.module == "" {
|
||||||
|
return refuse("a proposal names a module")
|
||||||
|
}
|
||||||
|
if !in.clear && in.values == nil {
|
||||||
|
return refuse("a proposal gives the values, or says --clear")
|
||||||
|
}
|
||||||
|
now := pr.now()
|
||||||
|
before, had, err := pr.layer(ctx, in.node, in.module)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
p := settingsProposal{Module: in.module, Node: in.node, Values: in.values, Clear: in.clear, Replace: in.replace,
|
||||||
|
Before: before, HadLayer: had, From: pr.caller, At: now, BeforeDigest: layerDigest(before)}
|
||||||
|
if in.clear {
|
||||||
|
// A clear binds the layer it removes: its digest is the fingerprint the operator reads.
|
||||||
|
p.Values, p.Digest = nil, layerDigest(before)
|
||||||
|
} else {
|
||||||
|
// Judged now, as SetSettings judges, so the operator is never asked about a layer the mesh would refuse —
|
||||||
|
// and judged again when the warrant is acted on.
|
||||||
|
if err := pr.judge(ctx, in.node, in.module, in.values); err != nil {
|
||||||
|
return refuse("%v", err)
|
||||||
|
}
|
||||||
|
_, _, removed := settingsChange(before, in.values)
|
||||||
|
if len(removed) > 0 && !in.replace {
|
||||||
|
return refuse("%s on %s: this layer would no longer set %s. A layer is replaced whole; read it with "+
|
||||||
|
"`settings show %s%s` and include what should stay, or add --replace if the removal is meant "+
|
||||||
|
"(novox/hq ADR 0217)", in.module, p.where(), strings.Join(removed, ", "), in.module, nodeFlag(in.node))
|
||||||
|
}
|
||||||
|
p.Digest = layerDigest(in.values)
|
||||||
|
}
|
||||||
|
var machines []string
|
||||||
|
if pr.machines != nil {
|
||||||
|
if machines, err = pr.machines(ctx); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
id := newProposalID()
|
||||||
|
q, options := p.ask(id, machines)
|
||||||
|
if err := q.Check(now); err != nil {
|
||||||
|
return refuse("%v", err)
|
||||||
|
}
|
||||||
|
words := []string{q.Headline, q.Explanation, q.OnExpiry}
|
||||||
|
for _, o := range q.Options {
|
||||||
|
words = append(words, o.Label, o.Does)
|
||||||
|
}
|
||||||
|
if refusal, ok := outward.Check(strings.Join(words, "\n"), machines...); !ok {
|
||||||
|
return refuse("the ask's words would carry %s, which may not leave the mesh, and the change could not be "+
|
||||||
|
"shown without it; %s", refusal, atTheTerminalInstead(in))
|
||||||
|
}
|
||||||
|
// Fail closed, before anything is kept: no router, no grant, no channel means no ask.
|
||||||
|
if pr.routerHere != nil {
|
||||||
|
here, err := pr.routerHere(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if !here {
|
||||||
|
return refuse("no router takes the controller's asks (no module holding the operator channel is assigned), "+
|
||||||
|
"so the operator cannot be asked; %s", atTheTerminalInstead(in))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if pr.grantHeld != nil {
|
||||||
|
held, why, err := pr.grantHeld(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if !held {
|
||||||
|
return refuse("the operator cannot be asked yet: %s; %s", why, atTheTerminalInstead(in))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
all, err := pr.store.All(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
open := 0
|
||||||
|
for _, r := range all {
|
||||||
|
if r.State != askOpen || !now.Before(r.Ask.Expires) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if r.Proposal != nil && r.Proposal.Module == p.Module && r.Proposal.Node == p.Node && r.Proposal.Digest == p.Digest &&
|
||||||
|
r.Proposal.Clear == p.Clear {
|
||||||
|
return refuse("the same change is already proposed as %s and waits for the operator's answer until %s; "+
|
||||||
|
"`settings proposals` lists it", r.ID, r.Ask.Expires.Local().Format("15:04"))
|
||||||
|
}
|
||||||
|
open++
|
||||||
|
}
|
||||||
|
if open >= askMostOpen {
|
||||||
|
return refuse("%d questions already wait for the operator's answer, and the operator is asked at most %d at "+
|
||||||
|
"once; `settings proposals` lists the proposals among them", open, askMostOpen)
|
||||||
|
}
|
||||||
|
// Kept before it is published, as the asker keeps every ask, so a warrant always finds it.
|
||||||
|
if err := pr.store.Create(ctx, asked{ID: id, Condition: q.About, Ask: q, Actions: p.actions(id), Options: options,
|
||||||
|
State: askOpen, Opened: now, Proposal: &p}); err != nil {
|
||||||
|
return "", fmt.Errorf("the proposal could not be kept in the controller's asks, so the operator was not asked: %w", err)
|
||||||
|
}
|
||||||
|
body, err := json.Marshal(q)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if err := pr.publish(ctx, asks.AskSubject(askerName), body, "ask."+id); err != nil {
|
||||||
|
_, _ = pr.store.Change(ctx, id, func(x *asked) bool {
|
||||||
|
if x.State != askOpen || x.Acted != "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
x.State, x.Ended, x.Acted = askUnsent, pr.now(), "nothing: it could not be published: "+err.Error()
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
return "", fmt.Errorf("the operator could not be asked (%v); the proposal %s is kept and will never become "+
|
||||||
|
"active. Propose it again, or %s", err, id, atTheTerminalInstead(in))
|
||||||
|
}
|
||||||
|
// The router's word, before answering: it refuses at once an ask no channel can carry (the serving controller
|
||||||
|
// hears that and ends the ask here), and records one it took.
|
||||||
|
taken := "the router has not said yet whether it took the ask; `settings proposals` shows where it stands"
|
||||||
|
for deadline := time.Now().Add(pr.waitFor); time.Now().Before(deadline); {
|
||||||
|
if r, err := pr.store.Get(ctx, id); err == nil && r != nil && r.State != askOpen {
|
||||||
|
why := r.Acted
|
||||||
|
if r.Warrant != nil && r.Warrant.Words != "" {
|
||||||
|
why = r.Warrant.Words
|
||||||
|
}
|
||||||
|
return "", fmt.Errorf("the router did not ask the operator: the ask %s %s (%s). Nothing changes; %s",
|
||||||
|
id, r.State, why, atTheTerminalInstead(in))
|
||||||
|
}
|
||||||
|
if pr.routerRecord != nil {
|
||||||
|
if state, err := pr.routerRecord(ctx, id); err == nil && state != "" {
|
||||||
|
taken = "the router took the ask and shows it on the channels that can carry it"
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
time.Sleep(pr.waitEvery)
|
||||||
|
}
|
||||||
|
var b strings.Builder
|
||||||
|
fmt.Fprintf(&b, "proposal %s: %s\n", id, q.Headline)
|
||||||
|
fmt.Fprintf(&b, " %s\n", taken)
|
||||||
|
fmt.Fprintf(&b, " the operator is asked on a channel that proves who answers, and reads the change with fingerprint %s\n",
|
||||||
|
fingerprint(p.Digest))
|
||||||
|
fmt.Fprintf(&b, " until %s nothing changes: the layer is set only on Approve, and discarded on Decline or at expiry\n",
|
||||||
|
q.Expires.Local().Format("2006-01-02 15:04"))
|
||||||
|
fmt.Fprintf(&b, " `settings proposals %s` shows it whole; once approved, `push %s` sends it", id, pushWord(p.Node))
|
||||||
|
return b.String(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func pushWord(node string) string {
|
||||||
|
if node == "" {
|
||||||
|
return "--behind"
|
||||||
|
}
|
||||||
|
return node
|
||||||
|
}
|
||||||
|
|
||||||
|
func newProposalID() string {
|
||||||
|
return "s" + strings.TrimPrefix(newAskID(), "c")
|
||||||
|
}
|
||||||
|
|
||||||
|
// How long propose waits for the router's word on a new ask, and how often it looks.
|
||||||
|
const (
|
||||||
|
routerAnswersWithin = 8 * time.Second
|
||||||
|
routerAnswersEvery = 250 * time.Millisecond
|
||||||
|
)
|
||||||
|
|
||||||
|
// proposeCommand is `settings propose`, on this controller's stores and bus.
|
||||||
|
func proposeCommand(ctx context.Context, module, node, valuesArg string, clear, replace bool) error {
|
||||||
|
var values map[string]any
|
||||||
|
if !clear {
|
||||||
|
if valuesArg == "" {
|
||||||
|
return errors.New("settings propose <module> <settings.json | {…}> [--node <node>] [--replace], or settings propose <module> --clear [--node <node>]")
|
||||||
|
}
|
||||||
|
var raw []byte
|
||||||
|
var err error
|
||||||
|
if strings.HasPrefix(strings.TrimSpace(valuesArg), "{") {
|
||||||
|
raw = []byte(valuesArg)
|
||||||
|
} else if raw, err = os.ReadFile(valuesArg); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &values); err != nil {
|
||||||
|
return fmt.Errorf("%s is not a settings file: %w", valuesArg, err)
|
||||||
|
}
|
||||||
|
} else if valuesArg != "" {
|
||||||
|
return errors.New("settings propose: --clear takes no values")
|
||||||
|
}
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
js, err := aBus()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
conn := js.Conn()
|
||||||
|
pr := proposer{
|
||||||
|
layer: open.inventory.Layer,
|
||||||
|
judge: open.inventory.JudgeSettings,
|
||||||
|
machines: func(ctx context.Context) ([]string, error) {
|
||||||
|
nodes, err := open.inventory.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var names []string
|
||||||
|
for _, n := range nodes {
|
||||||
|
names = append(names, n.Name)
|
||||||
|
}
|
||||||
|
return names, nil
|
||||||
|
},
|
||||||
|
store: busAsked{conn: conn},
|
||||||
|
publish: func(ctx context.Context, subject string, body []byte, id string) error {
|
||||||
|
_, err := js.Context().Publish(subject, body, nats.MsgId(id), nats.Context(ctx))
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
routerHere: routerHereIn(open.inventory),
|
||||||
|
grantHeld: grantHeldIn(open),
|
||||||
|
routerRecord: func(ctx context.Context, id string) (string, error) {
|
||||||
|
bucket, err := asksRecords(ctx, open.inventory)
|
||||||
|
if err != nil || bucket == "" {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
state, _, err := readRouterRecord(ctx, conn, bucket, askerName, id)
|
||||||
|
return state, err
|
||||||
|
},
|
||||||
|
now: time.Now,
|
||||||
|
caller: link.Caller(),
|
||||||
|
waitFor: routerAnswersWithin, waitEvery: routerAnswersEvery,
|
||||||
|
}
|
||||||
|
words, err := pr.propose(ctx, proposeInput{module: module, node: node, values: values, clear: clear, replace: replace})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(words)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- listing ---------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// proposalsCommand is `settings proposals [<id>]`: every proposal, newest first, and where each stands; with an
|
||||||
|
// id, the proposal whole — its values, the layer it was shown against, and its digest.
|
||||||
|
func proposalsCommand(ctx context.Context, id string) error {
|
||||||
|
return onTheBus(func(conn *nats.Conn) error {
|
||||||
|
all, err := busAsked{conn: conn}.All(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var proposals []asked
|
||||||
|
for _, r := range all {
|
||||||
|
if r.Proposal != nil {
|
||||||
|
proposals = append(proposals, r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Slice(proposals, func(i, j int) bool { return proposals[i].Opened.After(proposals[j].Opened) })
|
||||||
|
if id != "" {
|
||||||
|
for _, r := range proposals {
|
||||||
|
if r.ID == id {
|
||||||
|
fmt.Print(describeProposal(r, time.Now()))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return fmt.Errorf("no proposal %s is kept", id)
|
||||||
|
}
|
||||||
|
if len(proposals) == 0 {
|
||||||
|
fmt.Println("no settings have been proposed")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, r := range proposals {
|
||||||
|
fmt.Print(proposalLine(r, time.Now()))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// proposalState is where a proposal stands, in a word or two.
|
||||||
|
func proposalState(r asked, now time.Time) string {
|
||||||
|
switch {
|
||||||
|
case r.State == askOpen && now.Before(r.Ask.Expires):
|
||||||
|
return "waiting for the operator until " + r.Ask.Expires.Local().Format("2006-01-02 15:04")
|
||||||
|
case r.State == askOpen:
|
||||||
|
return "expired unanswered; discarded"
|
||||||
|
case r.Acted != "":
|
||||||
|
return r.State + ": " + r.Acted
|
||||||
|
}
|
||||||
|
return r.State
|
||||||
|
}
|
||||||
|
|
||||||
|
func proposalLine(r asked, now time.Time) string {
|
||||||
|
p := *r.Proposal
|
||||||
|
what := "set"
|
||||||
|
if p.Clear {
|
||||||
|
what = "clear"
|
||||||
|
}
|
||||||
|
keys := strings.Join(layerKeys(p.Values), ", ")
|
||||||
|
if p.Clear {
|
||||||
|
keys = "the whole layer"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s %s %s on %s (%s)\n by %s at %s; fingerprint %s\n %s\n", r.ID, what, p.Module, p.where(),
|
||||||
|
keys, p.From, p.At.Local().Format("2006-01-02 15:04"), fingerprint(p.Digest), proposalState(r, now))
|
||||||
|
}
|
||||||
|
|
||||||
|
func describeProposal(r asked, now time.Time) string {
|
||||||
|
p := *r.Proposal
|
||||||
|
var b strings.Builder
|
||||||
|
b.WriteString(proposalLine(r, now))
|
||||||
|
fmt.Fprintf(&b, " digest %s; the layer it was shown against %s\n", p.Digest, p.BeforeDigest)
|
||||||
|
shownValues, _ := json.MarshalIndent(p.Values, " ", " ")
|
||||||
|
if p.Clear {
|
||||||
|
fmt.Fprintf(&b, " clears the layer\n")
|
||||||
|
} else {
|
||||||
|
fmt.Fprintf(&b, " values:\n %s\n", shownValues)
|
||||||
|
}
|
||||||
|
if p.HadLayer {
|
||||||
|
shownBefore, _ := json.MarshalIndent(p.Before, " ", " ")
|
||||||
|
fmt.Fprintf(&b, " the layer as it stood:\n %s\n", shownBefore)
|
||||||
|
} else {
|
||||||
|
b.WriteString(" there was no layer\n")
|
||||||
|
}
|
||||||
|
if r.Warrant != nil && r.Warrant.By != nil {
|
||||||
|
fmt.Fprintf(&b, " answered: %s, through %s, at %s\n", r.Warrant.Says(), viaWords(*r.Warrant),
|
||||||
|
r.Warrant.At.Local().Format("2006-01-02 15:04"))
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- acting on the warrant --------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// setOnWarrant performs an approved proposal: the layer set or cleared as `settings set` and `settings clear` at the
|
||||||
|
// terminal do, with who approved it kept beside the layer. It answers the words of what changed.
|
||||||
|
type setOnWarrant func(ctx context.Context, p settingsProposal, askID, setBy string) (string, error)
|
||||||
|
|
||||||
|
// decideProposal is what the asker does with a warrant for a proposal (asker.Decided): nothing on Decline, and on
|
||||||
|
// Approve the act only when it is the one the option bound — the digest of the exact values kept here is the one
|
||||||
|
// in the act, and so the one the ask's option bound and the warrant's ask digest covers.
|
||||||
|
func (a *asker) decideProposal(ctx context.Context, r asked, act conditions.Action, w asks.Warrant) (string, error) {
|
||||||
|
p := *r.Proposal
|
||||||
|
if act.Arguments["answer"] != answerApprove {
|
||||||
|
return "nothing: the operator declined; the layer is unchanged", nil
|
||||||
|
}
|
||||||
|
if a.setLayer == nil {
|
||||||
|
return "", errors.New("this controller cannot set a layer on a warrant")
|
||||||
|
}
|
||||||
|
// The record's own proposal against the act the option bound: the act is composed again from the record —
|
||||||
|
// its module, machine, values (digested again), the layer they replace, whether a removal is meant, a clear,
|
||||||
|
// who proposed it and when — and must digest to what the option bound when the operator was shown it. A
|
||||||
|
// record changed after the ask, in any field, is refused and nothing is set.
|
||||||
|
again := p
|
||||||
|
again.Digest, again.BeforeDigest = layerDigest(p.Values), layerDigest(p.Before)
|
||||||
|
if p.Clear {
|
||||||
|
again.Digest = layerDigest(p.Before)
|
||||||
|
}
|
||||||
|
recomposed := again.actions(r.ID)
|
||||||
|
chosen := -1
|
||||||
|
for i, candidate := range recomposed {
|
||||||
|
if candidate.Arguments["answer"] == act.Arguments["answer"] {
|
||||||
|
chosen = i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
option, offered := r.Ask.Option(w.Option)
|
||||||
|
if chosen < 0 || !offered {
|
||||||
|
return "", fmt.Errorf("the proposal %s offers no answer %q: nothing is set", r.ID, act.Arguments["answer"])
|
||||||
|
}
|
||||||
|
if err := option.Performs(boundAct(recomposed[chosen])); err != nil {
|
||||||
|
return "", fmt.Errorf("the proposal kept for %s is not the one the operator was shown: %v", r.ID, err)
|
||||||
|
}
|
||||||
|
setBy := fmt.Sprintf("approved by %s via %s at %s (ask %s, proposed by %s)", byWords(w), viaWords(w),
|
||||||
|
w.At.Local().Format("2006-01-02 15:04"), r.ID, p.From)
|
||||||
|
return a.setLayer(ctx, p, r.ID, setBy)
|
||||||
|
}
|
||||||
|
|
||||||
|
// setLayerIn is setOnWarrant on this controller's stores: the layer must still be the one the operator was shown
|
||||||
|
// the change against (to-be 46 §10, step 7), then it is set with the same judgement as at the terminal.
|
||||||
|
func setLayerIn(open *stores) setOnWarrant {
|
||||||
|
return func(ctx context.Context, p settingsProposal, askID, setBy string) (string, error) {
|
||||||
|
inv := open.inventory
|
||||||
|
before, had, err := inv.Layer(ctx, p.Node, p.Module)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if layerDigest(before) != p.BeforeDigest || had != p.HadLayer {
|
||||||
|
return "", fmt.Errorf("the layer of %s on %s changed since the operator was shown the change: it is not set; "+
|
||||||
|
"propose it again", p.Module, p.where())
|
||||||
|
}
|
||||||
|
var changed string
|
||||||
|
if p.Clear {
|
||||||
|
if err := inv.ClearSettingsBy(ctx, p.Node, p.Module, setBy); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
changed = "the layer is removed"
|
||||||
|
} else {
|
||||||
|
added, altered, removed := settingsChange(before, p.Values)
|
||||||
|
if len(removed) > 0 && !p.Replace {
|
||||||
|
return "", fmt.Errorf("the layer would no longer set %s, and the removal was not meant: nothing is set",
|
||||||
|
strings.Join(removed, ", "))
|
||||||
|
}
|
||||||
|
if err := inv.SetSettingsBy(ctx, p.Node, p.Module, p.Values, setBy); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
var parts []string
|
||||||
|
for _, k := range added {
|
||||||
|
parts = append(parts, "+ "+k)
|
||||||
|
}
|
||||||
|
for _, k := range altered {
|
||||||
|
parts = append(parts, "~ "+k)
|
||||||
|
}
|
||||||
|
for _, k := range removed {
|
||||||
|
parts = append(parts, "- "+k)
|
||||||
|
}
|
||||||
|
changed = strings.Join(parts, ", ")
|
||||||
|
if changed == "" {
|
||||||
|
changed = "nothing changed"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return changed + " (ask " + askID + ")", nil
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,746 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/outward"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq ADR 0277: a trusted setting is proposed through a verb by anyone the bus admits, asked of the
|
||||||
|
// operator at the level approve with the exact change, and set only on the operator's warrant — once, for the
|
||||||
|
// exact values the option bound; declined, expired or refused, it is discarded; nothing is asked when nothing
|
||||||
|
// can carry the ask.
|
||||||
|
|
||||||
|
// aProposer is the propose path with its reaches faked: a layer as it stands, a judge that records what it
|
||||||
|
// judged, a memory store, and what was published.
|
||||||
|
type proposerRig struct {
|
||||||
|
pr proposer
|
||||||
|
store memAskedStore
|
||||||
|
sent []published
|
||||||
|
judged []map[string]any
|
||||||
|
before map[string]any
|
||||||
|
had bool
|
||||||
|
refusedBy string
|
||||||
|
now time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func newProposerRig(t *testing.T) *proposerRig {
|
||||||
|
r := &proposerRig{store: memAskedStore{}, now: time.Date(2026, 10, 10, 14, 5, 0, 0, time.UTC)}
|
||||||
|
r.pr = proposer{
|
||||||
|
layer: func(_ context.Context, node, module string) (map[string]any, bool, error) {
|
||||||
|
return r.before, r.had, nil
|
||||||
|
},
|
||||||
|
judge: func(_ context.Context, node, module string, values map[string]any) error {
|
||||||
|
r.judged = append(r.judged, values)
|
||||||
|
if r.refusedBy != "" {
|
||||||
|
return errors.New(r.refusedBy)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
machines: func(context.Context) ([]string, error) { return []string{"anchor", "laptop", "shanks"}, nil },
|
||||||
|
store: r.store,
|
||||||
|
publish: func(_ context.Context, subject string, body []byte, id string) error {
|
||||||
|
r.sent = append(r.sent, published{subject, id, body})
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
now: func() time.Time { return r.now },
|
||||||
|
caller: "g14/claude-code, through the mesh-controller seat",
|
||||||
|
waitFor: time.Millisecond,
|
||||||
|
waitEvery: time.Millisecond,
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *proposerRig) askSent(t *testing.T) asks.Ask {
|
||||||
|
t.Helper()
|
||||||
|
if len(r.sent) != 1 || r.sent[0].subject != asks.AskSubject("mesh-controller") {
|
||||||
|
t.Fatalf("published %+v", r.sent)
|
||||||
|
}
|
||||||
|
var q asks.Ask
|
||||||
|
if err := json.Unmarshal(r.sent[0].body, &q); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return q
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *proposerRig) theProposal(t *testing.T) asked {
|
||||||
|
t.Helper()
|
||||||
|
for _, a := range r.store {
|
||||||
|
if a.Proposal != nil {
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatal("no proposal is kept")
|
||||||
|
return asked{}
|
||||||
|
}
|
||||||
|
|
||||||
|
var mountsSources = map[string]any{"sources": "recalbox=smb://nas.lan/recalbox@/mnt/recalbox:ro", "shares": "library=/mnt/library"}
|
||||||
|
|
||||||
|
// The ask: at the level approve on both answers, each binding the proposal's act, with every key and its exact new
|
||||||
|
// value as far as the content rule lets it leave the mesh, the layer it was shown against, and nothing set.
|
||||||
|
func TestAProposalAsksAtTheLevelApproveWithTheExactChange(t *testing.T) {
|
||||||
|
r := newProposerRig(t)
|
||||||
|
r.before, r.had = map[string]any{"shares": "none", "old": "x"}, true
|
||||||
|
words, err := r.pr.propose(context.Background(), proposeInput{module: "mounts", node: "shanks", values: mountsSources, replace: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
q := r.askSent(t)
|
||||||
|
if err := q.Check(r.now); err != nil {
|
||||||
|
t.Fatalf("the ask is refused: %v", err)
|
||||||
|
}
|
||||||
|
if q.Headline != "Set mounts on shanks?" || q.About != "settings.mounts.shanks" || q.Who != asks.Operator ||
|
||||||
|
!q.Expires.Equal(r.now.Add(askApproveFor)) {
|
||||||
|
t.Errorf("the ask: %+v", q)
|
||||||
|
}
|
||||||
|
if len(q.Options) != 2 {
|
||||||
|
t.Fatalf("options %+v", q.Options)
|
||||||
|
}
|
||||||
|
for _, o := range q.Options {
|
||||||
|
if o.Level != asks.Approve || !strings.HasPrefix(o.Binds, "sha256:") {
|
||||||
|
t.Errorf("the option %s is %s and binds %q", o.ID, o.Level, o.Binds)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if q.Options[0].Binds == q.Options[1].Binds {
|
||||||
|
t.Error("Approve and Decline bind the same act")
|
||||||
|
}
|
||||||
|
for _, line := range []string{
|
||||||
|
"Set the settings of mounts on shanks to these values?",
|
||||||
|
"Proposed by g14/claude-code, through the mesh-controller seat, at " + r.now.Local().Format("15:04 on 2 Jan") + ".",
|
||||||
|
"+ sources: recalbox=‹address›@‹path›:ro",
|
||||||
|
"~ shares: library=‹path› (was: none)",
|
||||||
|
"- old (was: x)",
|
||||||
|
"Fingerprint " + fingerprint(layerDigest(mountsSources)) + ".",
|
||||||
|
"read it whole, with this fingerprint",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(q.Explanation, line) {
|
||||||
|
t.Errorf("the explanation lacks %q:\n%s", line, q.Explanation)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, leak := range []string{"nas.lan", "/mnt/recalbox", "/mnt/library", "smb://"} {
|
||||||
|
if strings.Contains(q.Explanation, leak) {
|
||||||
|
t.Errorf("the explanation carries %q, which may not leave the mesh", leak)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
all := []string{q.Headline, q.Explanation, q.OnExpiry}
|
||||||
|
for _, o := range q.Options {
|
||||||
|
all = append(all, o.Label, o.Does)
|
||||||
|
}
|
||||||
|
if refusal, ok := outward.Check(strings.Join(all, "\n"), "anchor", "laptop", "shanks"); !ok {
|
||||||
|
t.Errorf("the router would refuse the ask: %s", refusal)
|
||||||
|
}
|
||||||
|
// Kept as the controller's own ask, about no condition, with the proposal whole and its digests.
|
||||||
|
kept := r.theProposal(t)
|
||||||
|
p := kept.Proposal
|
||||||
|
if kept.State != askOpen || kept.Ask.Digest() != q.Digest() || p.Module != "mounts" || p.Node != "shanks" ||
|
||||||
|
p.Digest != layerDigest(mountsSources) || p.BeforeDigest != layerDigest(r.before) || !p.Replace || !p.HadLayer ||
|
||||||
|
p.From != r.pr.caller || kept.ofACondition() {
|
||||||
|
t.Errorf("kept %+v / %+v", kept, p)
|
||||||
|
}
|
||||||
|
// Each option binds exactly the act the controller will perform (boundAct over the kept action).
|
||||||
|
for i, act := range kept.Actions {
|
||||||
|
binds, _ := asks.ActDigest(boundAct(act))
|
||||||
|
if q.Options[i].Binds != binds || act.Arguments["values"] != p.Digest || act.Arguments["before"] != p.BeforeDigest ||
|
||||||
|
act.Verb != proposalVerb || act.Level != conditions.LevelApprove {
|
||||||
|
t.Errorf("the option %s does not bind the kept act: %+v", q.Options[i].ID, act)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(r.judged) != 1 || r.judged[0]["sources"] != mountsSources["sources"] {
|
||||||
|
t.Errorf("judged %v", r.judged)
|
||||||
|
}
|
||||||
|
if !strings.Contains(words, "nothing changes") || !strings.Contains(words, kept.ID) {
|
||||||
|
t.Errorf("the caller is told: %s", words)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A layer for the whole mesh is proposed too.
|
||||||
|
func TestAMeshWideLayerIsProposed(t *testing.T) {
|
||||||
|
r := newProposerRig(t)
|
||||||
|
if _, err := r.pr.propose(context.Background(), proposeInput{module: "notes", values: map[string]any{"x": "1"}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
q := r.askSent(t)
|
||||||
|
if q.Headline != "Set notes on the whole mesh?" || q.About != "settings.notes.mesh" ||
|
||||||
|
!strings.Contains(q.Explanation, "Set the settings of notes on the whole mesh to these values?") {
|
||||||
|
t.Errorf("%+v", q)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A proposal expired unanswered is kept so by the reconciling, and a warrant for it afterwards sets nothing.
|
||||||
|
func TestAnExpiredProposalIsKeptExpired(t *testing.T) {
|
||||||
|
r := newAskerRig(t)
|
||||||
|
calls := withSetLayer(r)
|
||||||
|
a := aProposalAsked(t, r, "s7", aProposal(r.now))
|
||||||
|
r.now = r.now.Add(askApproveFor + time.Minute)
|
||||||
|
if err := r.a.reconcile(context.Background()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := r.store["s7"]; got.State != string(asks.OutcomeExpired) || !strings.HasPrefix(got.Acted, "nothing") {
|
||||||
|
t.Errorf("kept as %+v", got)
|
||||||
|
}
|
||||||
|
answerWith(t, r, warrantOn(a, "approve", r.now.Add(-2*time.Minute)))
|
||||||
|
if len(*calls) != 0 {
|
||||||
|
t.Errorf("set after expiry: %+v", *calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A clear is proposed too, and shows the layer it removes.
|
||||||
|
func TestAClearIsProposedAndShowsWhatItRemoves(t *testing.T) {
|
||||||
|
r := newProposerRig(t)
|
||||||
|
r.before, r.had = map[string]any{"places": map[string]any{"data": map[string]any{"path": "/srv/notes", "owner": "1001:1001"}}}, true
|
||||||
|
if _, err := r.pr.propose(context.Background(), proposeInput{module: "notes", node: "laptop", clear: true}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
q := r.askSent(t)
|
||||||
|
if q.Headline != "Clear notes on laptop?" || !strings.Contains(q.Explanation, "- places.data.owner: 1001:1001") ||
|
||||||
|
!strings.Contains(q.Explanation, "- places.data.path: ‹path›") {
|
||||||
|
t.Errorf("%+v", q)
|
||||||
|
}
|
||||||
|
if p := r.theProposal(t).Proposal; !p.Clear || p.Digest != layerDigest(r.before) || len(r.judged) != 0 {
|
||||||
|
t.Errorf("a clear: %+v, judged %v", p, r.judged)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// What the phone is shown keeps a value's shape and passes the content rule: an address, a path, a secret's
|
||||||
|
// shape each replaced in place; a value every word of which may leave the mesh shown whole.
|
||||||
|
func TestAValueIsShownInItsShapeAndPassesTheContentRule(t *testing.T) {
|
||||||
|
for in, want := range map[any]string{
|
||||||
|
"recalbox=smb://nas.lan/recalbox@/mnt/recalbox:ro": "recalbox=‹address›@‹path›:ro",
|
||||||
|
"library=/mnt/library": "library=‹path›",
|
||||||
|
"none": "none",
|
||||||
|
"Inter 13": "Inter 13",
|
||||||
|
"10.77.0.9:53": "‹address›",
|
||||||
|
"jochen@example.com": "‹address›",
|
||||||
|
"nas.lan": "‹address›",
|
||||||
|
"anchor": "anchor",
|
||||||
|
"-----BEGIN CERTIFICATE-----": "‹withheld›",
|
||||||
|
42: "42",
|
||||||
|
true: "true",
|
||||||
|
} {
|
||||||
|
got, whole := sayableValue(in, []string{"anchor"})
|
||||||
|
if got != want {
|
||||||
|
t.Errorf("%v shown as %q, want %q", in, got, want)
|
||||||
|
}
|
||||||
|
if whole != (got == want && !strings.Contains(want, "‹")) {
|
||||||
|
t.Errorf("%v: whole %v", in, whole)
|
||||||
|
}
|
||||||
|
if _, ok := outward.Check(got, "anchor"); !ok {
|
||||||
|
t.Errorf("%v shown as %q, which the router refuses", in, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, v := range []any{[]any{"a", "/etc/x"}, map[string]any{"path": "/srv/x", "owner": "1001:1001"}} {
|
||||||
|
got, _ := sayableValue(v, nil)
|
||||||
|
if _, ok := outward.Check(got); !ok || strings.Contains(got, "/srv") || strings.Contains(got, "/etc") {
|
||||||
|
t.Errorf("%v shown as %q", v, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A proposal that the mesh would refuse to set, or that would silently remove a key, is refused before anybody is
|
||||||
|
// asked (ADR 0217 holds for a proposal as for a set).
|
||||||
|
func TestAProposalTheMeshWouldRefuseIsNotAsked(t *testing.T) {
|
||||||
|
r := newProposerRig(t)
|
||||||
|
r.before, r.had = map[string]any{"a": 1, "b": 2}, true
|
||||||
|
_, err := r.pr.propose(context.Background(), proposeInput{module: "notes", node: "laptop", values: map[string]any{"a": 1}})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "would no longer set b") || !strings.Contains(err.Error(), "ADR 0217") {
|
||||||
|
t.Errorf("a silent removal: %v", err)
|
||||||
|
}
|
||||||
|
r.refusedBy = "refused: notes on laptop cannot compose"
|
||||||
|
_, err = r.pr.propose(context.Background(), proposeInput{module: "notes", node: "laptop", values: map[string]any{"a": 1, "b": 3}})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "cannot compose") {
|
||||||
|
t.Errorf("a layer the mesh refuses: %v", err)
|
||||||
|
}
|
||||||
|
if len(r.sent) != 0 || len(r.store) != 0 {
|
||||||
|
t.Errorf("asked anyway: %+v %+v", r.sent, r.store)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fail closed: no router, no grant, a publish that fails, or the router's refusal each leave nothing waiting for
|
||||||
|
// an answer that cannot come, and name the terminal's line.
|
||||||
|
func TestAProposalFailsClosedWhenNothingCanCarryIt(t *testing.T) {
|
||||||
|
r := newProposerRig(t)
|
||||||
|
in := proposeInput{module: "mounts", node: "shanks", values: mountsSources}
|
||||||
|
r.pr.routerHere = func(context.Context) (bool, error) { return false, nil }
|
||||||
|
if _, err := r.pr.propose(context.Background(), in); err == nil || !strings.Contains(err.Error(), "no router") ||
|
||||||
|
!strings.Contains(err.Error(), "mesh-cli settings set mounts") {
|
||||||
|
t.Errorf("without a router: %v", err)
|
||||||
|
}
|
||||||
|
r.pr.routerHere = nil
|
||||||
|
r.pr.grantHeld = func(context.Context) (bool, string, error) { return false, "the bus's user list is behind", nil }
|
||||||
|
if _, err := r.pr.propose(context.Background(), in); err == nil || !strings.Contains(err.Error(), "user list is behind") {
|
||||||
|
t.Errorf("without the grant: %v", err)
|
||||||
|
}
|
||||||
|
if len(r.sent) != 0 || len(r.store) != 0 {
|
||||||
|
t.Fatalf("asked anyway: %+v %+v", r.sent, r.store)
|
||||||
|
}
|
||||||
|
r.pr.grantHeld = nil
|
||||||
|
r.pr.publish = func(context.Context, string, []byte, string) error { return errors.New("the bus is away") }
|
||||||
|
if _, err := r.pr.propose(context.Background(), in); err == nil || !strings.Contains(err.Error(), "the bus is away") ||
|
||||||
|
!strings.Contains(err.Error(), "never become active") {
|
||||||
|
t.Errorf("a publish that fails: %v", err)
|
||||||
|
}
|
||||||
|
if kept := r.theProposal(t); kept.State != askUnsent {
|
||||||
|
t.Errorf("an unpublished proposal is %s", kept.State)
|
||||||
|
}
|
||||||
|
// The router's refusal, heard by the serving controller and kept here, is said to the caller.
|
||||||
|
r = newProposerRig(t)
|
||||||
|
r.pr.publish = func(_ context.Context, _ string, _ []byte, id string) error {
|
||||||
|
ask := strings.TrimPrefix(id, "ask.")
|
||||||
|
r.store[ask] = func() asked {
|
||||||
|
a := r.store[ask]
|
||||||
|
a.State, a.Acted = string(asks.OutcomeRefused), "nothing: the ask refused: no channel can carry any of its answers now"
|
||||||
|
return a
|
||||||
|
}()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if _, err := r.pr.propose(context.Background(), in); err == nil || !strings.Contains(err.Error(), "did not ask the operator") ||
|
||||||
|
!strings.Contains(err.Error(), "no channel can carry") {
|
||||||
|
t.Errorf("the router's refusal: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The bounds: at most three asks open for the controller, and the same change not proposed twice.
|
||||||
|
func TestAProposalIsBounded(t *testing.T) {
|
||||||
|
r := newProposerRig(t)
|
||||||
|
in := proposeInput{module: "mounts", node: "shanks", values: mountsSources}
|
||||||
|
if _, err := r.pr.propose(context.Background(), in); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := r.pr.propose(context.Background(), in); err == nil || !strings.Contains(err.Error(), "already proposed") {
|
||||||
|
t.Errorf("the same change twice: %v", err)
|
||||||
|
}
|
||||||
|
for _, node := range []string{"laptop", "anchor"} {
|
||||||
|
if _, err := r.pr.propose(context.Background(), proposeInput{module: "mounts", node: node, values: mountsSources}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_, err := r.pr.propose(context.Background(), proposeInput{module: "notes", node: "laptop", values: map[string]any{"x": 1}})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "3 questions already wait") {
|
||||||
|
t.Errorf("a fourth: %v", err)
|
||||||
|
}
|
||||||
|
if len(r.sent) != 3 {
|
||||||
|
t.Errorf("published %d", len(r.sent))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- the warrant ------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// aProposalAsked keeps a proposal's ask in the asker rig's store, as propose keeps it.
|
||||||
|
func aProposalAsked(t *testing.T, r *askerRig, id string, p settingsProposal) asked {
|
||||||
|
t.Helper()
|
||||||
|
q, options := p.ask(id, nil)
|
||||||
|
if err := q.Check(r.now); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
a := asked{ID: id, Condition: q.About, Ask: q, Actions: p.actions(id), Options: options, State: askOpen, Opened: r.now, Proposal: &p}
|
||||||
|
r.store[id] = a
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
|
||||||
|
func aProposal(now time.Time) settingsProposal {
|
||||||
|
before := map[string]any{"shares": "none"}
|
||||||
|
return settingsProposal{Module: "mounts", Node: "shanks", Values: mountsSources, Replace: true, Before: before, HadLayer: true,
|
||||||
|
From: "g14/claude-code", At: now, Digest: layerDigest(mountsSources), BeforeDigest: layerDigest(before)}
|
||||||
|
}
|
||||||
|
|
||||||
|
// warrantOn is the router's warrant for a kept ask, choosing an option by id.
|
||||||
|
func warrantOn(a asked, option string, now time.Time) asks.Warrant {
|
||||||
|
o, _ := a.Ask.Option(option)
|
||||||
|
return asks.Warrant{Ask: a.ID, Asker: "mesh-controller", About: a.Ask.About, Outcome: asks.OutcomeChosen, Option: o.ID,
|
||||||
|
Label: o.Label, Level: o.Level, Channel: "telegram", Proofs: []string{"P1"}, At: now, AskDigest: a.Ask.Digest(),
|
||||||
|
By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}}
|
||||||
|
}
|
||||||
|
|
||||||
|
type setCall struct {
|
||||||
|
p settingsProposal
|
||||||
|
ask string
|
||||||
|
setBy string
|
||||||
|
}
|
||||||
|
|
||||||
|
func withSetLayer(r *askerRig) *[]setCall {
|
||||||
|
var calls []setCall
|
||||||
|
r.a.setLayer = func(_ context.Context, p settingsProposal, askID, setBy string) (string, error) {
|
||||||
|
calls = append(calls, setCall{p, askID, setBy})
|
||||||
|
return "+ sources, ~ shares", nil
|
||||||
|
}
|
||||||
|
return &calls
|
||||||
|
}
|
||||||
|
|
||||||
|
// On Approve the layer is set once, with who approved it and through which channel kept beside it, and the warrant
|
||||||
|
// is recorded as the operator's decision; heard again, nothing more happens.
|
||||||
|
func TestTheLayerIsSetOnceOnTheOperatorsApproval(t *testing.T) {
|
||||||
|
r := newAskerRig(t)
|
||||||
|
calls := withSetLayer(r)
|
||||||
|
a := aProposalAsked(t, r, "s1", aProposal(r.now))
|
||||||
|
if err := r.a.reconcile(context.Background()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := r.store["s1"]; got.State != askOpen {
|
||||||
|
t.Fatalf("the reconciling of conditions ended the proposal: %+v", got)
|
||||||
|
}
|
||||||
|
w := warrantOn(a, "approve", r.now.Add(time.Hour))
|
||||||
|
answerWith(t, r, w)
|
||||||
|
answerWith(t, r, w) // heard again, or replayed
|
||||||
|
if len(*calls) != 1 {
|
||||||
|
t.Fatalf("set %d time(s): %+v", len(*calls), *calls)
|
||||||
|
}
|
||||||
|
c := (*calls)[0]
|
||||||
|
if c.ask != "s1" || c.p.Digest != layerDigest(mountsSources) ||
|
||||||
|
!strings.HasPrefix(c.setBy, "approved by the operator, as telegram identity 42 via telegram (telegram), user id verified at ") ||
|
||||||
|
!strings.Contains(c.setBy, "(ask s1, proposed by g14/claude-code)") {
|
||||||
|
t.Errorf("set by %q for %+v", c.setBy, c.p)
|
||||||
|
}
|
||||||
|
if len(r.called)+len(r.silenced) != 0 {
|
||||||
|
t.Errorf("a verb was called: %v %v", r.called, r.silenced)
|
||||||
|
}
|
||||||
|
if len(r.acts) != 1 {
|
||||||
|
t.Fatalf("hand-acts %+v", r.acts)
|
||||||
|
}
|
||||||
|
act := r.acts[0]
|
||||||
|
if act.Verb != handActWarrant || act.By != "the operator, as telegram identity 42" || act.Ask != "s1" ||
|
||||||
|
!slices.Contains(act.Args, "answer=approve") || !slices.Contains(act.Args, "values="+layerDigest(mountsSources)) ||
|
||||||
|
!strings.HasPrefix(act.Outcome, "done") || !personsDecision(act) {
|
||||||
|
t.Errorf("the record: %+v", act)
|
||||||
|
}
|
||||||
|
if got := r.store["s1"]; got.State != string(asks.OutcomeChosen) || !strings.HasPrefix(got.Acted, "done") {
|
||||||
|
t.Errorf("kept as %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Decline, expiry, the router's refusal, a cancel: nothing is set, and the proposal is recorded as ended.
|
||||||
|
func TestDeclineExpiryAndRefusalDiscardAProposal(t *testing.T) {
|
||||||
|
for name, outcome := range map[string]asks.Outcome{"declined": asks.OutcomeChosen, "expired": asks.OutcomeExpired,
|
||||||
|
"refused": asks.OutcomeRefused, "cancelled": asks.OutcomeCancelled, "replaced": asks.OutcomeReplaced} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
r := newAskerRig(t)
|
||||||
|
calls := withSetLayer(r)
|
||||||
|
a := aProposalAsked(t, r, "s2", aProposal(r.now))
|
||||||
|
w := warrantOn(a, "decline", r.now.Add(time.Hour))
|
||||||
|
if outcome != asks.OutcomeChosen {
|
||||||
|
w = asks.Warrant{Ask: a.ID, Asker: "mesh-controller", Outcome: outcome, Words: "its time passed", At: r.now.Add(time.Hour)}
|
||||||
|
}
|
||||||
|
answerWith(t, r, w)
|
||||||
|
if len(*calls) != 0 {
|
||||||
|
t.Fatalf("set: %+v", *calls)
|
||||||
|
}
|
||||||
|
got := r.store["s2"]
|
||||||
|
if got.State != string(outcome) || got.Acted == "" || !strings.HasPrefix(got.Acted, "nothing") {
|
||||||
|
t.Errorf("kept as %+v", got)
|
||||||
|
}
|
||||||
|
if outcome == asks.OutcomeChosen && (len(r.acts) != 1 || !strings.Contains(r.acts[0].Outcome, "declined")) {
|
||||||
|
t.Errorf("a decline is a decision too: %+v", r.acts)
|
||||||
|
}
|
||||||
|
// An approval after it ended is refused.
|
||||||
|
answerWith(t, r, warrantOn(a, "approve", r.now.Add(2*time.Hour)))
|
||||||
|
if len(*calls) != 0 {
|
||||||
|
t.Fatalf("set after the end: %+v", *calls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The warrant binds the exact values: a record whose values changed after the ask, an action changed, a warrant for
|
||||||
|
// another ask's digest, at another level, or after expiry each set nothing.
|
||||||
|
func TestAWarrantSetsOnlyTheExactValuesTheOperatorWasShown(t *testing.T) {
|
||||||
|
cases := map[string]func(r *askerRig, a asked) asks.Warrant{
|
||||||
|
"the values changed in the record": func(r *askerRig, a asked) asks.Warrant {
|
||||||
|
a.Proposal.Values = map[string]any{"sources": "recalbox=smb://evil/recalbox@/mnt/recalbox", "shares": "library=/mnt/library"}
|
||||||
|
r.store[a.ID] = a
|
||||||
|
return warrantOn(a, "approve", r.now.Add(time.Hour))
|
||||||
|
},
|
||||||
|
"the layer it was shown against changed in the record": func(r *askerRig, a asked) asks.Warrant {
|
||||||
|
a.Proposal.Before = map[string]any{"shares": "other"}
|
||||||
|
r.store[a.ID] = a
|
||||||
|
return warrantOn(a, "approve", r.now.Add(time.Hour))
|
||||||
|
},
|
||||||
|
"the module changed in the record": func(r *askerRig, a asked) asks.Warrant {
|
||||||
|
a.Proposal.Module = "sshd"
|
||||||
|
r.store[a.ID] = a
|
||||||
|
return warrantOn(a, "approve", r.now.Add(time.Hour))
|
||||||
|
},
|
||||||
|
"the machine changed in the record": func(r *askerRig, a asked) asks.Warrant {
|
||||||
|
a.Proposal.Node = "anchor"
|
||||||
|
r.store[a.ID] = a
|
||||||
|
return warrantOn(a, "approve", r.now.Add(time.Hour))
|
||||||
|
},
|
||||||
|
"the removal became meant in the record": func(r *askerRig, a asked) asks.Warrant {
|
||||||
|
a.Proposal.Replace = !a.Proposal.Replace
|
||||||
|
r.store[a.ID] = a
|
||||||
|
return warrantOn(a, "approve", r.now.Add(time.Hour))
|
||||||
|
},
|
||||||
|
"the set became a clear in the record": func(r *askerRig, a asked) asks.Warrant {
|
||||||
|
a.Proposal.Clear = true
|
||||||
|
r.store[a.ID] = a
|
||||||
|
return warrantOn(a, "approve", r.now.Add(time.Hour))
|
||||||
|
},
|
||||||
|
"the action's digest was changed": func(r *askerRig, a asked) asks.Warrant {
|
||||||
|
a.Actions[0].Arguments["values"] = layerDigest(map[string]any{"sources": "x"})
|
||||||
|
r.store[a.ID] = a
|
||||||
|
return warrantOn(a, "approve", r.now.Add(time.Hour))
|
||||||
|
},
|
||||||
|
"another ask's digest": func(r *askerRig, a asked) asks.Warrant {
|
||||||
|
w := warrantOn(a, "approve", r.now.Add(time.Hour))
|
||||||
|
w.AskDigest = "sha256:0000"
|
||||||
|
return w
|
||||||
|
},
|
||||||
|
"at the level acknowledge": func(r *askerRig, a asked) asks.Warrant {
|
||||||
|
w := warrantOn(a, "approve", r.now.Add(time.Hour))
|
||||||
|
w.Level = asks.Acknowledge
|
||||||
|
return w
|
||||||
|
},
|
||||||
|
"after expiry": func(r *askerRig, a asked) asks.Warrant {
|
||||||
|
return warrantOn(a, "approve", r.now.Add(askApproveFor+time.Minute))
|
||||||
|
},
|
||||||
|
"nobody chose": func(r *askerRig, a asked) asks.Warrant {
|
||||||
|
w := warrantOn(a, "approve", r.now.Add(time.Hour))
|
||||||
|
w.By = nil
|
||||||
|
return w
|
||||||
|
},
|
||||||
|
"another asker's": func(r *askerRig, a asked) asks.Warrant {
|
||||||
|
w := warrantOn(a, "approve", r.now.Add(time.Hour))
|
||||||
|
w.Asker = "claude-code"
|
||||||
|
return w
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for name, tamper := range cases {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
r := newAskerRig(t)
|
||||||
|
calls := withSetLayer(r)
|
||||||
|
a := aProposalAsked(t, r, "s3", aProposal(r.now))
|
||||||
|
answerWith(t, r, tamper(r, a))
|
||||||
|
if len(*calls) != 0 {
|
||||||
|
t.Fatalf("set: %+v", *calls)
|
||||||
|
}
|
||||||
|
if got := r.store["s3"]; strings.HasPrefix(got.Acted, "done") {
|
||||||
|
t.Errorf("kept as done: %+v", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A proposal counts toward what the controller holds open, so a fourth ask is not attempted while three are.
|
||||||
|
func TestOpenProposalsCountTowardTheAsksHeldOpen(t *testing.T) {
|
||||||
|
r := newAskerRig(t)
|
||||||
|
for _, id := range []string{"s4", "s5", "s6"} {
|
||||||
|
aProposalAsked(t, r, id, aProposal(r.now))
|
||||||
|
}
|
||||||
|
r.open = []conditions.Condition{heldCondition()}
|
||||||
|
if err := r.a.reconcile(context.Background()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(r.sent) != 0 {
|
||||||
|
t.Errorf("asked beyond the bound: %d", len(r.sent))
|
||||||
|
}
|
||||||
|
for _, id := range []string{"s4", "s5", "s6"} {
|
||||||
|
if r.store[id].State != askOpen {
|
||||||
|
t.Errorf("%s was ended by the reconciling of conditions: %+v", id, r.store[id])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- the verb ---------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
func TestTheSettingsVerbComposesProposeAndProposals(t *testing.T) {
|
||||||
|
for name, c := range map[string]struct {
|
||||||
|
args map[string]any
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
"propose on a machine": {map[string]any{"module": "mounts", "node": "shanks", "values": `{"sources":"x"}`, "propose": "true"},
|
||||||
|
"settings propose mounts {\"sources\":\"x\"} --node shanks"},
|
||||||
|
"propose with replace": {map[string]any{"module": "mounts", "values": `{"a":1}`, "propose": "true", "replace": "true"},
|
||||||
|
"settings propose mounts {\"a\":1} --replace"},
|
||||||
|
"propose a clear": {map[string]any{"module": "mounts", "node": "shanks", "propose": "true", "clear": "true"},
|
||||||
|
"settings propose mounts --clear --node shanks"},
|
||||||
|
"the proposals": {map[string]any{"proposals": "true"}, "settings proposals"},
|
||||||
|
"one proposal": {map[string]any{"proposal": "s1"}, "settings proposals s1"},
|
||||||
|
} {
|
||||||
|
argv, err := argvFor("settings", c.args)
|
||||||
|
if err != nil || strings.Join(argv, " ") != c.want {
|
||||||
|
t.Errorf("%s: %v %v", name, argv, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for name, args := range map[string]map[string]any{
|
||||||
|
"propose without a module": {"values": `{"a":1}`, "propose": "true"},
|
||||||
|
"propose without values": {"module": "mounts", "propose": "true"},
|
||||||
|
"propose values and clear": {"module": "mounts", "values": `{"a":1}`, "clear": "true", "propose": "true"},
|
||||||
|
"proposals with a module": {"proposals": "true", "module": "mounts"},
|
||||||
|
"proposals and a proposal": {"proposals": "true", "proposal": "s1"},
|
||||||
|
"a proposal with values": {"proposal": "s1", "values": `{"a":1}`},
|
||||||
|
"proposals with a listing": {"proposals": "true", "list": "preferences"},
|
||||||
|
} {
|
||||||
|
if _, err := argvFor("settings", args); err == nil {
|
||||||
|
t.Errorf("%s was composed", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The generic command verb proposes nothing (it is the settings verb's), and lists proposals (a read).
|
||||||
|
if err := refusedAsTheGenericCommand([]string{"settings", "propose", "mounts", "{}", "--node", "shanks"}); err == nil {
|
||||||
|
t.Error("the generic command proposed")
|
||||||
|
}
|
||||||
|
if err := refusedAsTheGenericCommand([]string{"settings", "proposals"}); err != nil {
|
||||||
|
t.Errorf("the generic command may not list proposals: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- on the real stores -----------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// setLayerIn sets the layer as the terminal does — judged, the removal meant, the history kept — with who approved it
|
||||||
|
// beside it; and refuses once the layer is no longer the one the operator was shown the change against.
|
||||||
|
func TestSetOnAWarrantJudgesTheLayerAndKeepsWhoApprovedIt(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, catalogue.Manifest{Module: "notes", Version: "1",
|
||||||
|
// y is a preference with a default, so a layer may leave it out; x is the operator's own (ADR 0262).
|
||||||
|
Settings: map[string]catalogue.SettingDeclaration{"y": {Kind: "preference", Default: "10", Why: "a size"}},
|
||||||
|
Resources: []map[string]any{{"id": "data", "type": "directory", "mode": "0755"},
|
||||||
|
// A trusted file (unmarked), so its keys are the terminal's — and now the warrant's (novox/hq ADR 0277).
|
||||||
|
{"id": "rc", "type": "file", "path": "/etc/notes.conf", "mode": "0644", "content": "x = ${setting:x}\ny = ${setting:y}\n"}}})
|
||||||
|
if _, err := assign(ctx, open, "laptop", "notes"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
set := setLayerIn(open)
|
||||||
|
now := time.Now()
|
||||||
|
first := map[string]any{"x": "1", "y": "2"}
|
||||||
|
p := settingsProposal{Module: "notes", Node: "laptop", Values: first, From: "g14/claude-code", At: now,
|
||||||
|
Digest: layerDigest(first), BeforeDigest: layerDigest(nil)}
|
||||||
|
changed, err := set(ctx, p, "s9", "approved by the operator, as telegram identity 42 via telegram at 14:05 (ask s9)")
|
||||||
|
if err != nil || !strings.Contains(changed, "+ x") {
|
||||||
|
t.Fatalf("%q %v", changed, err)
|
||||||
|
}
|
||||||
|
layer, has, err := open.inventory.Layer(ctx, "laptop", "notes")
|
||||||
|
if err != nil || !has || layer["x"] != "1" {
|
||||||
|
t.Fatalf("the layer: %v %v %v", layer, has, err)
|
||||||
|
}
|
||||||
|
setBy, _, has, err := open.inventory.LayerOrigin(ctx, "laptop", "notes")
|
||||||
|
if err != nil || !has || !strings.HasPrefix(setBy, "approved by the operator, as telegram identity 42 via telegram") {
|
||||||
|
t.Fatalf("who set it: %q %v", setBy, err)
|
||||||
|
}
|
||||||
|
// Shown by `settings show`, at the terminal and through the verb.
|
||||||
|
out := captureStdout(t, func() {
|
||||||
|
if err := atTheTerminal(t, "settings", "show", "notes", "--node", "laptop"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if !strings.Contains(out, "approved by the operator, as telegram identity 42 via telegram") {
|
||||||
|
t.Errorf("settings show says:\n%s", out)
|
||||||
|
}
|
||||||
|
// The same proposal again: the layer is no longer the one the operator was shown it against.
|
||||||
|
if _, err := set(ctx, p, "s9", "approved …"); err == nil || !strings.Contains(err.Error(), "changed since the operator was shown") {
|
||||||
|
t.Errorf("a stale proposal: %v", err)
|
||||||
|
}
|
||||||
|
// A removal not meant is refused; one meant is taken, and the history says who had set the layer.
|
||||||
|
second := map[string]any{"x": "1"}
|
||||||
|
q := settingsProposal{Module: "notes", Node: "laptop", Values: second, Before: first, HadLayer: true,
|
||||||
|
From: "g14/claude-code", At: now, Digest: layerDigest(second), BeforeDigest: layerDigest(first)}
|
||||||
|
if _, err := set(ctx, q, "s10", "approved …"); err == nil || !strings.Contains(err.Error(), "removal was not meant") {
|
||||||
|
t.Errorf("a silent removal: %v", err)
|
||||||
|
}
|
||||||
|
// A layer the mesh refuses is refused here too, with the same words as at the terminal.
|
||||||
|
bad := q
|
||||||
|
bad.Values, bad.Digest = map[string]any{"x": "a\nb", "y": "2"}, layerDigest(map[string]any{"x": "a\nb", "y": "2"})
|
||||||
|
if _, err := set(ctx, bad, "s11", "approved …"); err == nil || !strings.Contains(err.Error(), "line break") {
|
||||||
|
t.Errorf("a line break on a warrant: %v", err)
|
||||||
|
}
|
||||||
|
if layer, _, _ := open.inventory.Layer(ctx, "laptop", "notes"); layer["y"] != "2" {
|
||||||
|
t.Fatalf("a refused act changed the layer: %v", layer)
|
||||||
|
}
|
||||||
|
q.Replace = true
|
||||||
|
if _, err := set(ctx, q, "s10", "approved later"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
past, err := open.inventory.SettingsHistory(ctx, "laptop", "notes")
|
||||||
|
if err != nil || len(past) != 1 || !strings.HasPrefix(past[0].SetBy, "approved by the operator") {
|
||||||
|
t.Errorf("the history: %+v %v", past, err)
|
||||||
|
}
|
||||||
|
// And a clear, keeping who cleared it with the copy.
|
||||||
|
c := settingsProposal{Module: "notes", Node: "laptop", Clear: true, Before: second, HadLayer: true, From: "x", At: now,
|
||||||
|
Digest: layerDigest(second), BeforeDigest: layerDigest(second)}
|
||||||
|
if _, err := set(ctx, c, "s12", "approved by the operator at 15:00"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, has, _ := open.inventory.Layer(ctx, "laptop", "notes"); has {
|
||||||
|
t.Error("the layer was not cleared")
|
||||||
|
}
|
||||||
|
past, _ = open.inventory.SettingsHistory(ctx, "laptop", "notes")
|
||||||
|
if len(past) != 2 || !strings.Contains(past[0].SetBy, "cleared approved by the operator at 15:00") {
|
||||||
|
t.Errorf("the history after a clear: %+v", past)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A layer set at the terminal says so, and one set through a verb names the verb (novox/hq ADR 0277).
|
||||||
|
func TestALayerSaysWhoSetIt(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, catalogue.Manifest{Module: "notes", Version: "1",
|
||||||
|
Resources: []map[string]any{{"id": "rc", "type": "file", "path": "/etc/notes.conf", "mode": "0644", "trusted": false,
|
||||||
|
"content": "x = ${setting:x}\n"}}})
|
||||||
|
if _, err := assign(ctx, open, "laptop", "notes"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := atTheTerminal(t, "settings", "set", "notes", `{"x":"1"}`, "--node", "laptop"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
setBy, _, _, _ := open.inventory.LayerOrigin(ctx, "laptop", "notes")
|
||||||
|
if !strings.HasPrefix(setBy, "set at the controller's terminal by ") {
|
||||||
|
t.Errorf("at the terminal: %q", setBy)
|
||||||
|
}
|
||||||
|
if err := throughVerb(t, "settings", map[string]any{"module": "notes", "node": "laptop", "values": `{"x":"2"}`}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
setBy, _, _, _ = open.inventory.LayerOrigin(ctx, "laptop", "notes")
|
||||||
|
if !strings.HasPrefix(setBy, "set by ") || !strings.Contains(setBy, "through settings") {
|
||||||
|
t.Errorf("through the verb: %q", setBy)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A trusted setting is still refused through the settings verb (novox/hq issue 339), and the refusal now names the
|
||||||
|
// proposal as the way.
|
||||||
|
func TestATrustedSettingThroughAVerbNamesTheProposal(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, catalogue.Manifest{Module: "notes", Version: "1",
|
||||||
|
Resources: []map[string]any{{"id": "data", "type": "directory", "mode": "0755"}}})
|
||||||
|
if _, err := assign(ctx, open, "laptop", "notes"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
err := throughVerb(t, "settings", map[string]any{"module": "notes", "node": "laptop",
|
||||||
|
"values": `{"places":{"data":{"path":"/srv/notes","owner":"1001:1001"}}}`})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "issue 339") || !strings.Contains(err.Error(), "propose") {
|
||||||
|
t.Errorf("%v", err)
|
||||||
|
}
|
||||||
|
if _, has, _ := open.inventory.Layer(ctx, "laptop", "notes"); has {
|
||||||
|
t.Error("a layer was kept")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// captureStdout runs f and answers what it printed to standard output.
|
||||||
|
func captureStdout(t *testing.T, f func()) string {
|
||||||
|
t.Helper()
|
||||||
|
before := os.Stdout
|
||||||
|
r, w, err := os.Pipe()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
os.Stdout = w
|
||||||
|
done := make(chan string)
|
||||||
|
go func() {
|
||||||
|
var b strings.Builder
|
||||||
|
_, _ = io.Copy(&b, r)
|
||||||
|
done <- b.String()
|
||||||
|
}()
|
||||||
|
f()
|
||||||
|
os.Stdout = before
|
||||||
|
_ = w.Close()
|
||||||
|
return <-done
|
||||||
|
}
|
||||||
@@ -37,6 +37,32 @@ type holding struct {
|
|||||||
shelf map[string]catalogue.Manifest
|
shelf map[string]catalogue.Manifest
|
||||||
// providers memoises providerFor by machine, consumer and provision.
|
// providers memoises providerFor by machine, consumer and provision.
|
||||||
providers map[string]providerLookup
|
providers map[string]providerLookup
|
||||||
|
// waits is what the waits of a statement are checked against, read as they are asked for (novox/hq issue 450).
|
||||||
|
waits operatorWaitFacts
|
||||||
|
}
|
||||||
|
|
||||||
|
// checked is a machine's newest statement as the controller judges it: each wait checked (ADR 0283 decision 3), so
|
||||||
|
// a wait that does not check out reads as unhealthy, as it did when the statement was judged (novox/hq issue 450).
|
||||||
|
// What is stored is what the machine said, the waits unchecked; read as stored, a provider whose wait failed its
|
||||||
|
// check seems to wait for the operator while its unhealthy condition is open.
|
||||||
|
func (h *holding) checked(machine string) []inventory.ResourceHealth {
|
||||||
|
rs := h.healths[machine].Resources
|
||||||
|
mods := waitingModules(rs)
|
||||||
|
if len(mods) == 0 {
|
||||||
|
return rs
|
||||||
|
}
|
||||||
|
if h.waits.manifests == nil {
|
||||||
|
h.waits.manifests = map[string]catalogue.Manifest{}
|
||||||
|
}
|
||||||
|
for _, module := range mods {
|
||||||
|
if _, has := h.waits.manifests[module]; !has {
|
||||||
|
if m, ok := h.manifestOf(module); ok {
|
||||||
|
h.waits.manifests[module] = m
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
readWaitFacts(h.ctx, h.inv, machine, mods, nil, &h.waits)
|
||||||
|
return checkWaiting(machine, rs, h.waits)
|
||||||
}
|
}
|
||||||
|
|
||||||
type providerLookup struct {
|
type providerLookup struct {
|
||||||
@@ -53,6 +79,15 @@ func readHolding(ctx context.Context, inv *inventory.Inventory, open []condition
|
|||||||
return &holding{ctx: ctx, inv: inv, healths: healths, open: open, providers: map[string]providerLookup{}}, nil
|
return &holding{ctx: ctx, inv: inv, healths: healths, open: open, providers: map[string]providerLookup{}}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// readHoldingFor is how a judging reads its holding: nothing without a store. A variable so a test can hand a
|
||||||
|
// judging the record it holds under (novox/hq issue 405).
|
||||||
|
var readHoldingFor = func(ctx context.Context, inv *inventory.Inventory, open []conditions.Condition) (*holding, error) {
|
||||||
|
if inv == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return readHolding(ctx, inv, open)
|
||||||
|
}
|
||||||
|
|
||||||
// heldFinding says a resource's state is a finding of its declared check that names a provision: what
|
// heldFinding says a resource's state is a finding of its declared check that names a provision: what
|
||||||
// may be held. Down and restarting are liveness, the resource's own.
|
// may be held. Down and restarting are liveness, the resource's own.
|
||||||
func heldFinding(r inventory.ResourceHealth) bool {
|
func heldFinding(r inventory.ResourceHealth) bool {
|
||||||
@@ -106,43 +141,138 @@ func (h *holding) lookUpProvider(machine, consumer, provision string) (catalogue
|
|||||||
return catalogue.Chosen{}, false
|
return catalogue.Chosen{}, false
|
||||||
}
|
}
|
||||||
|
|
||||||
// unhealthy says a provider is unhealthy on the record: its condition is open, or its machine's newest
|
// providerState is how a provider stands on the record: unhealthy when its unhealthy condition is open or its
|
||||||
// statement says a resource of it is unhealthy.
|
// machine's newest statement says a resource of it is unhealthy; else waiting for the operator when that statement
|
||||||
func (h *holding) unhealthy(p catalogue.Chosen) bool {
|
// says a resource of it waits, with the waits it names, or its needs-operator condition is open (waits then
|
||||||
key := moduleUnhealthyKey(p.Module, p.Node)
|
// unknown); else healthy.
|
||||||
|
func (h *holding) providerState(p catalogue.Chosen) (unhealthy, waiting bool, waits []inventory.Wait) {
|
||||||
for _, c := range h.open {
|
for _, c := range h.open {
|
||||||
if c.Key == key {
|
if c.Key == moduleUnhealthyKey(p.Module, p.Node) {
|
||||||
return true
|
return true, false, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, r := range h.healths[p.Node].Resources {
|
for _, r := range h.checked(p.Node) {
|
||||||
if r.Module == p.Module && r.State == link.StateUnhealthy {
|
if r.Module != p.Module {
|
||||||
return true
|
continue
|
||||||
|
}
|
||||||
|
switch r.State {
|
||||||
|
case link.StateUnhealthy:
|
||||||
|
return true, false, nil
|
||||||
|
case link.StateWaiting:
|
||||||
|
waiting = true
|
||||||
|
waits = append(waits, r.Waits...)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return false
|
if !waiting {
|
||||||
|
for _, c := range h.open {
|
||||||
|
waiting = waiting || c.Key == needsOperatorKey(p.Module, p.Node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false, waiting, waits
|
||||||
|
}
|
||||||
|
|
||||||
|
// manifestOf is a module's manifest from the catalogue, read once; false when it cannot be read.
|
||||||
|
func (h *holding) manifestOf(module string) (catalogue.Manifest, bool) {
|
||||||
|
if h.shelf == nil && h.inv != nil {
|
||||||
|
shelf, err := h.inv.Catalogue(h.ctx)
|
||||||
|
if err != nil {
|
||||||
|
return catalogue.Manifest{}, false
|
||||||
|
}
|
||||||
|
h.shelf = shelf
|
||||||
|
}
|
||||||
|
m, ok := h.shelf[module]
|
||||||
|
return m, ok
|
||||||
|
}
|
||||||
|
|
||||||
|
// covering is the waits of a provider that cover a provision it gives (novox/hq issue 405): a module that waits
|
||||||
|
// says nothing else of it is wrong and names each part that waits (ADR 0283 decision 1), so only a consumer of
|
||||||
|
// the waiting part waits on it. A wait covers a provision when its part is that provision, or the secret it waits
|
||||||
|
// for is the provision's shared credential (ADR 0158). Nothing when no wait can be matched: a consumer failing
|
||||||
|
// then is not held, since holding it would hide a fault that may be its own.
|
||||||
|
func (h *holding) covering(p catalogue.Chosen, provision string, waits []inventory.Wait) []inventory.Wait {
|
||||||
|
credential := ""
|
||||||
|
if m, ok := h.manifestOf(p.Module); ok {
|
||||||
|
credential, _ = m.SharedCredentialOf(provision)
|
||||||
|
}
|
||||||
|
var out []inventory.Wait
|
||||||
|
for _, w := range waits {
|
||||||
|
if w.Part == provision || (w.Secret != "" && w.Secret == credential) {
|
||||||
|
out = append(out, w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// heldUnderProvider is the provider a consumer's findings are held under, and — when that provider only waits for the
|
||||||
|
// operator, for the part the consumer needs — the waits that hold it.
|
||||||
|
type heldUnderProvider struct {
|
||||||
|
provider catalogue.Chosen
|
||||||
|
// waits is set when every finding held waits on a provider that only waits for the operator: the consumer's
|
||||||
|
// gate then reads as ADR 0254's waits for a person, a pass with the wait carried (ADR 0283 decision 4).
|
||||||
|
waits []inventory.Wait
|
||||||
}
|
}
|
||||||
|
|
||||||
// heldUnder is the provider a consumer's unhealthy resources wait on: when every one of them is a finding
|
// heldUnder is the provider a consumer's unhealthy resources wait on: when every one of them is a finding
|
||||||
// of a check naming a provision whose provider for this consumer is unhealthy on the record. False when any
|
// of a check naming a provision whose provider for this consumer is unhealthy on the record, or waits for the
|
||||||
// is the consumer's own.
|
// operator for the part that gives it (novox/hq issue 405). False when any is the consumer's own.
|
||||||
func (h *holding) heldUnder(machine, module string, rs []inventory.ResourceHealth) (catalogue.Chosen, bool) {
|
func (h *holding) heldUnder(machine, module string, rs []inventory.ResourceHealth) (catalogue.Chosen, bool) {
|
||||||
var on catalogue.Chosen
|
by, held := h.heldWith(machine, module, rs)
|
||||||
|
return by.provider, held
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *holding) heldWith(machine, module string, rs []inventory.ResourceHealth) (heldUnderProvider, bool) {
|
||||||
|
var on heldUnderProvider
|
||||||
|
onlyWaits := true
|
||||||
for _, r := range rs {
|
for _, r := range rs {
|
||||||
if r.State != link.StateUnhealthy {
|
if r.State != link.StateUnhealthy {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if !heldFinding(r) {
|
if !heldFinding(r) {
|
||||||
return catalogue.Chosen{}, false
|
return heldUnderProvider{}, false
|
||||||
}
|
}
|
||||||
p, ok := h.providerFor(machine, module, r.Needs)
|
p, ok := h.providerFor(machine, module, r.Needs)
|
||||||
if !ok || (p.Node == machine && p.Module == module) || !h.unhealthy(p) {
|
if !ok || (p.Node == machine && p.Module == module) {
|
||||||
|
return heldUnderProvider{}, false
|
||||||
|
}
|
||||||
|
unhealthy, waiting, waits := h.providerState(p)
|
||||||
|
switch {
|
||||||
|
case unhealthy:
|
||||||
|
onlyWaits = false
|
||||||
|
case waiting:
|
||||||
|
covered := h.covering(p, r.Needs, waits)
|
||||||
|
if len(covered) == 0 {
|
||||||
|
return heldUnderProvider{}, false
|
||||||
|
}
|
||||||
|
on.waits = append(on.waits, covered...)
|
||||||
|
default:
|
||||||
|
return heldUnderProvider{}, false
|
||||||
|
}
|
||||||
|
on.provider = p
|
||||||
|
}
|
||||||
|
if !onlyWaits {
|
||||||
|
on.waits = nil
|
||||||
|
}
|
||||||
|
return on, on.provider.Module != ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitingUncovered is a provider of a consumer's failing finding that waits for the operator for a part the
|
||||||
|
// finding cannot be matched to (novox/hq issue 405): the consumer is raised on its own, and its condition says
|
||||||
|
// the provider waits, so neither is hidden.
|
||||||
|
func (h *holding) waitingUncovered(machine, module string, rs []inventory.ResourceHealth) (catalogue.Chosen, bool) {
|
||||||
|
for _, r := range rs {
|
||||||
|
if r.State != link.StateUnhealthy || !heldFinding(r) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
p, ok := h.providerFor(machine, module, r.Needs)
|
||||||
|
if !ok || (p.Node == machine && p.Module == module) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if unhealthy, waiting, waits := h.providerState(p); !unhealthy && waiting && len(h.covering(p, r.Needs, waits)) == 0 {
|
||||||
|
return p, true
|
||||||
|
}
|
||||||
|
}
|
||||||
return catalogue.Chosen{}, false
|
return catalogue.Chosen{}, false
|
||||||
}
|
}
|
||||||
on = p
|
|
||||||
}
|
|
||||||
return on, on.Module != ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// waitersOn is every consumer held under a provider, as "<module> on <machine>", sorted.
|
// waitersOn is every consumer held under a provider, as "<module> on <machine>", sorted.
|
||||||
func (h *holding) waitersOn(p catalogue.Chosen) []string {
|
func (h *holding) waitersOn(p catalogue.Chosen) []string {
|
||||||
@@ -165,8 +295,8 @@ func (h *holding) waitersOn(p catalogue.Chosen) []string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// heldModules is, for one machine's statement, each module whose finding is held, with the provider.
|
// heldModules is, for one machine's statement, each module whose finding is held, with the provider.
|
||||||
func (h *holding) heldModules(machine string) map[string]catalogue.Chosen {
|
func (h *holding) heldModules(machine string) map[string]heldUnderProvider {
|
||||||
out := map[string]catalogue.Chosen{}
|
out := map[string]heldUnderProvider{}
|
||||||
byModule := map[string][]inventory.ResourceHealth{}
|
byModule := map[string][]inventory.ResourceHealth{}
|
||||||
for _, r := range h.healths[machine].Resources {
|
for _, r := range h.healths[machine].Resources {
|
||||||
if r.Module != "" && r.State == link.StateUnhealthy {
|
if r.Module != "" && r.State == link.StateUnhealthy {
|
||||||
@@ -174,7 +304,7 @@ func (h *holding) heldModules(machine string) map[string]catalogue.Chosen {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
for module, rs := range byModule {
|
for module, rs := range byModule {
|
||||||
if on, held := h.heldUnder(machine, module, rs); held {
|
if on, held := h.heldWith(machine, module, rs); held {
|
||||||
out[module] = on
|
out[module] = on
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+111
-8
@@ -76,6 +76,21 @@ func serve(ctx context.Context) (err error) {
|
|||||||
}
|
}
|
||||||
defer open.Close()
|
defer open.Close()
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
|
// **What this build reads of the store's schema, on record** (novox/hq issue 352): the highest migration
|
||||||
|
// it carries, by its version, so a gate that would put this build back later knows it reads the store
|
||||||
|
// as it is then. A build that does not know its version records nothing, and is never put back.
|
||||||
|
if build := runningBuild(); build != "" {
|
||||||
|
if reach, err := schemaReach(); err != nil {
|
||||||
|
fmt.Printf("what this build reads of the store's schema is not recorded: %v\n", err)
|
||||||
|
} else if err := inv.RecordSchemaReach(ctx, build, reach); err != nil {
|
||||||
|
fmt.Printf("what this build (%s) reads of the store's schema is not recorded: %v\n", build, err)
|
||||||
|
} else {
|
||||||
|
fmt.Printf("this build (%s) reads the store's schema up to migration %04d; recorded\n", build, reach)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
fmt.Printf("this process was not told which build it is (%s), so what it reads of the store's schema is not "+
|
||||||
|
"recorded, and a gate will never put it back\n", RunningBuildVar)
|
||||||
|
}
|
||||||
|
|
||||||
ident, err := openIdentity(ctx)
|
ident, err := openIdentity(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -167,6 +182,9 @@ func serve(ctx context.Context) (err error) {
|
|||||||
// Open plans move on a timer as well as on outcomes (novox/hq ADR 0162): a tier waiting for
|
// Open plans move on a timer as well as on outcomes (novox/hq ADR 0162): a tier waiting for
|
||||||
// machines to report moves when they have, and a plan left by a replaced controller resumes.
|
// machines to report moves when they have, and a plan left by a replaced controller resumes.
|
||||||
go planTicker(ctx, open)
|
go planTicker(ctx, open)
|
||||||
|
// And the merge window (novox/hq ADR 0276): a batch is cut into its walk when the window closes, which no
|
||||||
|
// merge or outcome says.
|
||||||
|
go batchCutter(ctx, open)
|
||||||
// And the pending assignments settled on a tick of their own (novox/hq ADR 0261): made once their module
|
// And the pending assignments settled on a tick of their own (novox/hq ADR 0261): made once their module
|
||||||
// is registered, ended with why when its build will not register it, raised and cleared as conditions.
|
// is registered, ended with why when its build will not register it, raised and cleared as conditions.
|
||||||
// Never by a read.
|
// Never by a read.
|
||||||
@@ -339,9 +357,15 @@ func declare(ctx context.Context, args []string) error {
|
|||||||
// The epoch it carried, if a person wrote one in, is what the machine heard.
|
// The epoch it carried, if a person wrote one in, is what the machine heard.
|
||||||
var carried struct {
|
var carried struct {
|
||||||
Epoch uint64 `json:"epoch"`
|
Epoch uint64 `json:"epoch"`
|
||||||
|
Sequence int64 `json:"sequence"`
|
||||||
|
Generation int64 `json:"generation"`
|
||||||
}
|
}
|
||||||
_ = json.Unmarshal(raw, &carried)
|
_ = json.Unmarshal(raw, &carried)
|
||||||
if _, err := recordSent(ctx, inv, node, raw, nil, carried.Epoch); err != nil {
|
// And recorded as every send is (novox/hq issue 234): by hand, from what it carried; the modules it
|
||||||
|
// named are not known, since the mesh did not compose it.
|
||||||
|
if _, err := recordSent(ctx, inv, node, raw, nil, carried.Epoch, sentRecord{sequence: carried.Sequence,
|
||||||
|
epoch: carried.Epoch, generation: carried.Generation, toldGeneration: carried.Generation,
|
||||||
|
sender: senderOf(callerOf(ctx)) + ", a declaration sent by hand"}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw))
|
fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw))
|
||||||
@@ -780,7 +804,7 @@ func composeEach(names []string, allot func(node string) (order, error),
|
|||||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
declared.Sequence, declared.Epoch = numbered.sequence, numbered.epoch
|
numbered.stamp(&declared)
|
||||||
if len(declared.Resources) == 0 {
|
if len(declared.Resources) == 0 {
|
||||||
// Sent, not skipped (novox/hq issue 127). A node whose declaration composes to
|
// Sent, not skipped (novox/hq issue 127). A node whose declaration composes to
|
||||||
// nothing may have HELD something before — the broker opening a placement gave it,
|
// nothing may have HELD something before — the broker opening a placement gave it,
|
||||||
@@ -998,7 +1022,8 @@ func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (stri
|
|||||||
}
|
}
|
||||||
// After it is away, not before. A digest recorded for something that failed to send would make
|
// After it is away, not before. A digest recorded for something that failed to send would make
|
||||||
// the machine look current for a declaration it never received.
|
// the machine look current for a declaration it never received.
|
||||||
digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds, s.declared.Epoch)
|
digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds, s.declared.Epoch,
|
||||||
|
sentRecordOf(ctx, s.declared))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
@@ -1210,7 +1235,7 @@ func sendToEach(ctx context.Context, open *stores, names []string) ([]string, er
|
|||||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
declared.Sequence, declared.Epoch = numbered.sequence, numbered.epoch
|
numbered.stamp(&declared)
|
||||||
reportLeftOut(name, declared)
|
reportLeftOut(name, declared)
|
||||||
sending = append(sending, readyNode{name, declared})
|
sending = append(sending, readyNode{name, declared})
|
||||||
}
|
}
|
||||||
@@ -1275,6 +1300,15 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, se
|
|||||||
if _, err := broker.RaiseBuckets(broker.OnConn(bus.Conn), buckets); err != nil {
|
if _, err := broker.RaiseBuckets(broker.OnConn(bus.Conn), buckets); err != nil {
|
||||||
return fmt.Errorf("the modules' state could not be asserted on the bus: %w", err)
|
return fmt.Errorf("the modules' state could not be asserted on the bus: %w", err)
|
||||||
}
|
}
|
||||||
|
// **And every declared log, for the same reason** (novox/hq ADR 0297 §2): a membership names its
|
||||||
|
// logs, and a module whose log does not exist fails its first append.
|
||||||
|
logs, err := open.inventory.DeclaredLogs(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the modules' logs could not be read, so no log was asserted: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := broker.RaiseLogs(broker.OnConn(bus.Conn), logs); err != nil {
|
||||||
|
return fmt.Errorf("the modules' logs could not be asserted on the bus: %w", err)
|
||||||
|
}
|
||||||
// Every membership is tried, and the first failure named once.
|
// Every membership is tried, and the first failure named once.
|
||||||
issued := 0
|
issued := 0
|
||||||
refused := map[string]error{}
|
refused := map[string]error{}
|
||||||
@@ -1375,6 +1409,11 @@ func wouldSendFrom(ctx context.Context, open *stores,
|
|||||||
if declared.Epoch, err = open.inventory.SentEpoch(ctx, n.ID); err != nil {
|
if declared.Epoch, err = open.inventory.SentEpoch(ctx, n.ID); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
// And the generation it was last sent, for the same reason (novox/hq issue 234): an assignment
|
||||||
|
// elsewhere in the mesh is not a change of this machine.
|
||||||
|
if declared.Generation, err = open.inventory.SentGeneration(ctx, n.ID); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
body, err := declared.Body()
|
body, err := declared.Body()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -1453,13 +1492,28 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string)
|
|||||||
fmt.Printf("the bus holds state nothing declares any more, kept because it is data: %s — "+
|
fmt.Printf("the bus holds state nothing declares any more, kept because it is data: %s — "+
|
||||||
"removing it is a person's act\n", strings.Join(undeclared, ", "))
|
"removing it is a person's act\n", strings.Join(undeclared, ", "))
|
||||||
}
|
}
|
||||||
|
// Every module's log (novox/hq ADR 0297), from the catalogue, as its buckets: one that nothing
|
||||||
|
// declares any more is said and kept — a log is a module's record, and no path of the mesh removes it.
|
||||||
|
logs, err := inv.DeclaredLogs(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
unlogged, err := broker.RaiseLogs(js, logs)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(unlogged) > 0 {
|
||||||
|
fmt.Printf("the bus holds logs nothing declares any more, kept because they are data: %s — "+
|
||||||
|
"removing one is a person's act\n", strings.Join(unlogged, ", "))
|
||||||
|
}
|
||||||
// And how every module hears what it consumes: asserted with the rest above, counted here.
|
// And how every module hears what it consumes: asserted with the rest above, counted here.
|
||||||
hearing, err := moduleConsumerCount(ctx, inv)
|
hearing, err := moduleConsumerCount(ctx, inv)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
fmt.Printf("the bus at %s has its streams, %d machine(s) can hear a declaration, %d module(s) "+
|
fmt.Printf("the bus at %s has its streams, %d machine(s) can hear a declaration, %d module(s) "+
|
||||||
"can hear what they consume, and %d bucket(s) of state\n", broker.BareAddress(address), len(names), hearing, len(buckets))
|
"can hear what they consume, %d bucket(s) of state and %d log(s)\n", broker.BareAddress(address), len(names), hearing,
|
||||||
|
len(buckets), len(logs))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1513,6 +1567,12 @@ var epochForActs = func(ctx context.Context) (uint64, error) { return theLease.e
|
|||||||
type order struct {
|
type order struct {
|
||||||
sequence int64
|
sequence int64
|
||||||
epoch uint64
|
epoch uint64
|
||||||
|
// generation is the assignment generation read before the composition, and readsGeneration whether the
|
||||||
|
// machine's node-engine said it reads one (novox/hq issue 234); acting is the lease epoch the sender acts
|
||||||
|
// under, whether or not the machine is sent it — both kept for the record of the send.
|
||||||
|
generation int64
|
||||||
|
readsGeneration bool
|
||||||
|
acting uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
// allot takes the next sequence for a machine — the number its next declaration carries — under the
|
// allot takes the next sequence for a machine — the number its next declaration carries — under the
|
||||||
@@ -1526,6 +1586,7 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, e
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return order{}, err
|
return order{}, err
|
||||||
}
|
}
|
||||||
|
acting := epoch
|
||||||
if epoch > 0 {
|
if epoch > 0 {
|
||||||
reads, err := inv.ReadsEpoch(ctx, record.ID)
|
reads, err := inv.ReadsEpoch(ctx, record.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1535,11 +1596,24 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, e
|
|||||||
epoch = 0
|
epoch = 0
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// **The generation is read here, before the composition reads a single assignment** (novox/hq issue
|
||||||
|
// 234). A generation only grows, so one read before is never newer than the view composed after it:
|
||||||
|
// the declaration may claim a generation older than its content, never newer — and a claim newer than
|
||||||
|
// the content is exactly the stale send the machine must be able to refuse.
|
||||||
|
generation, err := inv.AssignmentGeneration(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return order{}, err
|
||||||
|
}
|
||||||
|
readsGeneration, err := inv.ReadsGeneration(ctx, record.ID)
|
||||||
|
if err != nil {
|
||||||
|
return order{}, err
|
||||||
|
}
|
||||||
seq, err := inv.NextSequence(ctx, record.ID)
|
seq, err := inv.NextSequence(ctx, record.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return order{}, err
|
return order{}, err
|
||||||
}
|
}
|
||||||
return order{sequence: seq, epoch: epoch}, nil
|
return order{sequence: seq, epoch: epoch, generation: generation, readsGeneration: readsGeneration,
|
||||||
|
acting: acting}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// recordSent writes down what a machine was just sent, and returns the digest.
|
// recordSent writes down what a machine was just sent, and returns the digest.
|
||||||
@@ -1554,7 +1628,7 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, e
|
|||||||
// And the build of each module it carried (novox/hq issue 259, ADR 0221), nil when that is not known:
|
// And the build of each module it carried (novox/hq issue 259, ADR 0221), nil when that is not known:
|
||||||
// what tells a machine held back by a policy or a plan from one a push left behind.
|
// what tells a machine held back by a policy or a plan from one a push left behind.
|
||||||
func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte,
|
func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte,
|
||||||
builds map[string]string, epoch uint64) (string, error) {
|
builds map[string]string, epoch uint64, sent sentRecord) (string, error) {
|
||||||
kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
record, err := inv.NodeByName(kept, node)
|
record, err := inv.NodeByName(kept, node)
|
||||||
@@ -1562,7 +1636,21 @@ func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body
|
|||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
digest := digestOf(body)
|
digest := digestOf(body)
|
||||||
if err := inv.RecordSentUnder(kept, record.ID, digest, builds, epoch); err != nil {
|
// The digest and the generation it carried are written in one transaction (novox/hq issue 234), so the
|
||||||
|
// would-send is never stamped with a generation the machine was not sent; and the send itself, who sent
|
||||||
|
// it and from which generation, beside them. A record of the send that cannot be written does not make a
|
||||||
|
// send that is away look failed: it is said, loudly, and the machine's own record stands.
|
||||||
|
err = inv.RecordSentWith(kept, record.ID, digest, builds, epoch, sent.toldGeneration, inventory.Send{
|
||||||
|
Sequence: sent.sequence, Epoch: int64(sent.epoch), Sender: sent.sender, Generation: sent.generation,
|
||||||
|
Digest: digest, Modules: sent.modules})
|
||||||
|
var unrecorded *inventory.SendNotRecordedError
|
||||||
|
switch {
|
||||||
|
case errors.As(err, &unrecorded):
|
||||||
|
fmt.Fprintf(os.Stderr, "mesh-controller: SEND NOT RECORDED: %s was sent declaration %s (sequence %d), and who "+
|
||||||
|
"sent it and from which generation could not be written down, so a refusal of it will name no sender "+
|
||||||
|
"(novox/hq issue 234): %v\n", node, short(digest), sent.sequence, unrecorded.Err)
|
||||||
|
fmt.Printf("%s: sent, and the record of who sent it could NOT be written: %v\n", node, unrecorded.Err)
|
||||||
|
case err != nil:
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
// And what it was, summarised, so the next push can be compared with it (novox/hq ADR 0217).
|
// And what it was, summarised, so the next push can be compared with it (novox/hq ADR 0217).
|
||||||
@@ -1598,3 +1686,18 @@ func reportUnheldPushed(w io.Writer, named bool, asked []string, unheld map[stri
|
|||||||
fmt.Fprintf(w, "%s: %d unmet seat dependenc(ies) — see `status`\n", node, len(lines))
|
fmt.Fprintf(w, "%s: %d unmet seat dependenc(ies) — see `status`\n", node, len(lines))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// schemaReach is the highest migration this build carries for the inventory's store (novox/hq issue 352).
|
||||||
|
func schemaReach() (int, error) {
|
||||||
|
migrations, err := inventory.Migrations()
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
reach := 0
|
||||||
|
for _, m := range migrations {
|
||||||
|
if m.Number > reach {
|
||||||
|
reach = m.Number
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return reach, nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -135,12 +135,19 @@ func TestRetryRefusesWhatItCannotResume(t *testing.T) {
|
|||||||
if err := retryRefusal(stopped, nil); err == nil || !strings.Contains(err.Error(), "nothing in tier 0") {
|
if err := retryRefusal(stopped, nil); err == nil || !strings.Contains(err.Error(), "nothing in tier 0") {
|
||||||
t.Errorf("a plan with nothing failed to build was retried: %v", err)
|
t.Errorf("a plan with nothing failed to build was retried: %v", err)
|
||||||
}
|
}
|
||||||
// Another branch's newer plan, an older one, and a failed one do not supersede it.
|
// A failed or done plan does not supersede it.
|
||||||
|
if err := retryRefusal(failed, []inventory.Plan{plan("plan-4", inventory.PlanFailed, at.Add(time.Hour)),
|
||||||
|
plan("plan-5", inventory.PlanDone, at.Add(time.Hour))}); err != nil {
|
||||||
|
t.Errorf("refused for a plan that does not supersede it: %v", err)
|
||||||
|
}
|
||||||
|
// Another branch's open walk, or an older one, does not supersede it, and is one walk open: one at a time
|
||||||
|
// (novox/hq ADR 0276).
|
||||||
other := plan("plan-3", inventory.PlanBuilding, at.Add(time.Hour))
|
other := plan("plan-3", inventory.PlanBuilding, at.Add(time.Hour))
|
||||||
other.Branch = "release"
|
other.Branch = "release"
|
||||||
if err := retryRefusal(failed, []inventory.Plan{other, plan("plan-0", inventory.PlanBuilding, at.Add(-time.Hour)),
|
for _, open := range []inventory.Plan{other, plan("plan-0", inventory.PlanBuilding, at.Add(-time.Hour))} {
|
||||||
plan("plan-4", inventory.PlanFailed, at.Add(time.Hour))}); err != nil {
|
if err := retryRefusal(failed, []inventory.Plan{open}); err == nil || !strings.Contains(err.Error(), "one walk at a time") {
|
||||||
t.Errorf("refused for a plan that does not supersede it: %v", err)
|
t.Errorf("retried beside the open walk %s: %v", open.ID, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -180,12 +180,35 @@ func (f moveFacts) moves(node string, modules []string, sent map[string]string,
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// walkedBy is the open plan that has started walking a module's build — sent it to a first machine,
|
// walkedBy is the open plan that has started walking a module's build — sent it to a first machine, not
|
||||||
// not yet passed — other than to this machine; empty when none does.
|
// yet passed — and whose walk this move would cross; empty when none does. A move of the same build to a
|
||||||
func (f moveFacts) walkedBy(module, node string) string {
|
// machine that plan already sent it is not a crossing: the build is there. A move of **another** build of
|
||||||
|
// the module to that machine is (novox/hq issue 352): on 2026-10-09 a merge's plan sent the control node a
|
||||||
|
// newer controller while a release's gate was judging the controller there, the release's gate read the
|
||||||
|
// machine's report against the newer send, failed three builds and put them back under the new plan's
|
||||||
|
// feet. A release keeps no module records: its open gate's carried moves are its walk.
|
||||||
|
func (f moveFacts) walkedBy(module, node, to string) string {
|
||||||
for _, p := range f.plans {
|
for _, p := range f.plans {
|
||||||
|
if !p.Open() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if p.Release != nil {
|
||||||
|
g := p.Release.Gate
|
||||||
|
if g == nil || g.Verdict != "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, c := range g.Carried {
|
||||||
|
if c.Module == module && !(slices.Contains(g.Machines, node) && sameCommit(c.To, to)) {
|
||||||
|
return p.ID
|
||||||
|
}
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
s, holds := p.Modules[module]
|
s, holds := p.Modules[module]
|
||||||
if !p.Open() || !holds || s == nil || s.FirstAt == nil || s.SentAt != nil || slices.Contains(s.First, node) {
|
if !holds || s == nil || s.FirstAt == nil || s.SentAt != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if slices.Contains(s.First, node) && sameCommit(s.Commit, to) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if s.Gate != nil && s.Gate.Verdict == inventory.GatePassed {
|
if s.Gate != nil && s.Gate.Verdict == inventory.GatePassed {
|
||||||
@@ -277,11 +300,19 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
|
|||||||
if own(mv.Module) {
|
if own(mv.Module) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if id := f.walkedBy(mv.Module, node); id != "" {
|
if id := f.walkedBy(mv.Module, node, mv.To); id != "" {
|
||||||
return nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
return nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
||||||
mv.Module, short(mv.To), node, id)
|
mv.Module, short(mv.To), node, id)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// **And the plan's own modules wait too** (novox/hq issue 352): a walk of the same module by another
|
||||||
|
// plan, or a release, on this machine is not crossed with a newer build; this send waits for its gate.
|
||||||
|
for _, o := range owns {
|
||||||
|
if id := f.walkedBy(o.Module, node, o.To); id != "" {
|
||||||
|
return nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
||||||
|
o.Module, short(o.To), node, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
for _, o := range owns {
|
for _, o := range owns {
|
||||||
i := slices.IndexFunc(moves, func(mv inventory.CarriedMove) bool { return mv.Module == o.Module })
|
i := slices.IndexFunc(moves, func(mv inventory.CarriedMove) bool { return mv.Module == o.Module })
|
||||||
switch {
|
switch {
|
||||||
@@ -526,7 +557,7 @@ func waitingMoves(ctx context.Context, open *stores, all bool) (map[string][]inv
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
for _, mv := range moves {
|
for _, mv := range moves {
|
||||||
if f.walkedBy(mv.Module, n.Name) == "" {
|
if f.walkedBy(mv.Module, n.Name, mv.To) == "" {
|
||||||
out[n.Name] = append(out[n.Name], mv)
|
out[n.Name] = append(out[n.Name], mv)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -658,7 +689,7 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
|||||||
r.Next++
|
r.Next++
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves}
|
r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves, Sent: sentNow(ctx, open.inventory, sent)}
|
||||||
p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves))
|
p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves))
|
||||||
if said := recreationsSaid(moves); said != "" {
|
if said := recreationsSaid(moves); said != "" {
|
||||||
p.Note += "; " + said
|
p.Note += "; " + said
|
||||||
@@ -693,6 +724,15 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
|||||||
r.Next++
|
r.Next++
|
||||||
r.Gate = nil
|
r.Gate = nil
|
||||||
return true, nil
|
return true, nil
|
||||||
|
case inventory.GateSuperseded:
|
||||||
|
// Another send moved a judged module on the judged machine (novox/hq issue 352): no verdict on what
|
||||||
|
// was carried, nothing put back, and this release ends; the builds still waiting are released again
|
||||||
|
// by the next pass, judged afresh.
|
||||||
|
p.State = inventory.PlanSuperseded
|
||||||
|
p.Note = fmt.Sprintf("superseded on %s: %s — nothing judged, nothing put back; what still waits is released again",
|
||||||
|
strings.Join(g.Machines, ", "), g.Why)
|
||||||
|
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||||
|
return true, nil
|
||||||
}
|
}
|
||||||
p.Note = ""
|
p.Note = ""
|
||||||
batched, back := batchingRollbacks(ctx)
|
batched, back := batchingRollbacks(ctx)
|
||||||
|
|||||||
+293
-141
@@ -138,9 +138,11 @@ func reachableFrom(moved []string, edges []inventory.Edge) []string {
|
|||||||
grew = false
|
grew = false
|
||||||
for _, e := range edges {
|
for _, e := range edges {
|
||||||
// Built-by and worker-of order a plan; neither widens it. A new build machine changes
|
// Built-by and worker-of order a plan; neither widens it. A new build machine changes
|
||||||
// nothing it builds, and a new controller changes nothing about the holder it orders —
|
// nothing it builds, and a new controller changes nothing about the holder it orders. A
|
||||||
// what packages the controller's source is already a code edge.
|
// packages edge, read from a record made before novox/hq ADR 0267, widens nothing either:
|
||||||
if e.Kind == inventory.EdgeBuiltBy || e.Kind == inventory.EdgeWorkerOf {
|
// a shared file moves each module whose build source holds it, directly.
|
||||||
|
if e.Kind == inventory.EdgeBuiltBy || e.Kind == inventory.EdgeWorkerOf || e.Kind == inventory.EdgePackages ||
|
||||||
|
e.Kind == edgeGroupOrder {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if in[e.To] && !in[e.From] {
|
if in[e.To] && !in[e.From] {
|
||||||
@@ -179,22 +181,26 @@ func hasCycle(tiers [][]string, edges []inventory.Edge) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// planFor is the plan a merge produces: the moved modules and everything reachable from them,
|
// planOfMerge is the plan one merge produces: the moved modules and everything reachable from them,
|
||||||
// tiered, with the merge it answers.
|
// tiered, with the merge it answers.
|
||||||
func planOfMerge(m link.SourceMoved, moved []string, edges []inventory.Edge) inventory.Plan {
|
func planOfMerge(m link.SourceMoved, moved []string, edges []inventory.Edge) inventory.Plan {
|
||||||
|
p := planOfMoves(moved, edges)
|
||||||
|
merged, _ := time.Parse(time.RFC3339Nano, m.MergedAt)
|
||||||
|
p.Repository, p.Branch, p.Commit, p.Merged = m.Owner+"/"+m.Repo, m.Base, m.Commit, merged.UTC()
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
// planOfMoves is the walk of a set of moved modules (novox/hq ADR 0162, 0276): they and everything reachable
|
||||||
|
// from them, tiered along the graph, with no merge named yet.
|
||||||
|
func planOfMoves(moved []string, edges []inventory.Edge) inventory.Plan {
|
||||||
set := reachableFrom(moved, edges)
|
set := reachableFrom(moved, edges)
|
||||||
tiers := tiersOf(set, edges)
|
tiers := tiersOf(set, edges)
|
||||||
modules := map[string]*inventory.PlanModule{}
|
modules := map[string]*inventory.PlanModule{}
|
||||||
for _, name := range set {
|
for _, name := range set {
|
||||||
modules[name] = &inventory.PlanModule{}
|
modules[name] = &inventory.PlanModule{}
|
||||||
}
|
}
|
||||||
merged, _ := time.Parse(time.RFC3339Nano, m.MergedAt)
|
|
||||||
return inventory.Plan{
|
return inventory.Plan{
|
||||||
ID: fmt.Sprintf("plan-%d", time.Now().UnixNano()),
|
ID: fmt.Sprintf("plan-%d", time.Now().UnixNano()),
|
||||||
Repository: m.Owner + "/" + m.Repo,
|
|
||||||
Branch: m.Base,
|
|
||||||
Commit: m.Commit,
|
|
||||||
Merged: merged.UTC(),
|
|
||||||
Created: time.Now().UTC(),
|
Created: time.Now().UTC(),
|
||||||
State: inventory.PlanBuilding,
|
State: inventory.PlanBuilding,
|
||||||
Tiers: tiers,
|
Tiers: tiers,
|
||||||
@@ -202,92 +208,6 @@ func planOfMerge(m link.SourceMoved, moved []string, edges []inventory.Edge) inv
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// supersededBy is what a newer plan takes over from the open plans it supersedes (novox/hq issue
|
|
||||||
// 254, ADR 0218): the modules they had not finished, and those plans closed as superseded.
|
|
||||||
//
|
|
||||||
// **A merge looked at no plan but its own.** Two merges of one repository a few minutes apart were
|
|
||||||
// two open plans asking for the same modules, each sending machines what it built; and a plan that
|
|
||||||
// would never move again — waiting on a report that could not come, at 97b1b2b — stayed open for
|
|
||||||
// ever beside the newer ones, read as work in progress by everyone who looked. The newer merge is the
|
|
||||||
// newer intent for that repository and branch, so its plan takes over: every open plan of the same
|
|
||||||
// repository and branch **created before it** — by the time the plans were made, never by comparing
|
|
||||||
// commits, which have no order of their own — gives up the modules it had not built, and those are
|
|
||||||
// planned again in the newer plan beside what the newer merge moved.
|
|
||||||
//
|
|
||||||
// "Not built" is a module not yet asked, or asked and not answered; **and a module built and not
|
|
||||||
// yet sent to its machines**, where its policy rolls it out: closed, the older plan would never send
|
|
||||||
// it, and the catalogue announces no move for a rebuild (issue 189), so the newer plan builds and
|
|
||||||
// sends it. A build the older plan asked still finishes and registers as any build does — ordered by
|
|
||||||
// when it was asked (issue 219), so the newer plan's ask, made later, is the one that stands.
|
|
||||||
//
|
|
||||||
// A plan with no branch recorded is from before branches were kept, and is superseded by the next
|
|
||||||
// plan of its repository: what it had not built is folded in, so nothing is lost by it.
|
|
||||||
//
|
|
||||||
// **Newer is the branch's order, not the plans'** (novox/hq issue 349). A merge the bus did not hand
|
|
||||||
// over is acted on late, by the catch-up, so its plan is made after the plan of a merge that came after
|
|
||||||
// it — and that later-made plan of the earlier commit superseded the later merge's, and built what it
|
|
||||||
// folded in from the commit before the later merge: twice on 2026-10-09, once a security fix. So where
|
|
||||||
// both plans know when their merge was made, that decides; only where one does not do the plans' own
|
|
||||||
// times. A merge older than the newest planned merge of its branch never reaches here at its own commit:
|
|
||||||
// it is planned at that merge's commit, which contains it (laterOnTheBranch).
|
|
||||||
func supersededBy(newer inventory.Plan, open []inventory.Plan, rollsOut func(string) bool) ([]string, []inventory.Plan) {
|
|
||||||
folded := map[string]bool{}
|
|
||||||
var closed []inventory.Plan
|
|
||||||
for _, old := range open {
|
|
||||||
if old.ID == newer.ID || !old.Open() || !strings.EqualFold(old.Repository, newer.Repository) ||
|
|
||||||
(old.Branch != "" && old.Branch != newer.Branch) || !earlierOnTheBranch(old, newer) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
var took []string
|
|
||||||
for name, s := range old.Modules {
|
|
||||||
if s != nil && s.State == planDeleted {
|
|
||||||
continue // deleted at its source: nothing to plan again
|
|
||||||
}
|
|
||||||
if s == nil || s.State != "built" || (s.SentAt == nil && rollsOut(name)) {
|
|
||||||
folded[name] = true
|
|
||||||
took = append(took, name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.Strings(took)
|
|
||||||
old.State = inventory.PlanSuperseded
|
|
||||||
old.Note = fmt.Sprintf("superseded at tier %d by %s (%s at %s)", old.Tier, newer.ID, newer.Repository, short(newer.Commit))
|
|
||||||
if len(took) > 0 {
|
|
||||||
old.Note += "; " + strings.Join(took, ", ") + " planned there again"
|
|
||||||
}
|
|
||||||
closed = append(closed, old)
|
|
||||||
}
|
|
||||||
out := make([]string, 0, len(folded))
|
|
||||||
for name := range folded {
|
|
||||||
out = append(out, name)
|
|
||||||
}
|
|
||||||
sort.Strings(out)
|
|
||||||
return out, closed
|
|
||||||
}
|
|
||||||
|
|
||||||
// earlierOnTheBranch says plan a answers a merge made before b's: by when the forge made each merge where
|
|
||||||
// both are known, else by when each plan was made. A merge's own plan made again at the same commit
|
|
||||||
// (the same merge time) is ordered by when it was made. Pure.
|
|
||||||
func earlierOnTheBranch(a, b inventory.Plan) bool {
|
|
||||||
if !a.Merged.IsZero() && !b.Merged.IsZero() && !a.Merged.Equal(b.Merged) {
|
|
||||||
return a.Merged.Before(b.Merged)
|
|
||||||
}
|
|
||||||
return a.Created.Before(b.Created)
|
|
||||||
}
|
|
||||||
|
|
||||||
// laterOnTheBranch says the plan newest answers a merge into m's branch made after m: then newest's commit
|
|
||||||
// contains m's change, and m is planned there, so the newest commit of the branch is what is built (novox/hq
|
|
||||||
// issue 349). newest is the newest merge's plan of the branch in any state: a later plan already done
|
|
||||||
// built what it moved at its commit, and m planned at its own would build m's dependents back at the older
|
|
||||||
// one. Pure.
|
|
||||||
func laterOnTheBranch(m link.SourceMoved, newest inventory.Plan) bool {
|
|
||||||
merged, err := time.Parse(time.RFC3339Nano, m.MergedAt)
|
|
||||||
if err != nil || newest.Release != nil || newest.Commit == m.Commit {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return strings.EqualFold(newest.Repository, m.Owner+"/"+m.Repo) && newest.Branch == m.Base &&
|
|
||||||
newest.Merged.After(merged)
|
|
||||||
}
|
|
||||||
|
|
||||||
// gates is what the next tier needs running from this one: a module of the tier that a later
|
// gates is what the next tier needs running from this one: a module of the tier that a later
|
||||||
// tier is built by — the runtime dependency — and whose policy rolls it out, must be applied by
|
// tier is built by — the runtime dependency — and whose policy rolls it out, must be applied by
|
||||||
// the machines running it before the next tier is asked. A base an image stands on need only be
|
// the machines running it before the next tier is asked. A base an image stands on need only be
|
||||||
@@ -315,8 +235,11 @@ func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool)
|
|||||||
seen := map[string]bool{}
|
seen := map[string]bool{}
|
||||||
var out []string
|
var out []string
|
||||||
for _, e := range edges {
|
for _, e := range edges {
|
||||||
if later[e.From] && inTier[e.To] && e.Kind == inventory.EdgeBuiltBy && !seen[e.To] && rollsOut(e.To) &&
|
// A delivery group's order inside a walk (novox/hq ADR 0276 decision 5) gates as built-by does: the
|
||||||
!isBaseOf(e.From, e.To, edges, all) {
|
// member before is running on its machines before the member after is asked.
|
||||||
|
ordered := e.Kind == edgeGroupOrder ||
|
||||||
|
e.Kind == inventory.EdgeBuiltBy && !isBaseOf(e.From, e.To, edges, all)
|
||||||
|
if later[e.From] && inTier[e.To] && ordered && !seen[e.To] && rollsOut(e.To) {
|
||||||
seen[e.To] = true
|
seen[e.To] = true
|
||||||
out = append(out, e.To)
|
out = append(out, e.To)
|
||||||
}
|
}
|
||||||
@@ -407,8 +330,15 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
|
|||||||
}
|
}
|
||||||
source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
|
source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
|
||||||
fmt.Printf(" tier %d: ", p.Tier)
|
fmt.Printf(" tier %d: ", p.Tier)
|
||||||
// The branch it follows, never a commit a build once named (novox/hq 04-ISSUES/215).
|
// The branch it follows, never a commit a build once named (novox/hq 04-ISSUES/215): the branch contains every
|
||||||
id, err := askABuild(ctx, source, e.Source.Path, followedBranch(e.Source.Ref))
|
// commit a batch's walk carries — but for a merge walked alone after a failed walk (novox/hq ADR 0276
|
||||||
|
// decision 3), built on its own commit to find which merge brought the failure.
|
||||||
|
ref := followedBranch(e.Source.Ref)
|
||||||
|
carried := p.CommitOn(e.Source.Repository, ref)
|
||||||
|
if p.Delivery != nil && p.Delivery.Alone && carried != "" {
|
||||||
|
ref = carried
|
||||||
|
}
|
||||||
|
id, err := askABuild(ctx, source, e.Source.Path, ref)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
state.State = "failed"
|
state.State = "failed"
|
||||||
state.Why = err.Error()
|
state.Why = err.Error()
|
||||||
@@ -416,8 +346,14 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
|
|||||||
p.Note = fmt.Sprintf("%s could not be asked for: %v", name, err)
|
p.Note = fmt.Sprintf("%s could not be asked for: %v", name, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
// The commit of the module's own repository the walk carries (novox/hq ADR 0276): a batch's walk carries
|
||||||
|
// one per repository.
|
||||||
|
commit := carried
|
||||||
|
if commit == "" {
|
||||||
|
commit = p.Commit
|
||||||
|
}
|
||||||
recordAsked(ctx, inventory.BuildRequest{ID: id, Repository: e.Source.Repository, Seat: e.Source.Seat,
|
recordAsked(ctx, inventory.BuildRequest{ID: id, Repository: e.Source.Repository, Seat: e.Source.Seat,
|
||||||
Path: e.Source.Path, Ref: followedBranch(e.Source.Ref), Commit: p.Commit, For: "plan"})
|
Path: e.Source.Path, Ref: followedBranch(e.Source.Ref), Commit: commit, For: "plan"})
|
||||||
state.State = "asked"
|
state.State = "asked"
|
||||||
state.AskedAt = &now
|
state.AskedAt = &now
|
||||||
state.Build = id
|
state.Build = id
|
||||||
@@ -510,6 +446,47 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string,
|
|||||||
advanceHeld(ctx, open)
|
advanceHeld(ctx, open)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// startedBeside is the id of a started walk open beside this one — a merge's walk past its wait, not the
|
||||||
|
// backlog's — or empty: one walk at a time (novox/hq ADR 0276).
|
||||||
|
func startedBeside(ctx context.Context, inv *inventory.Inventory, id string, created time.Time) (string, error) {
|
||||||
|
plans, err := inv.OpenPlans(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
for _, q := range plans {
|
||||||
|
if q.ID == id || q.Release != nil || q.Waiting() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Started: a tier asked, or on its way (let go, or waiting for nobody) before this one.
|
||||||
|
if q.Tier > 0 || askedAny(q) || q.Created.Before(created) {
|
||||||
|
return q.ID, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// deferredNote begins the note of a walk deferred behind another.
|
||||||
|
const deferredNote = "let go; starts once "
|
||||||
|
|
||||||
|
// deferred says a walk has its word and has not started: let go while another walk was started, it waits for
|
||||||
|
// that one to end (startedBeside), and its note says so. Read as a wait, not as a tier running late: `plans`
|
||||||
|
// and `status` say its note, and S3 leaves it out. A let-go walk whose first ask failed carries that error as
|
||||||
|
// its note instead, and is watched as before.
|
||||||
|
func deferred(p inventory.Plan) bool {
|
||||||
|
return p.Open() && p.Release == nil && p.Tier == 0 && !askedAny(p) && p.Delivery != nil &&
|
||||||
|
p.Delivery.Awaits != "" && p.Delivery.Go != nil && strings.HasPrefix(p.Note, deferredNote)
|
||||||
|
}
|
||||||
|
|
||||||
|
// askedAny says a plan asked any module.
|
||||||
|
func askedAny(p inventory.Plan) bool {
|
||||||
|
for _, s := range p.Modules {
|
||||||
|
if s != nil && s.State != "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// advancePlans moves every open plan as far as the facts allow: a tier whose modules are all built
|
// advancePlans moves every open plan as far as the facts allow: a tier whose modules are all built
|
||||||
// and whose gates are applied gives way to the next; the last tier done is the plan done. Called
|
// and whose gates are applied gives way to the next; the last tier done is the plan done. Called
|
||||||
// after every outcome and on a timer, so a plan waiting on a machine's report moves when it comes.
|
// after every outcome and on a timer, so a plan waiting on a machine's report moves when it comes.
|
||||||
@@ -533,9 +510,23 @@ func advancePlans(ctx context.Context, open *stores) {
|
|||||||
advanceHeld(ctx, open)
|
advanceHeld(ctx, open)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// keepWalkPhases keeps where the walks that ended lately spent their time (novox/hq ADR 0282), outside the hold
|
||||||
|
// on the plans so it never lengthens it: measured, never acted on, and an error only said.
|
||||||
|
func keepWalkPhases(ctx context.Context, open *stores) {
|
||||||
|
if err := recordWalkPhases(ctx, open.inventory, time.Now()); err != nil {
|
||||||
|
fmt.Printf("plans: the phases of the walks ended lately could not be kept: %v\n", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// advanceHeld is advancePlans for a caller already holding the plans.
|
// advanceHeld is advancePlans for a caller already holding the plans.
|
||||||
func advanceHeld(ctx context.Context, open *stores) {
|
func advanceHeld(ctx context.Context, open *stores) {
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
|
// A walk that ended lets the next batch be cut at once, not at the cutter's next look (novox/hq ADR 0276).
|
||||||
|
defer func() {
|
||||||
|
if err := cutBatchesHeld(ctx, open, time.Now().UTC()); err != nil {
|
||||||
|
fmt.Printf("batches: %v\n", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
plans, err := inv.OpenPlans(ctx)
|
plans, err := inv.OpenPlans(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Printf("plans: cannot read them: %v\n", err)
|
fmt.Printf("plans: cannot read them: %v\n", err)
|
||||||
@@ -636,6 +627,18 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if unasked == len(tier) {
|
if unasked == len(tier) {
|
||||||
|
// **One walk at a time** (novox/hq ADR 0276): a walk about to ask its first tier while another started
|
||||||
|
// walk is open waits for that one to end, let go or not, and says so.
|
||||||
|
if p.Tier == 0 {
|
||||||
|
if behind, err := startedBeside(ctx, inv, p.ID, p.Created); err != nil {
|
||||||
|
return false, err
|
||||||
|
} else if behind != "" {
|
||||||
|
note := fmt.Sprintf("%s%s ended — one walk at a time", deferredNote, behind)
|
||||||
|
changed := p.Note != note
|
||||||
|
p.Note = note
|
||||||
|
return changed, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
if err := askTier(ctx, inv, p); err != nil {
|
if err := askTier(ctx, inv, p); err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
@@ -646,27 +649,10 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
// the controller in its first tier: the build that produced the new one is recorded, and the
|
// the controller in its first tier: the build that produced the new one is recorded, and the
|
||||||
// plan never hears it. The record is the fact; a build recorded after the ask is that tier's
|
// plan never hears it. The record is the fact; a build recorded after the ask is that tier's
|
||||||
// outcome, whoever was listening.
|
// outcome, whoever was listening.
|
||||||
recorded := map[string][]inventory.Build{}
|
recorded, byID, err := recordsOfAsked(ctx, inv, p, tier)
|
||||||
byID := map[string]inventory.Build{}
|
|
||||||
for _, m := range tier {
|
|
||||||
if s := p.Modules[m]; s != nil && s.State == "asked" {
|
|
||||||
builds, err := inv.Builds(ctx, m, 5)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
recorded[m] = builds
|
|
||||||
// Its own ask's record, by id — found even when the outcome named no module (ADR 0219).
|
|
||||||
if s.Build != "" {
|
|
||||||
b, found, err := inv.BuildByID(ctx, s.Build)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
if found {
|
|
||||||
byID[s.Build] = b
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if settleFromRecords(p, tier, recorded, byID) {
|
if settleFromRecords(p, tier, recorded, byID) {
|
||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
@@ -793,6 +779,15 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
case inventory.GateFailed:
|
case inventory.GateFailed:
|
||||||
failFirstSend(ctx, open, p, m, state, state.Gate.Machines, state.Gate.Why, step.rest)
|
failFirstSend(ctx, open, p, m, state, state.Gate.Machines, state.Gate.Why, step.rest)
|
||||||
return true, nil
|
return true, nil
|
||||||
|
case inventory.GateSuperseded:
|
||||||
|
// Another send moved this module on its first machine (novox/hq issue 352): the build is not
|
||||||
|
// judged, not marked, not put back; the plan ends here, said, and a newer plan carries on.
|
||||||
|
state.Why = "superseded: " + state.Gate.Why
|
||||||
|
p.State = inventory.PlanSuperseded
|
||||||
|
p.Note = fmt.Sprintf("%s's judging on %s was superseded: %s", m, strings.Join(state.Gate.Machines, ", "),
|
||||||
|
state.Gate.Why)
|
||||||
|
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||||
|
return true, nil
|
||||||
case inventory.GatePassed:
|
case inventory.GatePassed:
|
||||||
if !state.Gate.Kept {
|
if !state.Gate.Kept {
|
||||||
gatePassed(ctx, open, p, m, state)
|
gatePassed(ctx, open, p, m, state)
|
||||||
@@ -860,8 +855,12 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
strings.Join(machines, ", "), p.Tier, err)
|
strings.Join(machines, ", "), p.Tier, err)
|
||||||
}
|
}
|
||||||
now := time.Now().UTC()
|
now := time.Now().UTC()
|
||||||
|
// What each machine of the rest was sent, kept for when it reports it applied (novox/hq ADR 0282 decision
|
||||||
|
// 6): measured, never acted on.
|
||||||
|
sentWhat := sentNow(ctx, open.inventory, sent)
|
||||||
for _, m := range rest {
|
for _, m := range rest {
|
||||||
p.Modules[m].SentAt = &now
|
p.Modules[m].SentAt = &now
|
||||||
|
p.Modules[m].Rest = restOf(restTo[m], sent, sentWhat)
|
||||||
}
|
}
|
||||||
fmt.Printf("%s: tier %d built; sent %s to %s, one send each\n", p.ID, p.Tier, strings.Join(rest, ", "),
|
fmt.Printf("%s: tier %d built; sent %s to %s, one send each\n", p.ID, p.Tier, strings.Join(rest, ", "),
|
||||||
strings.Join(sent, ", "))
|
strings.Join(sent, ", "))
|
||||||
@@ -942,6 +941,20 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// restOf is, for one module, every machine of its rest that the send reached and what it carried there.
|
||||||
|
func restOf(to, sent []string, what map[string]inventory.SentDeclaration) map[string]inventory.SentDeclaration {
|
||||||
|
out := map[string]inventory.SentDeclaration{}
|
||||||
|
for _, n := range to {
|
||||||
|
if slices.Contains(sent, n) {
|
||||||
|
out[n] = what[n]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(out) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// firstSend sends one machine, in one send, every module of the plan's tier whose first machine it is
|
// firstSend sends one machine, in one send, every module of the plan's tier whose first machine it is
|
||||||
// (novox/hq issue 281), and records the send on each: what that machine ran of it before — read once,
|
// (novox/hq issue 281), and records the send on each: what that machine ran of it before — read once,
|
||||||
// before the send, so a module the same send carries is never read as already moved — the machines it
|
// before the send, so a module the same send carries is never read as already moved — the machines it
|
||||||
@@ -964,6 +977,9 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
|||||||
}
|
}
|
||||||
now := time.Now().UTC()
|
now := time.Now().UTC()
|
||||||
lead := modules[0]
|
lead := modules[0]
|
||||||
|
// What each machine was just sent, kept on the gate (novox/hq issue 352): its report is held against
|
||||||
|
// this send, whatever it is sent after.
|
||||||
|
sentWhat := sentNow(ctx, inv, sent)
|
||||||
for _, m := range modules {
|
for _, m := range modules {
|
||||||
s := p.Modules[m]
|
s := p.Modules[m]
|
||||||
// What the first machine ran before: what a failed gate puts back (ADR 0236).
|
// What the first machine ran before: what a failed gate puts back (ADR 0236).
|
||||||
@@ -972,7 +988,7 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
|||||||
}
|
}
|
||||||
s.First, s.FirstAt = sent, &now
|
s.First, s.FirstAt = sent, &now
|
||||||
s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning(sent, runningOf[m]),
|
s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning(sent, runningOf[m]),
|
||||||
From: s.Previous, To: s.Commit, Since: &now}
|
From: s.Previous, To: s.Commit, Since: &now, Sent: sentWhat}
|
||||||
s.GatedBy = ""
|
s.GatedBy = ""
|
||||||
if m == lead {
|
if m == lead {
|
||||||
s.Gate.Carried = carried
|
s.Gate.Carried = carried
|
||||||
@@ -1131,8 +1147,15 @@ func nextRollout(s inventory.PlanModule, running []string, together bool, report
|
|||||||
var waiting, failed []string
|
var waiting, failed []string
|
||||||
for _, n := range s.First {
|
for _, n := range s.First {
|
||||||
r, said := byNode[n]
|
r, said := byNode[n]
|
||||||
// Only a report about what it was last sent says anything about this build.
|
// Only a report about what this plan sent it — or what it was sent after that — says anything about
|
||||||
if !said || r.At == nil || !r.Current {
|
// this build (novox/hq issue 352); a plan from before sends were kept on the gate reads the report
|
||||||
|
// against the send made last, as before.
|
||||||
|
reported := r.Current
|
||||||
|
if s.Gate != nil && s.Gate.Sent != nil {
|
||||||
|
sent, kept := s.Gate.Sent[n]
|
||||||
|
reported = kept && sent.ReportsOn(r)
|
||||||
|
}
|
||||||
|
if !said || r.At == nil || !reported {
|
||||||
waiting = append(waiting, n)
|
waiting = append(waiting, n)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -1179,6 +1202,7 @@ func sayUnsent(p *inventory.Plan, rollsOut func(string) bool) {
|
|||||||
// planTicker advances open plans on a timer, for the steps outcomes alone cannot take.
|
// planTicker advances open plans on a timer, for the steps outcomes alone cannot take.
|
||||||
func planTicker(ctx context.Context, open *stores) {
|
func planTicker(ctx context.Context, open *stores) {
|
||||||
advancePlans(ctx, open)
|
advancePlans(ctx, open)
|
||||||
|
keepWalkPhases(ctx, open)
|
||||||
tick := time.NewTicker(30 * time.Second)
|
tick := time.NewTicker(30 * time.Second)
|
||||||
defer tick.Stop()
|
defer tick.Stop()
|
||||||
for {
|
for {
|
||||||
@@ -1187,6 +1211,7 @@ func planTicker(ctx context.Context, open *stores) {
|
|||||||
return
|
return
|
||||||
case <-tick.C:
|
case <-tick.C:
|
||||||
advancePlans(ctx, open)
|
advancePlans(ctx, open)
|
||||||
|
keepWalkPhases(ctx, open)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1220,24 +1245,38 @@ func inTierSince(p inventory.Plan) time.Time {
|
|||||||
// planLineWith is planLine knowing whether the build seat is paused (novox/hq ADR 0219): a plan
|
// planLineWith is planLine knowing whether the build seat is paused (novox/hq ADR 0219): a plan
|
||||||
// waiting on builds nobody will take until a person resumes the seat says so, and is not late. bound is
|
// waiting on builds nobody will take until a person resumes the seat says so, and is not late. bound is
|
||||||
// the plan's tier bound, the one its stalled condition is raised at (tierBounds): LATE is that condition
|
// the plan's tier bound, the one its stalled condition is raised at (tierBounds): LATE is that condition
|
||||||
// said on the line (novox/hq issue 296).
|
// said on the line (novox/hq issue 296). The machines running what it moves are not named: planLineOn.
|
||||||
func planLineWith(p inventory.Plan, now time.Time, pause pauseView, bound time.Duration) string {
|
func planLineWith(p inventory.Plan, now time.Time, pause pauseView, bound time.Duration) string {
|
||||||
|
return planLineOn(p, now, pause, bound, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// planLineOn is planLineWith naming the plan by what it moves and where (planHeadline), given what runs each
|
||||||
|
// module; nil names no machine.
|
||||||
|
func planLineOn(p inventory.Plan, now time.Time, pause pauseView, bound time.Duration, running func(string) []string) string {
|
||||||
|
name := planHeadline(p, running)
|
||||||
where := fmt.Sprintf("tier %d of %d", min(p.Tier+1, len(p.Tiers)), len(p.Tiers))
|
where := fmt.Sprintf("tier %d of %d", min(p.Tier+1, len(p.Tiers)), len(p.Tiers))
|
||||||
switch p.State {
|
switch p.State {
|
||||||
|
case inventory.PlanAssembling, inventory.PlanQueued:
|
||||||
|
// A batch not yet a walk (novox/hq ADR 0276): what it holds and how long is left.
|
||||||
|
return batchWords(p, now)
|
||||||
case inventory.PlanDone:
|
case inventory.PlanDone:
|
||||||
return fmt.Sprintf("%s %s done, %d tier(s)", p.Repository, short(p.Commit), len(p.Tiers))
|
return fmt.Sprintf("%s · done, %d tier(s)", name, len(p.Tiers))
|
||||||
case inventory.PlanFailed:
|
case inventory.PlanFailed:
|
||||||
return fmt.Sprintf("%s %s FAILED at %s: %s", p.Repository, short(p.Commit), where, p.Note)
|
return fmt.Sprintf("%s · FAILED at %s: %s", name, where, p.Note)
|
||||||
case inventory.PlanSuperseded:
|
case inventory.PlanSuperseded:
|
||||||
return fmt.Sprintf("%s %s %s", p.Repository, short(p.Commit), p.Note)
|
return fmt.Sprintf("%s · %s", name, p.Note)
|
||||||
}
|
}
|
||||||
since := now.Sub(inTierSince(p)).Round(time.Second)
|
since := now.Sub(inTierSince(p)).Round(time.Second)
|
||||||
if p.Waiting() {
|
if p.Waiting() {
|
||||||
// Waiting for its delivery's word is no lateness of the walk's (novox/hq ADR 0239).
|
// Waiting for its delivery's word is no lateness of the walk's (novox/hq ADR 0239).
|
||||||
return fmt.Sprintf("%s %s %s, %s, for %s", p.Repository, short(p.Commit), where, waitingNote(p), since)
|
return fmt.Sprintf("%s · %s, %s, for %s", name, where, waitingNote(p), since)
|
||||||
|
}
|
||||||
|
if deferred(p) {
|
||||||
|
// Nor is waiting for the walk before it to end (one walk at a time, novox/hq ADR 0276).
|
||||||
|
return fmt.Sprintf("%s · %s, %s, for %s", name, where, p.Note, since)
|
||||||
}
|
}
|
||||||
if waiting, paused := pausedWaiting(p, pause, now); paused {
|
if waiting, paused := pausedWaiting(p, pause, now); paused {
|
||||||
return fmt.Sprintf("%s %s %s, %s", p.Repository, short(p.Commit), where, waiting)
|
return fmt.Sprintf("%s · %s, %s", name, where, waiting)
|
||||||
}
|
}
|
||||||
late := ""
|
late := ""
|
||||||
if since > bound {
|
if since > bound {
|
||||||
@@ -1247,7 +1286,53 @@ func planLineWith(p inventory.Plan, now time.Time, pause pauseView, bound time.D
|
|||||||
if p.State == inventory.PlanRolling {
|
if p.State == inventory.PlanRolling {
|
||||||
what = p.Note
|
what = p.Note
|
||||||
}
|
}
|
||||||
return fmt.Sprintf("%s %s %s, %s for %s%s", p.Repository, short(p.Commit), where, what, since, late)
|
return fmt.Sprintf("%s · %s, %s for %s%s", name, where, what, since, late)
|
||||||
|
}
|
||||||
|
|
||||||
|
// planHeadline names a plan as a person knows it (asked by the operator, 2026-10-10: a plan called by its
|
||||||
|
// repository alone said nothing of what it delivers): each repository as its pull request and title, what the
|
||||||
|
// plan moves, and the machines running that — "mesh-catalog #175 a tap shows its outcome · messenger,
|
||||||
|
// telegram → novox". A record naming no pull request is named by its commit, as before; one naming no moves
|
||||||
|
// says none; running nil names no machine.
|
||||||
|
func planHeadline(p inventory.Plan, running func(string) []string) string {
|
||||||
|
var repos []string
|
||||||
|
moves := map[string]bool{}
|
||||||
|
for _, c := range p.Carried() {
|
||||||
|
seg := repoName(c.Repository) + " " + short(c.Commit)
|
||||||
|
if p.Delivery != nil {
|
||||||
|
for _, m := range p.Delivery.Merges {
|
||||||
|
if !strings.EqualFold(m.Repository, c.Repository) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, n := range m.Moves {
|
||||||
|
moves[n] = true
|
||||||
|
}
|
||||||
|
if m.Commit == c.Commit && m.Number > 0 {
|
||||||
|
seg = repoName(c.Repository) + " " + pullWords(m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
repos = append(repos, seg)
|
||||||
|
}
|
||||||
|
out := strings.Join(repos, " + ")
|
||||||
|
if len(moves) == 0 {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
names := sortedKeysOf(boolsToStrings(moves))
|
||||||
|
out += " · " + strings.Join(names, ", ")
|
||||||
|
if running == nil {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
nodes := map[string]bool{}
|
||||||
|
for _, n := range names {
|
||||||
|
for _, node := range running(n) {
|
||||||
|
nodes[node] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(nodes) > 0 {
|
||||||
|
out += " → " + strings.Join(sortedKeysOf(boolsToStrings(nodes)), ", ")
|
||||||
|
}
|
||||||
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// planFailedBuild marks the module a failed build was for when the result names no module: by the
|
// planFailedBuild marks the module a failed build was for when the result names no module: by the
|
||||||
@@ -1389,8 +1474,11 @@ func plansCommand(ctx context.Context, args []string) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
bounds := readTierBounds(ctx, inv, now)
|
bounds := readTierBounds(ctx, inv, now)
|
||||||
fmt.Printf("%s — %s\n", p.ID, planLineWith(p, now, buildSeatPause(ctx, inv, []inventory.Plan{p}),
|
fmt.Printf("%s — %s\n", p.ID, planLineOn(p, now, buildSeatPause(ctx, inv, []inventory.Plan{p}),
|
||||||
bounds.of(p.Repository)))
|
bounds.of(p.Repository), func(module string) []string {
|
||||||
|
on, _ := inv.Running(ctx, module)
|
||||||
|
return on
|
||||||
|
}))
|
||||||
if r := p.Release; r != nil {
|
if r := p.Release; r != nil {
|
||||||
// A release plan's walk (ADR 0236): machines done, the one judged, those to come.
|
// A release plan's walk (ADR 0236): machines done, the one judged, those to come.
|
||||||
fmt.Printf(" machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "),
|
fmt.Printf(" machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "),
|
||||||
@@ -1514,14 +1602,34 @@ func plansCommand(ctx context.Context, args []string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if len(plans) == 0 {
|
// **The batch being assembled first** (novox/hq ADR 0276 decision 7): what it holds, how long is left,
|
||||||
|
// and that its plan is not yet calculated.
|
||||||
|
batches, err := inv.Batches(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(plans) == 0 && len(batches) == 0 {
|
||||||
fmt.Println("no merge has produced a plan yet")
|
fmt.Println("no merge has produced a plan yet")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
for _, b := range batches {
|
||||||
|
fmt.Printf("%-28s %s\n", b.ID, batchWords(b, now))
|
||||||
|
// One line per repository: its pull request, what it answers, what it moves.
|
||||||
|
for _, line := range batchLines(b) {
|
||||||
|
fmt.Printf("%-28s %s\n", "", line)
|
||||||
|
}
|
||||||
|
}
|
||||||
pause := buildSeatPause(ctx, inv, plans)
|
pause := buildSeatPause(ctx, inv, plans)
|
||||||
bounds := readTierBounds(ctx, inv, now)
|
bounds := readTierBounds(ctx, inv, now)
|
||||||
|
running := func(module string) []string {
|
||||||
|
on, _ := inv.Running(ctx, module)
|
||||||
|
return on
|
||||||
|
}
|
||||||
for _, p := range plans {
|
for _, p := range plans {
|
||||||
fmt.Printf("%-28s %s\n", p.ID, planLineWith(p, now, pause, bounds.of(p.Repository)))
|
if p.Batch() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fmt.Printf("%-28s %s\n", p.ID, planLineOn(p, now, pause, bounds.of(p.Repository), running))
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -1542,11 +1650,12 @@ func planWhatIf(ctx context.Context, inv *inventory.Inventory, repository string
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
read, err := inv.ReadRepositories(ctx)
|
read, err := readForPlanning(ctx, inv)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
var from, packaging []inventory.Entry
|
var from, packaging []inventory.Entry
|
||||||
|
deleted := map[string]bool{}
|
||||||
named := map[string]bool{}
|
named := map[string]bool{}
|
||||||
for _, name := range modules {
|
for _, name := range modules {
|
||||||
named[name] = true
|
named[name] = true
|
||||||
@@ -1556,6 +1665,9 @@ func planWhatIf(ctx context.Context, inv *inventory.Inventory, repository string
|
|||||||
// request's check ask it (novox/hq ADR 0238).
|
// request's check ask it (novox/hq ADR 0238).
|
||||||
r := reachOfMerge(m, entries, read, nil)
|
r := reachOfMerge(m, entries, read, nil)
|
||||||
from, packaging = append(append([]inventory.Entry{}, r.Touched...), r.Deleted...), r.Packaging
|
from, packaging = append(append([]inventory.Entry{}, r.Touched...), r.Deleted...), r.Packaging
|
||||||
|
for _, e := range r.Deleted {
|
||||||
|
deleted[e.Manifest.Module] = true
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
for _, e := range entries {
|
for _, e := range entries {
|
||||||
switch {
|
switch {
|
||||||
@@ -1582,6 +1694,18 @@ func planWhatIf(ctx context.Context, inv *inventory.Inventory, repository string
|
|||||||
}
|
}
|
||||||
p := planOfMerge(m, names, edges)
|
p := planOfMerge(m, names, edges)
|
||||||
fmt.Printf("a merge of %s would build %d module(s) in %d tier(s):\n", repository, len(p.Modules), len(p.Tiers))
|
fmt.Printf("a merge of %s would build %d module(s) in %d tier(s):\n", repository, len(p.Modules), len(p.Tiers))
|
||||||
|
why := map[string]string{}
|
||||||
|
for _, e := range packaging {
|
||||||
|
why[e.Manifest.Module] = whyMoved(e, read[e.Manifest.Module], m)
|
||||||
|
}
|
||||||
|
if len(named) == 0 {
|
||||||
|
for _, e := range from {
|
||||||
|
why[e.Manifest.Module] = whyMoved(e, read[e.Manifest.Module], m)
|
||||||
|
if deleted[e.Manifest.Module] {
|
||||||
|
why[e.Manifest.Module] = "its manifest is removed: deleted at its source, forgotten where nothing holds it, not built"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
rolls := map[string]string{}
|
rolls := map[string]string{}
|
||||||
for i, tier := range p.Tiers {
|
for i, tier := range p.Tiers {
|
||||||
fmt.Printf(" tier %d\n", i)
|
fmt.Printf(" tier %d\n", i)
|
||||||
@@ -1599,19 +1723,19 @@ func planWhatIf(ctx context.Context, inv *inventory.Inventory, repository string
|
|||||||
rolls[name] = how
|
rolls[name] = how
|
||||||
}
|
}
|
||||||
fmt.Printf(" %-22s %s\n", name, how)
|
fmt.Printf(" %-22s %s\n", name, how)
|
||||||
|
reason := why[name]
|
||||||
|
switch {
|
||||||
|
case reason == "" && len(named) > 0 && named[name]:
|
||||||
|
reason = "named"
|
||||||
|
case reason == "":
|
||||||
|
reason = "it stands on a module the merge moves"
|
||||||
|
}
|
||||||
|
fmt.Printf(" %-22s why: %s\n", "", reason)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if hasCycle(p.Tiers, edges) {
|
if hasCycle(p.Tiers, edges) {
|
||||||
fmt.Println(" the last tier depends on itself and would be built together, in no order")
|
fmt.Println(" the last tier depends on itself and would be built together, in no order")
|
||||||
}
|
}
|
||||||
if len(packaging) > 0 {
|
|
||||||
var also []string
|
|
||||||
for _, e := range packaging {
|
|
||||||
also = append(also, e.Manifest.Module)
|
|
||||||
}
|
|
||||||
fmt.Printf(" %s package source from %s, so they are rebuilt without their own source moving\n",
|
|
||||||
strings.Join(also, ", "), repository)
|
|
||||||
}
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1635,6 +1759,34 @@ func splitList(s string) []string {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// recordsOfAsked is what settleFromRecords reads: for every module of the tier still `asked`, its last
|
||||||
|
// builds and the record of its own ask, by id.
|
||||||
|
func recordsOfAsked(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan, tier []string) (
|
||||||
|
map[string][]inventory.Build, map[string]inventory.Build, error) {
|
||||||
|
recorded := map[string][]inventory.Build{}
|
||||||
|
byID := map[string]inventory.Build{}
|
||||||
|
for _, m := range tier {
|
||||||
|
if s := p.Modules[m]; s != nil && s.State == "asked" {
|
||||||
|
builds, err := inv.Builds(ctx, m, 5)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
recorded[m] = builds
|
||||||
|
// Its own ask's record, by id — found even when the outcome named no module (ADR 0219).
|
||||||
|
if s.Build != "" {
|
||||||
|
b, found, err := inv.BuildByID(ctx, s.Build)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
if found {
|
||||||
|
byID[s.Build] = b
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return recorded, byID, nil
|
||||||
|
}
|
||||||
|
|
||||||
// settleFromRecords marks every module of the tier still `asked` built — or failed — from a build
|
// settleFromRecords marks every module of the tier still `asked` built — or failed — from a build
|
||||||
// recorded after it was asked, and says whether it changed anything (novox/hq 04-ISSUES/214).
|
// recorded after it was asked, and says whether it changed anything (novox/hq 04-ISSUES/214).
|
||||||
// Newest first, as Builds answers: the first record after the ask is the outcome of that ask.
|
// Newest first, as Builds answers: the first record after the ask is the outcome of that ask.
|
||||||
|
|||||||
@@ -59,17 +59,18 @@ func TestAMergeIsPlannedInTiersAlongTheThreeKindsOfDependency(t *testing.T) {
|
|||||||
t.Fatalf("no cycle here: %v", tiers)
|
t.Fatalf("no cycle here: %v", tiers)
|
||||||
}
|
}
|
||||||
|
|
||||||
// The controller alone moved: the proxy with it, nothing else.
|
// The controller alone moved: the controller alone — what packages its repository moves only when the
|
||||||
small := reachableFrom([]string{"mesh-controller"}, edges)
|
// change is in its own build source (novox/hq ADR 0267), so a packages edge widens nothing.
|
||||||
if len(small) != 3 {
|
if alone := reachableFrom([]string{"mesh-controller"}, edges); len(alone) != 1 {
|
||||||
t.Fatalf("a controller merge rebuilds the controller and what packages it: %v", small)
|
t.Fatalf("a controller merge rebuilds the controller alone: %v", alone)
|
||||||
}
|
}
|
||||||
// The builder and the proxy package the controller's source, which orders nothing. The builder holds
|
// A change to a package all three build from moves all three. The builder holds
|
||||||
// the build seat, whose worker the controller defines, so it follows the controller (worker-of,
|
// the build seat, whose worker the controller defines, so it follows the controller (worker-of,
|
||||||
// novox/hq issue 206), and the controller's built-by edge to it yields: the controller is built by the
|
// novox/hq issue 206), and the controller's built-by edge to it yields: the controller is built by the
|
||||||
// build machine that is running. The proxy is built by the new builder: the controller, the builder,
|
// build machine that is running. The proxy is built by the new builder: the controller, the builder,
|
||||||
// the proxy — the live plan of every controller merge. (This read "the builder, then the controller
|
// the proxy — the live plan of every controller merge. (This read "the builder, then the controller
|
||||||
// and the proxy together" before issue 206, and the fixture had no worker-of edge.)
|
// and the proxy together" before issue 206, and the fixture had no worker-of edge.)
|
||||||
|
small := reachableFrom([]string{"mesh-controller", "builder", "route-proxy"}, edges)
|
||||||
smallTiers := tiersOf(small, edges)
|
smallTiers := tiersOf(small, edges)
|
||||||
if got := tiered(smallTiers); got != "mesh-controller | builder | route-proxy" {
|
if got := tiered(smallTiers); got != "mesh-controller | builder | route-proxy" {
|
||||||
t.Fatalf("the controller, then the builder, then the proxy: %v", smallTiers)
|
t.Fatalf("the controller, then the builder, then the proxy: %v", smallTiers)
|
||||||
@@ -243,7 +244,7 @@ func TestAChangeToTheBuildAgentRebuildsTheBuildAgentAlone(t *testing.T) {
|
|||||||
} {
|
} {
|
||||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "abc", Paths: paths,
|
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "abc", Paths: paths,
|
||||||
ModuleDirs: []string{"modules/build-agent"}, ModuleDirsSaid: true}
|
ModuleDirs: []string{"modules/build-agent"}, ModuleDirsSaid: true}
|
||||||
touched := whatTheMergeTouched(entries, entries, m)
|
touched := whatTheMergeTouched(entries, entries, m, nil)
|
||||||
if len(touched) != 1 || touched[0].Manifest.Module != "build-agent" {
|
if len(touched) != 1 || touched[0].Manifest.Module != "build-agent" {
|
||||||
t.Fatalf("%v touched %v", paths, touched)
|
t.Fatalf("%v touched %v", paths, touched)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -28,6 +28,11 @@ import (
|
|||||||
func TestMain(m *testing.M) {
|
func TestMain(m *testing.M) {
|
||||||
// The process a mesh-cli test runs as a command line: it says the verb and the caller it was given, and
|
// The process a mesh-cli test runs as a command line: it says the verb and the caller it was given, and
|
||||||
// ends (meshcli_test.go).
|
// ends (meshcli_test.go).
|
||||||
|
// The process a mesh-cli test runs as `secret accept`: it reads its value exactly as `secret accept` does
|
||||||
|
// (valueFor) and says whether it is the one the test gave, never the value (meshcli_stdin_test.go).
|
||||||
|
if want := os.Getenv(secretAcceptWants); want != "" {
|
||||||
|
os.Exit(readAsSecretAccept(want, os.Args[1:]))
|
||||||
|
}
|
||||||
if os.Getenv(echoEnvironment) != "" {
|
if os.Getenv(echoEnvironment) != "" {
|
||||||
fmt.Printf("verb=%q caller=%q terminal=%v\n", os.Getenv("MESH_VERB"), os.Getenv("MESH_CALLER"), startedAtTheTerminal())
|
fmt.Printf("verb=%q caller=%q terminal=%v\n", os.Getenv("MESH_VERB"), os.Getenv("MESH_CALLER"), startedAtTheTerminal())
|
||||||
os.Exit(0)
|
os.Exit(0)
|
||||||
|
|||||||
@@ -260,10 +260,10 @@ func refusedAsTheGenericCommand(argv []string) error {
|
|||||||
// A layer is written through the settings verb, never the generic one (novox/hq issue 339): the
|
// A layer is written through the settings verb, never the generic one (novox/hq issue 339): the
|
||||||
// settings verb is where what a verb may not set is refused, and one route is one set of words.
|
// settings verb is where what a verb may not set is refused, and one route is one set of words.
|
||||||
// The command refuses places and accesses through any verb as well; this says so before it runs.
|
// The command refuses places and accesses through any verb as well; this says so before it runs.
|
||||||
if argv[0] == "settings" && slices.ContainsFunc(argv[1:], func(w string) bool { return w == "set" || w == "clear" }) {
|
if argv[0] == "settings" && slices.ContainsFunc(argv[1:], func(w string) bool { return w == "set" || w == "clear" || w == "propose" }) {
|
||||||
return &heldAtTheTerminal{msg: "settings are set and cleared through the settings verb, not the " +
|
return &heldAtTheTerminal{msg: "settings are set, cleared and proposed through the settings verb, not the " +
|
||||||
"generic command; and places and accesses only at the controller's terminal (novox/hq issue 339). " +
|
"generic command; and places and accesses are set at the controller's terminal or on the operator's " +
|
||||||
"Nothing was done"}
|
"warrant (novox/hq issue 339, ADR 0277). Nothing was done"}
|
||||||
}
|
}
|
||||||
// The generic verb only reads (novox/hq ADR 0266): what writes has a named verb that composes its own
|
// The generic verb only reads (novox/hq ADR 0266): what writes has a named verb that composes its own
|
||||||
// line, or is the operator's at the controller's terminal.
|
// line, or is the operator's at the controller's terminal.
|
||||||
@@ -536,6 +536,11 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
|||||||
argv = append(argv, "--condition", c)
|
argv = append(argv, "--condition", c)
|
||||||
}
|
}
|
||||||
return argv, nil
|
return argv, nil
|
||||||
|
case "warranted":
|
||||||
|
if err := need("asker", "ask"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return []string{"hand-act", "warrant", "--asker", str("asker"), "--ask", str("ask")}, nil
|
||||||
case "hand-acts":
|
case "hand-acts":
|
||||||
argv := []string{"hand-acts", "--json"}
|
argv := []string{"hand-acts", "--json"}
|
||||||
if d := str("days"); d != "" {
|
if d := str("days"); d != "" {
|
||||||
@@ -762,6 +767,23 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
|||||||
argv = append(argv, "--probe", p)
|
argv = append(argv, "--probe", p)
|
||||||
}
|
}
|
||||||
return append(argv, "--json"), nil
|
return append(argv, "--json"), nil
|
||||||
|
case "give":
|
||||||
|
// The same line as secret-ask with the desk named (novox/hq ADR 0277): one path, one set of bounds.
|
||||||
|
if err := need("node", "module", "secret", "at"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return []string{"secret", "ask", str("node"), str("module"), str("secret"), "--at", str("at")}, nil
|
||||||
|
case "secret-ask":
|
||||||
|
// A module's own secret asked for, typed by the operator at the desk (novox/hq ADR 0277): never a value
|
||||||
|
// in the arguments. The desk is the module's machine unless at names another.
|
||||||
|
if err := need("node", "module", "secret"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
argv := []string{"secret", "ask", str("node"), str("module"), str("secret")}
|
||||||
|
if at := str("at"); at != "" {
|
||||||
|
argv = append(argv, "--at", at)
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
case "rotate":
|
case "rotate":
|
||||||
if p := str("provision"); p != "" {
|
if p := str("provision"); p != "" {
|
||||||
argv := []string{"rotate", p}
|
argv := []string{"rotate", p}
|
||||||
@@ -825,6 +847,21 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
|||||||
if str("module") == "" && on("clear") {
|
if str("module") == "" && on("clear") {
|
||||||
return nil, errors.New("settings: a module is needed to clear a layer; name it with module")
|
return nil, errors.New("settings: a module is needed to clear a layer; name it with module")
|
||||||
}
|
}
|
||||||
|
// The proposals, listed or one whole (novox/hq ADR 0277): a read, needing no module.
|
||||||
|
if on("proposals") || str("proposal") != "" {
|
||||||
|
if str("module") != "" || str("values") != "" || str("node") != "" || on("clear") || on("replace") ||
|
||||||
|
on("history") || str("list") != "" || on("propose") || (on("proposals") && str("proposal") != "") {
|
||||||
|
return nil, errors.New("settings: proposals lists what was proposed and proposal shows one; either takes nothing else")
|
||||||
|
}
|
||||||
|
argv := []string{"settings", "proposals"}
|
||||||
|
if id := str("proposal"); id != "" {
|
||||||
|
argv = append(argv, id)
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
|
}
|
||||||
|
if str("module") == "" && on("propose") {
|
||||||
|
return nil, errors.New("settings: a module is needed to propose a layer; name it with module")
|
||||||
|
}
|
||||||
if list := str("list"); list != "" || str("module") == "" {
|
if list := str("list"); list != "" || str("module") == "" {
|
||||||
if list != "" && list != "preferences" {
|
if list != "" && list != "preferences" {
|
||||||
return nil, fmt.Errorf("settings lists %q only; %q is not a listing", "preferences", list)
|
return nil, fmt.Errorf("settings lists %q only; %q is not a listing", "preferences", list)
|
||||||
@@ -845,6 +882,22 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
|||||||
}
|
}
|
||||||
var argv []string
|
var argv []string
|
||||||
switch {
|
switch {
|
||||||
|
case on("propose"):
|
||||||
|
// A proposal: the values, or clear, put to the operator (novox/hq ADR 0277). Nothing is set here.
|
||||||
|
argv = []string{"settings", "propose", str("module")}
|
||||||
|
switch {
|
||||||
|
case on("clear") && str("values") != "":
|
||||||
|
return nil, errors.New("settings: a proposal gives values or says clear, not both")
|
||||||
|
case on("clear"):
|
||||||
|
argv = append(argv, "--clear")
|
||||||
|
case str("values") != "":
|
||||||
|
argv = append(argv, str("values"))
|
||||||
|
if on("replace") {
|
||||||
|
argv = append(argv, "--replace")
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return nil, errors.New("settings: a proposal gives the values, or says clear")
|
||||||
|
}
|
||||||
case on("clear"):
|
case on("clear"):
|
||||||
argv = []string{"settings", "clear", str("module")}
|
argv = []string{"settings", "clear", str("module")}
|
||||||
case str("values") != "":
|
case str("values") != "":
|
||||||
@@ -930,6 +983,8 @@ func repairingCommand(argv []string) string {
|
|||||||
return "plans " + argv[1]
|
return "plans " + argv[1]
|
||||||
case argv[0] == "broker" && len(argv) > 1 && argv[1] == "consumer-reset":
|
case argv[0] == "broker" && len(argv) > 1 && argv[1] == "consumer-reset":
|
||||||
return "broker consumer-reset"
|
return "broker consumer-reset"
|
||||||
|
case argv[0] == "hand-act" && len(argv) > 1 && argv[1] == "warrant":
|
||||||
|
return "" // the router's record of a person's answer, never a repair (novox/hq ADR 0274)
|
||||||
case argv[0] == "hand-act" && len(argv) > 1 && argv[1] == "drill":
|
case argv[0] == "hand-act" && len(argv) > 1 && argv[1] == "drill":
|
||||||
return "hand-act drill"
|
return "hand-act drill"
|
||||||
case argv[0] == "hand-act":
|
case argv[0] == "hand-act":
|
||||||
@@ -1311,7 +1366,7 @@ func splitCommandLine(line string) ([]string, error) {
|
|||||||
var words []string
|
var words []string
|
||||||
var cur strings.Builder
|
var cur strings.Builder
|
||||||
inWord := false
|
inWord := false
|
||||||
quote := rune(0)
|
quote, opened := rune(0), 0
|
||||||
runes := []rune(line)
|
runes := []rune(line)
|
||||||
for i := 0; i < len(runes); i++ {
|
for i := 0; i < len(runes); i++ {
|
||||||
r := runes[i]
|
r := runes[i]
|
||||||
@@ -1332,7 +1387,7 @@ func splitCommandLine(line string) ([]string, error) {
|
|||||||
cur.WriteRune(r)
|
cur.WriteRune(r)
|
||||||
}
|
}
|
||||||
case r == '\'' || r == '"':
|
case r == '\'' || r == '"':
|
||||||
quote = r
|
quote, opened = r, i
|
||||||
inWord = true
|
inWord = true
|
||||||
case r == '\\' && i+1 < len(runes):
|
case r == '\\' && i+1 < len(runes):
|
||||||
i++
|
i++
|
||||||
@@ -1350,7 +1405,7 @@ func splitCommandLine(line string) ([]string, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if quote != 0 {
|
if quote != 0 {
|
||||||
return nil, fmt.Errorf("command has an unclosed %c quote", quote)
|
return nil, unclosedQuote(quote, opened)
|
||||||
}
|
}
|
||||||
if inWord {
|
if inWord {
|
||||||
words = append(words, cur.String())
|
words = append(words, cur.String())
|
||||||
@@ -1358,6 +1413,17 @@ func splitCommandLine(line string) ([]string, error) {
|
|||||||
return words, nil
|
return words, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// unclosedQuote is the refusal of a line whose quote is never closed. Most often an apostrophe inside
|
||||||
|
// a single-quoted value ended that quote early and a later quote was left open, so the refusal says
|
||||||
|
// where the open quote is and how a quote is written inside a quoted value — the shell's own two
|
||||||
|
// ways, which this splitter already reads (novox/hq issue 294). It quotes none of the line: a refusal
|
||||||
|
// is kept on the bus as the call's answer, and the line may carry a setting's value.
|
||||||
|
func unclosedQuote(quote rune, opened int) error {
|
||||||
|
return fmt.Errorf("command has an unclosed %c quote, opened at character %d — an apostrophe "+
|
||||||
|
"inside a single-quoted value ends it; write a ' inside single quotes as '\\'' (it'\\''s), or use "+
|
||||||
|
"double quotes and write \\\" for a \" and \\\\ for a \\ inside them", quote, opened+1)
|
||||||
|
}
|
||||||
|
|
||||||
// seatAnnouncement is what the controller says it serves on the bus (novox/hq ADR 0197): the
|
// seatAnnouncement is what the controller says it serves on the bus (novox/hq ADR 0197): the
|
||||||
// mesh-controller seat, one endpoint per verb it answers, each with the seat's own description and
|
// mesh-controller seat, one endpoint per verb it answers, each with the seat's own description and
|
||||||
// argument schema — the same facts `tools` answers from the records, as NATS's services format.
|
// argument schema — the same facts `tools` answers from the records, as NATS's services format.
|
||||||
@@ -1435,7 +1501,7 @@ var commandReadForms = map[string]func(rest []string) bool{
|
|||||||
"conditions": func(r []string) bool { return flagsOnly(r) || subIn(r, "list", "show", "history") },
|
"conditions": func(r []string) bool { return flagsOnly(r) || subIn(r, "list", "show", "history") },
|
||||||
"node": func(r []string) bool { return subIn(r, "list", "show") },
|
"node": func(r []string) bool { return subIn(r, "list", "show") },
|
||||||
"module": func(r []string) bool { return subIn(r, "list") },
|
"module": func(r []string) bool { return subIn(r, "list") },
|
||||||
"settings": func(r []string) bool { return subIn(r, "show", "preferences") },
|
"settings": func(r []string) bool { return subIn(r, "show", "preferences", "proposals") },
|
||||||
"retire": func(r []string) bool { return subIn(r, "list") },
|
"retire": func(r []string) bool { return subIn(r, "list") },
|
||||||
"cleanup": func(r []string) bool { return subIn(r, "list") },
|
"cleanup": func(r []string) bool { return subIn(r, "list") },
|
||||||
"delivery": func(r []string) bool { return subIn(r, "plan", "walks") },
|
"delivery": func(r []string) bool { return subIn(r, "plan", "walks") },
|
||||||
@@ -1495,6 +1561,24 @@ var terminalOnlyCommands = map[string]string{
|
|||||||
"licence": "the licences' secrets",
|
"licence": "the licences' secrets",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// givenAtTheDesk is exactly the line the `give` and `secret-ask` verbs compose, and nothing beside it: `secret ask
|
||||||
|
// <node> <module> <secret>`, with `--at <machine>` or no other word — no value, no file, no provider (novox/hq
|
||||||
|
// ADR 0277). The terminal's own `secret accept … --at-desk` is the terminal's.
|
||||||
|
func givenAtTheDesk(argv []string) bool {
|
||||||
|
if (len(argv) != 5 && len(argv) != 7) || argv[0] != "secret" || argv[1] != "ask" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, w := range argv[2:5] {
|
||||||
|
if w == "" || strings.HasPrefix(w, "-") {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(argv) == 5 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return argv[5] == "--at" && argv[6] != "" && !strings.HasPrefix(argv[6], "-")
|
||||||
|
}
|
||||||
|
|
||||||
// terminalOnly refuses, through any verb, a command that is the operator's at the controller's terminal
|
// terminalOnly refuses, through any verb, a command that is the operator's at the controller's terminal
|
||||||
// alone (novox/hq ADR 0266). **Every `node` subcommand that is not a read**: `node account` and
|
// alone (novox/hq ADR 0266). **Every `node` subcommand that is not a read**: `node account` and
|
||||||
// `node agent-account` above all. Whoever may call a verb includes agents, and an agent that named itself
|
// `node agent-account` above all. Whoever may call a verb includes agents, and an agent that named itself
|
||||||
@@ -1508,8 +1592,10 @@ func terminalOnly(argv []string) error {
|
|||||||
return terminalRefusal("%s is run at the controller's terminal only, never through a verb: it holds %s, and "+
|
return terminalRefusal("%s is run at the controller's terminal only, never through a verb: it holds %s, and "+
|
||||||
"whoever may call a verb includes agents (novox/hq ADR 0266). Nothing was done", argv[0], what)
|
"whoever may call a verb includes agents (novox/hq ADR 0266). Nothing was done", argv[0], what)
|
||||||
}
|
}
|
||||||
// Of a secret's commands only rotation, which seals the new value to the machine that uses it.
|
// Of a secret's commands only rotation, which seals the new value to the machine that uses it, and the
|
||||||
if argv[0] == "secret" && (len(argv) < 2 || argv[1] != "rotate") {
|
// `give` verb's own line: an own secret typed by the operator into the desk's hidden prompt, sealed to this
|
||||||
|
// call and then to the module's machine, so no value travels in the verb or its answer (hq ADR 0259 §10).
|
||||||
|
if argv[0] == "secret" && (len(argv) < 2 || argv[1] != "rotate") && !givenAtTheDesk(argv) {
|
||||||
return terminalRefusal("secret %s is run at the controller's terminal only, never through a verb: accepting, "+
|
return terminalRefusal("secret %s is run at the controller's terminal only, never through a verb: accepting, "+
|
||||||
"recovering or exporting a secret hands it to whoever asks, and that includes agents (novox/hq ADR "+
|
"recovering or exporting a secret hands it to whoever asks, and that includes agents (novox/hq ADR "+
|
||||||
"0266). Nothing was done", strings.Join(argv[1:], " "))
|
"0266). Nothing was done", strings.Join(argv[1:], " "))
|
||||||
|
|||||||
@@ -275,6 +275,7 @@ var accountedFlags = map[string]map[string]string{
|
|||||||
"json": "set by the verb: the answer is data",
|
"json": "set by the verb: the answer is data",
|
||||||
"all": "withheld: every measurement of a fortnight is more than a call should carry; `command` reaches it",
|
"all": "withheld: every measurement of a fortnight is more than a call should carry; `command` reaches it",
|
||||||
},
|
},
|
||||||
|
// The desk path of `secret accept` (novox/hq ADR 0259 §10): a value is never an argument of a call.
|
||||||
"hand-acts": {"json": "set by the verb: the answer is data"},
|
"hand-acts": {"json": "set by the verb: the answer is data"},
|
||||||
"conditions": {"json": "set by the verb: the answer is data"},
|
"conditions": {"json": "set by the verb: the answer is data"},
|
||||||
"retire": {"json": "set by the verb: the answer is data"},
|
"retire": {"json": "set by the verb: the answer is data"},
|
||||||
|
|||||||
@@ -415,3 +415,48 @@ func TestNoVerbSetsTheOperatorsKeyOrRevealsASecret(t *testing.T) {
|
|||||||
t.Fatalf("behind %v: %v", behind, err)
|
t.Fatalf("behind %v: %v", behind, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A value with a quote in it can be said on a `command` line, as in a shell, and a line whose quote
|
||||||
|
// is left open is refused naming where it opened and how a quote is written, quoting none of it (novox/hq issue 294: an
|
||||||
|
// apostrophe inside a single-quoted JSON value cut the line, and the refusal named no cause).
|
||||||
|
func TestAQuotedValueCanHoldAQuote(t *testing.T) {
|
||||||
|
for _, c := range []struct{ line, want string }{
|
||||||
|
{`settings set claude-code '{"role":"the operator'\''s laptop"}'`, `{"role":"the operator's laptop"}`},
|
||||||
|
{`settings set claude-code "{\"role\":\"the operator's laptop\"}"`, `{"role":"the operator's laptop"}`},
|
||||||
|
{"x \"a \\\\ b\nc\"", "a \\ b\nc"},
|
||||||
|
{`x 'a\b'`, `a\b`},
|
||||||
|
} {
|
||||||
|
argv, err := splitCommandLine(c.line)
|
||||||
|
if err != nil || argv[len(argv)-1] != c.want {
|
||||||
|
t.Errorf("%s: read back %q %v, want %q", c.line, argv, err, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := splitCommandLine(`settings set claude-code '{"role":"the operator's laptop"}' --node g14`)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("an apostrophe that leaves a quote open was accepted")
|
||||||
|
}
|
||||||
|
for _, want := range []string{"character 57", `'\''`, `\"`} {
|
||||||
|
if !strings.Contains(err.Error(), want) {
|
||||||
|
t.Errorf("the refusal does not say %q: %v", want, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(err.Error(), "laptop") || strings.Contains(err.Error(), "g14") {
|
||||||
|
t.Errorf("the refusal quotes the line, which may carry a setting's value: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every value reads back as it was when written the way the refusal says: single-quoted with '\”
|
||||||
|
// for each quote, or double-quoted with \ before each " and \ — newlines and backslashes included.
|
||||||
|
func TestAQuotedValueRoundTrips(t *testing.T) {
|
||||||
|
for _, v := range []string{`plain`, `it's`, `say "hi"`, `back\slash\`, "two\nlines", `'"\'\"`, `''`, ``} {
|
||||||
|
single := "x '" + strings.ReplaceAll(v, "'", `'\''`) + "'"
|
||||||
|
double := `x "` + strings.NewReplacer(`\`, `\\`, `"`, `\"`).Replace(v) + `"`
|
||||||
|
for _, line := range []string{single, double} {
|
||||||
|
argv, err := splitCommandLine(line)
|
||||||
|
if err != nil || len(argv) != 2 || argv[1] != v {
|
||||||
|
t.Errorf("%s: read back %q %v, want %q", line, argv, err, v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -39,6 +39,8 @@ func secretCommand(ctx context.Context, args []string) error {
|
|||||||
}
|
}
|
||||||
switch args[0] {
|
switch args[0] {
|
||||||
case "accept":
|
case "accept":
|
||||||
|
case "ask":
|
||||||
|
return secretAsk(ctx, args[1:])
|
||||||
case "rotate":
|
case "rotate":
|
||||||
return secretRotate(ctx, args[1:])
|
return secretRotate(ctx, args[1:])
|
||||||
case "recover":
|
case "recover":
|
||||||
@@ -55,6 +57,9 @@ func secretCommand(ctx context.Context, args []string) error {
|
|||||||
provider := set.String("provider", "",
|
provider := set.String("provider", "",
|
||||||
"the node providing <name>: the value becomes the PAIR credential between <module> on <node> "+
|
"the node providing <name>: the value becomes the PAIR credential between <module> on <node> "+
|
||||||
"and that provider, sealed to both — the vault's operator-delivered secret (ADR 0092)")
|
"and that provider, sealed to both — the vault's operator-delivered secret (ADR 0092)")
|
||||||
|
desk := set.String("at-desk", "",
|
||||||
|
"ask the operator for the value in a prompt that does not show it, on this machine's desk; the "+
|
||||||
|
"answer comes back sealed to this call alone (novox/hq ADR 0259 §10)")
|
||||||
local := set.String("local", "",
|
local := set.String("local", "",
|
||||||
"with --provider: the name the credential goes by inside <module>, where its manifest keeps "+
|
"with --provider: the name the credential goes by inside <module>, where its manifest keeps "+
|
||||||
"several for <name> (ADR 0094)")
|
"several for <name> (ADR 0094)")
|
||||||
@@ -65,6 +70,18 @@ func secretCommand(ctx context.Context, args []string) error {
|
|||||||
return errors.New(secretUsage)
|
return errors.New(secretUsage)
|
||||||
}
|
}
|
||||||
node, module, name := rest[0], rest[1], rest[2]
|
node, module, name := rest[0], rest[1], rest[2]
|
||||||
|
// A value comes from the terminal or the desk, never through a verb (the review of 2026-10-09, M4): a
|
||||||
|
// verb's caller may be an agent, and a value it chose would become what a module acts with.
|
||||||
|
if verb, through := throughAVerb(); through && *desk == "" {
|
||||||
|
return fmt.Errorf("a secret's value is given at the controller's terminal or at the desk (`give`), never "+
|
||||||
|
"through a verb (this line came through %q): nothing was read or sealed", verb)
|
||||||
|
}
|
||||||
|
if *desk != "" {
|
||||||
|
if *from != "" || *provider != "" {
|
||||||
|
return errors.New("--at-desk gives a module's own secret, and takes neither --from nor --provider")
|
||||||
|
}
|
||||||
|
return askAtDesk(ctx, node, module, name, *desk)
|
||||||
|
}
|
||||||
|
|
||||||
value, err := valueFor(node, module, name, *from)
|
value, err := valueFor(node, module, name, *from)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -93,10 +110,26 @@ func secretCommand(ctx context.Context, args []string) error {
|
|||||||
fmt.Printf(" run `push %s` and `push %s` to send it\n", *provider, node)
|
fmt.Printf(" run `push %s` and `push %s` to send it\n", *provider, node)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
untilStart, err := open.inventory.AcceptGivenSecret(ctx, node, module, name, value)
|
// A trusted party's secret is announced before it is kept (the confirmation review of 2026-10-09, N1-give):
|
||||||
|
// on every channel, the one it replaces among them, which still runs on its old value until the next push.
|
||||||
|
// Not announced, it is not kept: a channel whose token changed unheard of answers for somebody else.
|
||||||
|
trusted, err := open.inventory.RunsAsItsOwnAccount(ctx, module)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
untilStart, unannounced, err := keepGiven(trusted,
|
||||||
|
func() error { return announceSecretGiven(ctx, node, module, name, "at the controller's terminal") },
|
||||||
|
func() (bool, error) { return open.inventory.AcceptGivenSecret(ctx, node, module, name, value) })
|
||||||
|
if err != nil {
|
||||||
|
if trusted && unannounced != nil {
|
||||||
|
return fmt.Errorf("%s runs as an account of its own, and the change of its %s could not be announced on "+
|
||||||
|
"your channels first, so nothing was kept: %w", module, name, err)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if unannounced != nil {
|
||||||
|
fmt.Printf(" this change could NOT be announced on the operator's channels: %v\n", unannounced)
|
||||||
|
}
|
||||||
// Not printed back, and there is nowhere it could be printed from: it is sealed to that
|
// Not printed back, and there is nowhere it could be printed from: it is sealed to that
|
||||||
// machine and the mesh cannot read it again.
|
// machine and the mesh cannot read it again.
|
||||||
fmt.Printf("%s on %s now holds %q, sealed to that machine.\n", module, node, name)
|
fmt.Printf("%s on %s now holds %q, sealed to that machine.\n", module, node, name)
|
||||||
@@ -117,8 +150,25 @@ func secretCommand(ctx context.Context, args []string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// secretAsk is `secret ask <node> <module> <name> [--at <machine>]` (novox/hq ADR 0277): the operator is asked
|
||||||
|
// for a module's own secret in a prompt at the desk, which only they answer. The one `secret` line a verb may
|
||||||
|
// run beside rotate (givenAtTheDesk): it carries no value and answers none.
|
||||||
|
func secretAsk(ctx context.Context, args []string) error {
|
||||||
|
rest, flags := split(args)
|
||||||
|
set := flag.NewFlagSet("secret ask", flag.ContinueOnError)
|
||||||
|
at := set.String("at", "", "the machine the operator sits at, where the prompt opens; the module's machine when absent")
|
||||||
|
if err := set.Parse(flags); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(rest) != 3 {
|
||||||
|
return errors.New("secret ask <node> <module> <name> [--at <machine>]")
|
||||||
|
}
|
||||||
|
return askAtDesk(ctx, rest[0], rest[1], rest[2], *at)
|
||||||
|
}
|
||||||
|
|
||||||
const secretUsage = "secret rotate <node> <module> <name> [--why <text> [--cause <word>]]\n" +
|
const secretUsage = "secret rotate <node> <module> <name> [--why <text> [--cause <word>]]\n" +
|
||||||
"secret accept <node> <module> <name> [--from <file>] [--provider <node> [--local <name>]]\n" +
|
"secret ask <node> <module> <name> [--at <machine>]\n" +
|
||||||
|
"secret accept <node> <module> <name> [--from <file> | --at-desk <machine>] [--provider <node> [--local <name>]]\n" +
|
||||||
"secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>] [--provider <node>]\n" +
|
"secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>] [--provider <node>]\n" +
|
||||||
"secret export [--out <file>]"
|
"secret export [--out <file>]"
|
||||||
|
|
||||||
@@ -369,6 +419,8 @@ func valueFor(node, module, name, from string) (string, error) {
|
|||||||
fmt.Fprintf(os.Stderr,
|
fmt.Fprintf(os.Stderr,
|
||||||
"reading %s's %q for %s from standard input; it is not echoed anywhere\n",
|
"reading %s's %q for %s from standard input; it is not echoed anywhere\n",
|
||||||
module, name, node)
|
module, name, node)
|
||||||
|
// At a terminal, what is typed is not shown either: echo off while it is read.
|
||||||
|
defer hideTyping(os.Stdin)()
|
||||||
line, err := bufio.NewReader(os.Stdin).ReadString('\n')
|
line, err := bufio.NewReader(os.Stdin).ReadString('\n')
|
||||||
if err != nil && line == "" {
|
if err != nil && line == "" {
|
||||||
return "", fmt.Errorf("nothing was given on standard input: %w", err)
|
return "", fmt.Errorf("nothing was given on standard input: %w", err)
|
||||||
@@ -452,3 +504,23 @@ func whoAsked() string {
|
|||||||
}
|
}
|
||||||
return "the mesh"
|
return "the mesh"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// keepGiven keeps a value given at the controller's terminal, and announces it on the operator's channels
|
||||||
|
// (the confirmation review of 2026-10-09, N1-give). **A trusted party's — a module running as an account of its
|
||||||
|
// own: the router, a verified channel — is announced before it is kept, and not kept when the announcement
|
||||||
|
// fails**: a channel whose token changed unheard of answers for somebody else. Any other module's is kept first
|
||||||
|
// and announced after, and a failed announcement is said (unannounced) without undoing it.
|
||||||
|
func keepGiven(trusted bool, announce func() error, keep func() (bool, error)) (untilStart bool, unannounced, err error) {
|
||||||
|
if trusted {
|
||||||
|
if err := announce(); err != nil {
|
||||||
|
return false, err, err
|
||||||
|
}
|
||||||
|
untilStart, err = keep()
|
||||||
|
return untilStart, nil, err
|
||||||
|
}
|
||||||
|
untilStart, err = keep()
|
||||||
|
if err != nil {
|
||||||
|
return false, nil, err
|
||||||
|
}
|
||||||
|
return untilStart, announce(), nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -27,6 +27,19 @@ type sendable struct {
|
|||||||
// said it reads one is sent none, because an older node-engine refuses a key it does not know, whole
|
// said it reads one is sent none, because an older node-engine refuses a key it does not know, whole
|
||||||
// (link/order.go, the contract).
|
// (link/order.go, the contract).
|
||||||
Epoch uint64
|
Epoch uint64
|
||||||
|
// Generation is the assignment generation it was composed from (novox/hq issue 234): a machine that
|
||||||
|
// applied a later one refuses it, so a send composed from a view of the assignments the mesh has moved
|
||||||
|
// past cannot undeclare what is still assigned. Zero is not sent at all — every machine whose
|
||||||
|
// node-engine has not said it reads one is sent none, for the reason the epoch is not (an older
|
||||||
|
// node-engine refuses a key it does not know, whole).
|
||||||
|
Generation int64
|
||||||
|
// composedFrom is the generation read before this declaration was composed, and actingEpoch the
|
||||||
|
// lease epoch its sender acted under — whether or not the machine is sent either — for the record of
|
||||||
|
// the send; never on the wire.
|
||||||
|
composedFrom int64
|
||||||
|
actingEpoch uint64
|
||||||
|
// modules are the modules this declaration names, for the record of the send; never on the wire.
|
||||||
|
modules []string
|
||||||
// Adoption is nil for a converged node, and then the body is byte for byte what it was before
|
// Adoption is nil for a converged node, and then the body is byte for byte what it was before
|
||||||
// adoption existed: an older host parses the envelope strictly and would refuse the key.
|
// adoption existed: an older host parses the envelope strictly and would refuse the key.
|
||||||
Adoption *adoptionEnvelope
|
Adoption *adoptionEnvelope
|
||||||
@@ -94,6 +107,9 @@ func (s sendable) Body() ([]byte, error) {
|
|||||||
if len(s.LeftOut) > 0 {
|
if len(s.LeftOut) > 0 {
|
||||||
envelope["left_out"] = s.LeftOut
|
envelope["left_out"] = s.LeftOut
|
||||||
}
|
}
|
||||||
|
if s.Generation > 0 {
|
||||||
|
envelope["generation"] = s.Generation
|
||||||
|
}
|
||||||
// An empty declaration is deliberate here — the node owns nothing the mesh put there
|
// An empty declaration is deliberate here — the node owns nothing the mesh put there
|
||||||
// (novox/hq issue 127) — and the host refuses an empty body unless it is told the emptiness
|
// (novox/hq issue 127) — and the host refuses an empty body unless it is told the emptiness
|
||||||
// is meant, so a truncated or mis-composed body is never mistaken for "own nothing".
|
// is meant, so a truncated or mis-composed body is never mistaken for "own nothing".
|
||||||
|
|||||||
@@ -84,7 +84,7 @@ const (
|
|||||||
|
|
||||||
// callBounds are the verbs that may run longer than callDefault, and how long (S7).
|
// callBounds are the verbs that may run longer than callDefault, and how long (S7).
|
||||||
var callBounds = map[string]time.Duration{
|
var callBounds = map[string]time.Duration{
|
||||||
"push": 30 * time.Minute, "rotate": 30 * time.Minute, "assign": 15 * time.Minute,
|
"push": 30 * time.Minute, "rotate": 30 * time.Minute, "give": 5 * time.Minute, "assign": 15 * time.Minute,
|
||||||
"unassign": 15 * time.Minute, "command": 30 * time.Minute, "doctor": 3 * time.Minute,
|
"unassign": 15 * time.Minute, "command": 30 * time.Minute, "doctor": 3 * time.Minute,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -210,6 +210,36 @@ var signalsTable = []signalRow{
|
|||||||
newest: func(f *signalFacts) time.Time {
|
newest: func(f *signalFacts) time.Time {
|
||||||
return newestOf(f.waits, func(w waitFacts) time.Time { return w.since })
|
return newestOf(f.waits, func(w waitFacts) time.Time { return w.since })
|
||||||
}},
|
}},
|
||||||
|
{Row: "S18", Signal: "a batch of merges is cut into its walk", Emitter: "controller's merge window",
|
||||||
|
Trigger: "each batch (novox/hq ADR 0276)",
|
||||||
|
Bound: "a minute past merge-window-at-most, while no walk is open: the controller failed to cut it",
|
||||||
|
Kind: kindBatchNotCut, Severity: conditions.Warning, Phase: 3,
|
||||||
|
needs: func(f *signalFacts) error { return f.plansErr }, watch: watchBatchesNotCut,
|
||||||
|
newest: func(f *signalFacts) time.Time {
|
||||||
|
return newestOf(f.batches, func(b batchFacts) time.Time { return b.atMost })
|
||||||
|
}},
|
||||||
|
{Row: "S19", Signal: "a batch waiting behind an open walk is cut when it ends", Emitter: "controller's merge window",
|
||||||
|
Trigger: "each batch closed while a walk is open (novox/hq ADR 0276)",
|
||||||
|
Bound: "the walk's bound, a tier's bound for each of its tiers, from when the batch closed: naming the walk",
|
||||||
|
Kind: kindBatchBehindWalk, Severity: conditions.Warning, Phase: 3,
|
||||||
|
needs: func(f *signalFacts) error { return f.plansErr }, watch: watchBatchesBehind,
|
||||||
|
newest: func(f *signalFacts) time.Time {
|
||||||
|
return newestOf(f.batches, func(b batchFacts) time.Time { return b.closed })
|
||||||
|
}},
|
||||||
|
{Row: "S20", Signal: "a send refused for the assignment generation it was composed from",
|
||||||
|
Emitter: "node-engine", Trigger: "each refusal (novox/hq issue 234)",
|
||||||
|
Bound: "none: raised at the first refusal, naming its sender, the generation it came from and the one the " +
|
||||||
|
"machine applied; cleared an hour after the last",
|
||||||
|
Kind: kindOlderGeneration, Severity: conditions.Warning, Phase: 2,
|
||||||
|
needs: func(f *signalFacts) error { return f.refusedSendsErr }, watch: watchGenerationRefusals,
|
||||||
|
newest: func(f *signalFacts) time.Time {
|
||||||
|
return newestOf(f.refusedSends, func(s inventory.Send) time.Time {
|
||||||
|
if s.RefusedAt == nil {
|
||||||
|
return time.Time{}
|
||||||
|
}
|
||||||
|
return *s.RefusedAt
|
||||||
|
})
|
||||||
|
}},
|
||||||
{Row: "S17", Signal: "a send held for the bus's planned step is told to a person", Emitter: "controller's plan",
|
{Row: "S17", Signal: "a send held for the bus's planned step is told to a person", Emitter: "controller's plan",
|
||||||
Trigger: "each send refused because it would replace the bus outside its step (novox/hq issue 336)",
|
Trigger: "each send refused because it would replace the bus outside its step (novox/hq issue 336)",
|
||||||
Bound: "none: raised at the first refusal, for the operator, naming what waits, the bus build from and to, " +
|
Bound: "none: raised at the first refusal, for the operator, naming what waits, the bus build from and to, " +
|
||||||
@@ -372,8 +402,8 @@ func watchWaits(f *signalFacts) []conditions.Observation {
|
|||||||
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: w.id, Kind: kindWalkWaiting,
|
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: w.id, Kind: kindWalkWaiting,
|
||||||
Severity: severity,
|
Severity: severity,
|
||||||
Summary: fmt.Sprintf("the walk of %s %s has waited %s for %s's word to start: `mesh-delivery.show` for the "+
|
Summary: fmt.Sprintf("the walk of %s %s has waited %s for %s's word to start: `mesh-delivery.show` for the "+
|
||||||
"delivery that landed as %s says why; `plans go %s --why …` starts it by hand", w.repository,
|
"deliveries that landed as %s says why; `plans go %s --why …` starts it by hand", w.repository,
|
||||||
short(w.commit), ago(in), w.awaits, short(w.commit), w.id),
|
short(w.commit), ago(in), w.awaits, mergesWords(w), w.id),
|
||||||
Said: fmt.Sprintf("waiting since %s for %s", w.since.UTC().Format(time.RFC3339), w.awaits),
|
Said: fmt.Sprintf("waiting since %s for %s", w.since.UTC().Format(time.RFC3339), w.awaits),
|
||||||
Headline: deliveryName(w.modules, w.repository) + " waiting to start",
|
Headline: deliveryName(w.modules, w.repository) + " waiting to start",
|
||||||
Explanation: walkWaitingWords(w, in, severity),
|
Explanation: walkWaitingWords(w, in, severity),
|
||||||
@@ -384,6 +414,19 @@ func watchWaits(f *signalFacts) []conditions.Observation {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// mergesWords is the merges a waiting walk answers (novox/hq ADR 0276): every delivery it carries, not one
|
||||||
|
// commit that a supersession may already have ended (issue 362). Its own commit for a walk naming none.
|
||||||
|
func mergesWords(w waitFacts) string {
|
||||||
|
if len(w.merges) == 0 {
|
||||||
|
return short(w.commit)
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
for _, m := range w.merges {
|
||||||
|
out = append(out, m.Repository+"@"+short(m.Commit))
|
||||||
|
}
|
||||||
|
return readableList(out)
|
||||||
|
}
|
||||||
|
|
||||||
func watchLoop(f *signalFacts) []conditions.Observation {
|
func watchLoop(f *signalFacts) []conditions.Observation {
|
||||||
if f.loop.pending == 0 {
|
if f.loop.pending == 0 {
|
||||||
return nil
|
return nil
|
||||||
@@ -494,6 +537,14 @@ func watchAdvisories(f *signalFacts) []conditions.Observation {
|
|||||||
if a.Kind == link.AdvisoryConsumerLost && !f.lostConsumers[a.Stream+"."+a.Consumer] {
|
if a.Kind == link.AdvisoryConsumerLost && !f.lostConsumers[a.Stream+"."+a.Consumer] {
|
||||||
continue // it exists again, or the mesh no longer expects it: a removal, not a loss
|
continue // it exists again, or the mesh no longer expects it: a removal, not a loss
|
||||||
}
|
}
|
||||||
|
if a.Kind == link.AdvisoryMaxDeliveries && a.Token == "" {
|
||||||
|
// A consumer's max-deliveries key is the dead-letter row's (novox/hq issue 330): said while
|
||||||
|
// DEAD_LETTERS holds what it gave up on, cleared when that is delivered again or dropped. The
|
||||||
|
// listener records no such advisory today; one read here as well added a look to the count and
|
||||||
|
// overwrote the row's words on every look (novox/hq issue 440). Only one that could not be kept
|
||||||
|
// (AdvisoryNotKept), which DEAD_LETTERS cannot say, is said from here.
|
||||||
|
continue
|
||||||
|
}
|
||||||
severity := conditions.Warning
|
severity := conditions.Warning
|
||||||
times := ""
|
times := ""
|
||||||
if a.Count > 1 {
|
if a.Count > 1 {
|
||||||
@@ -504,7 +555,10 @@ func watchAdvisories(f *signalFacts) []conditions.Observation {
|
|||||||
machine = f.host
|
machine = f.host
|
||||||
}
|
}
|
||||||
o := conditions.Observation{Scope: conditions.ScopeBus, ID: a.ID, Kind: a.Kind, Token: a.Token,
|
o := conditions.Observation{Scope: conditions.ScopeBus, ID: a.ID, Kind: a.Kind, Token: a.Token,
|
||||||
Machine: machine, Severity: severity, Summary: a.Said + times, Said: a.Said}
|
Machine: machine, Severity: severity, Summary: a.Said + times, Said: a.Said,
|
||||||
|
// The advisory is remembered and read again on every look for an hour: the condition counts
|
||||||
|
// what the bus said and when, not the looks (novox/hq issue 402).
|
||||||
|
Happened: a.Last, Times: a.Count}
|
||||||
if a.Kind == link.AdvisoryMaxDeliveries && a.Token == link.AdvisoryNotKept {
|
if a.Kind == link.AdvisoryMaxDeliveries && a.Token == link.AdvisoryNotKept {
|
||||||
o.Machine = consumerMachine(a.Stream, a.Consumer)
|
o.Machine = consumerMachine(a.Stream, a.Consumer)
|
||||||
o.Headline = clip(conditions.Capital(fmt.Sprintf("%s gave up on a message, not kept",
|
o.Headline = clip(conditions.Capital(fmt.Sprintf("%s gave up on a message, not kept",
|
||||||
@@ -536,7 +590,17 @@ func watchDeadLetters(f *signalFacts) []conditions.Observation {
|
|||||||
messages, them = fmt.Sprintf("%d messages", n), "them"
|
messages, them = fmt.Sprintf("%d messages", n), "them"
|
||||||
}
|
}
|
||||||
who := consumerWho(stream, consumer)
|
who := consumerWho(stream, consumer)
|
||||||
|
// DEAD_LETTERS is a record of what was given up on: the condition says when the newest held message
|
||||||
|
// was kept, and its count is a running total of the messages given up on while it is open, never
|
||||||
|
// how often the controller looked (novox/hq issues 402 and 440). It is at least the number held now,
|
||||||
|
// and does not go down when one is delivered again or dropped. Without that time it counts its
|
||||||
|
// looks, as a source of the present does.
|
||||||
|
happened, times := f.deadLettersNewest[key], 0
|
||||||
|
if !happened.IsZero() {
|
||||||
|
times = n
|
||||||
|
}
|
||||||
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: key, Kind: link.AdvisoryMaxDeliveries,
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: key, Kind: link.AdvisoryMaxDeliveries,
|
||||||
|
Happened: happened, Times: times,
|
||||||
Machine: consumerMachine(stream, consumer), Severity: conditions.Warning,
|
Machine: consumerMachine(stream, consumer), Severity: conditions.Warning,
|
||||||
Summary: fmt.Sprintf("%s gave up on %s; %s kept in %s until delivered again or dropped, with why, "+
|
Summary: fmt.Sprintf("%s gave up on %s; %s kept in %s until delivered again or dropped, with why, "+
|
||||||
"through the controller's dead-letters verb", link.ConsumerInWords(stream, consumer), messages,
|
"through the controller's dead-letters verb", link.ConsumerInWords(stream, consumer), messages,
|
||||||
|
|||||||
@@ -100,14 +100,15 @@ var suppressions = map[string]suppression{
|
|||||||
inside: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-29 * time.Minute))} },
|
inside: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-29 * time.Minute))} },
|
||||||
past: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-31 * time.Minute))} },
|
past: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-31 * time.Minute))} },
|
||||||
},
|
},
|
||||||
|
// A message given up on and not kept: the one max-deliveries advisory S9 says itself (issue 440).
|
||||||
"S9": {
|
"S9": {
|
||||||
inside: func(f *signalFacts) {
|
inside: func(f *signalFacts) {
|
||||||
f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop", Said: "gave up",
|
f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop",
|
||||||
First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-61 * time.Minute), Count: 1}}
|
Token: link.AdvisoryNotKept, Said: "gave up, not kept", First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-61 * time.Minute), Count: 1}}
|
||||||
},
|
},
|
||||||
past: func(f *signalFacts) {
|
past: func(f *signalFacts) {
|
||||||
f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop", Said: "gave up",
|
f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop",
|
||||||
First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-59 * time.Minute), Count: 1}}
|
Token: link.AdvisoryNotKept, Said: "gave up, not kept", First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-59 * time.Minute), Count: 1}}
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"S10": {
|
"S10": {
|
||||||
@@ -142,6 +143,28 @@ var suppressions = map[string]suppression{
|
|||||||
since: f.now.Add(-31 * time.Minute)}}
|
since: f.now.Add(-31 * time.Minute)}}
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
// A batch still assembling past its maximum with no walk open (novox/hq ADR 0276): a minute's grace.
|
||||||
|
"S18": {
|
||||||
|
inside: func(f *signalFacts) {
|
||||||
|
f.batches = []batchFacts{{id: "plan-3", state: inventory.PlanAssembling, grouped: "novox/app@c0ffee11",
|
||||||
|
atMost: f.now.Add(-59 * time.Second), closed: f.now.Add(-59 * time.Second)}}
|
||||||
|
},
|
||||||
|
past: func(f *signalFacts) {
|
||||||
|
f.batches = []batchFacts{{id: "plan-3", state: inventory.PlanAssembling, grouped: "novox/app@c0ffee11",
|
||||||
|
atMost: f.now.Add(-61 * time.Second), closed: f.now.Add(-61 * time.Second)}}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
// A batch queued behind an open walk past that walk's bound, named.
|
||||||
|
"S19": {
|
||||||
|
inside: func(f *signalFacts) {
|
||||||
|
f.batches = []batchFacts{{id: "plan-3", state: inventory.PlanQueued, grouped: "novox/app@c0ffee11",
|
||||||
|
closed: f.now.Add(-59 * time.Minute), behind: "plan-1", walkBound: time.Hour}}
|
||||||
|
},
|
||||||
|
past: func(f *signalFacts) {
|
||||||
|
f.batches = []batchFacts{{id: "plan-3", state: inventory.PlanQueued, grouped: "novox/app@c0ffee11",
|
||||||
|
closed: f.now.Add(-61 * time.Minute), behind: "plan-1", walkBound: time.Hour}}
|
||||||
|
},
|
||||||
|
},
|
||||||
// A send refused because it would replace the bus outside its planned step: said at its first refusal,
|
// A send refused because it would replace the bus outside its planned step: said at its first refusal,
|
||||||
// whatever the bound (novox/hq issue 336). Inside: a new bus build waits, and no send was refused for it.
|
// whatever the bound (novox/hq issue 336). Inside: a new bus build waits, and no send was refused for it.
|
||||||
"S17": {
|
"S17": {
|
||||||
@@ -155,6 +178,14 @@ var suppressions = map[string]suppression{
|
|||||||
commit: "c0ffee001122", modules: []string{"app"}, since: f.now.Add(-time.Second)}}}
|
commit: "c0ffee001122", modules: []string{"app"}, since: f.now.Add(-time.Second)}}}
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
// A send refused by its machine for the generation it was composed from (novox/hq issue 234): said at
|
||||||
|
// once, naming its sender, for an hour after. Inside: the last such refusal was more than an hour ago.
|
||||||
|
"S20": {
|
||||||
|
inside: func(f *signalFacts) {
|
||||||
|
f.refusedSends = []inventory.Send{refusedSend(f.now.Add(-61 * time.Minute))}
|
||||||
|
},
|
||||||
|
past: func(f *signalFacts) { f.refusedSends = []inventory.Send{refusedSend(f.now.Add(-time.Second))} },
|
||||||
|
},
|
||||||
// Twice by hand within a fortnight is a healer wanted; once, or the first of two a day too old, is not.
|
// Twice by hand within a fortnight is a healer wanted; once, or the first of two a day too old, is not.
|
||||||
"S15": {
|
"S15": {
|
||||||
inside: func(f *signalFacts) {
|
inside: func(f *signalFacts) {
|
||||||
|
|||||||
@@ -196,6 +196,11 @@ func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, e
|
|||||||
if report.Ordered() {
|
if report.Ordered() {
|
||||||
link.StaleRefusals.Lifetime(report.Node, report.RefusedOlder, now)
|
link.StaleRefusals.Lifetime(report.Node, report.RefusedOlder, now)
|
||||||
}
|
}
|
||||||
|
// A send refused for the generation it was composed from is kept on the send it refused, so S20 names
|
||||||
|
// its sender (novox/hq issue 234).
|
||||||
|
if report.OlderGeneration != nil {
|
||||||
|
heardGenerationRefusal(ctx, l.Enrolment.Inventory, report)
|
||||||
|
}
|
||||||
// What the machine's witnesses put back and stand by (novox/hq ADR 0236): read by the gate and its
|
// What the machine's witnesses put back and stand by (novox/hq ADR 0236): read by the gate and its
|
||||||
// probe. Only from an account of the machine — not a word that a declaration was set aside, nor a rekey.
|
// probe. Only from an account of the machine — not a word that a declaration was set aside, nor a rekey.
|
||||||
if report.Superseded == "" && report.Rekey == nil && report.Node != "" {
|
if report.Superseded == "" && report.Rekey == nil && report.Node != "" {
|
||||||
|
|||||||
@@ -1,99 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"reflect"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// novox/hq issue 254, ADR 0218: a newer plan takes over what the older open plans of its repository
|
|
||||||
// and branch had not built, and closes them as superseded; another repository's plan, another
|
|
||||||
// branch's, and a plan made after it are left alone.
|
|
||||||
func TestANewerPlanSupersedesTheOlderOpenPlansOfItsRepository(t *testing.T) {
|
|
||||||
at := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
|
||||||
sent := at.Add(time.Minute)
|
|
||||||
plan := func(id, repository, branch string, created time.Time, modules map[string]*inventory.PlanModule) inventory.Plan {
|
|
||||||
return inventory.Plan{ID: id, Repository: repository, Branch: branch, Commit: id + "-commit",
|
|
||||||
Created: created, State: inventory.PlanRolling, Modules: modules}
|
|
||||||
}
|
|
||||||
older := plan("plan-1", "novox/mesh-catalog", "main", at, map[string]*inventory.PlanModule{
|
|
||||||
"gitea": {State: "built", SentAt: &sent}, // done with: stays done
|
|
||||||
"keycloak": {State: "asked"}, // asked, not answered: folded
|
|
||||||
"plex": {}, // not yet asked: folded
|
|
||||||
"agent": {State: "built"}, // built, rolls out, not sent: folded
|
|
||||||
"notes": {State: "built"}, // built, records: nothing to send
|
|
||||||
})
|
|
||||||
stuck := plan("plan-0", "Novox/Mesh-Catalog", "", at.Add(-time.Hour), map[string]*inventory.PlanModule{
|
|
||||||
"runtime": {State: "asked"},
|
|
||||||
})
|
|
||||||
other := plan("plan-2", "novox/mesh-controller", "main", at, map[string]*inventory.PlanModule{"mesh-controller": {}})
|
|
||||||
release := plan("plan-3", "novox/mesh-catalog", "release", at, map[string]*inventory.PlanModule{"lemurs": {}})
|
|
||||||
later := plan("plan-5", "novox/mesh-catalog", "main", at.Add(2*time.Hour), map[string]*inventory.PlanModule{"later": {}})
|
|
||||||
done := plan("plan-6", "novox/mesh-catalog", "main", at, map[string]*inventory.PlanModule{"finished": {}})
|
|
||||||
done.State = inventory.PlanDone
|
|
||||||
|
|
||||||
newer := plan("plan-4", "novox/mesh-catalog", "main", at.Add(time.Hour), nil)
|
|
||||||
newer.Commit = "97b1b2b0c0ffee"
|
|
||||||
rollsOut := func(m string) bool { return m != "notes" }
|
|
||||||
folded, closed := supersededBy(newer, []inventory.Plan{stuck, older, other, release, later, done, newer}, rollsOut)
|
|
||||||
|
|
||||||
if want := []string{"agent", "keycloak", "plex", "runtime"}; !reflect.DeepEqual(folded, want) {
|
|
||||||
t.Fatalf("folded %v, wanted %v", folded, want)
|
|
||||||
}
|
|
||||||
var ids []string
|
|
||||||
for _, p := range closed {
|
|
||||||
ids = append(ids, p.ID)
|
|
||||||
if p.State != inventory.PlanSuperseded || p.Open() {
|
|
||||||
t.Errorf("%s was left %s", p.ID, p.State)
|
|
||||||
}
|
|
||||||
if !strings.Contains(p.Note, "plan-4") || !strings.Contains(p.Note, "97b1b2b0") {
|
|
||||||
t.Errorf("%s does not name the plan that superseded it: %q", p.ID, p.Note)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if want := []string{"plan-0", "plan-1"}; !reflect.DeepEqual(ids, want) {
|
|
||||||
t.Fatalf("superseded %v, wanted %v — another repository, another branch, a later plan and a "+
|
|
||||||
"finished one are left alone", ids, want)
|
|
||||||
}
|
|
||||||
if other.State != inventory.PlanRolling {
|
|
||||||
t.Fatal("the plan handed in was changed in place")
|
|
||||||
}
|
|
||||||
if line := planLine(closed[1], time.Now()); !strings.Contains(line, "superseded") {
|
|
||||||
t.Fatalf("a superseded plan reads %q", line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// novox/hq issue 254: a person closes a plan that will not move again, by its id.
|
|
||||||
func TestAPersonClosesAStuckPlan(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
stuck := inventory.Plan{ID: "plan-97b1b2b", Repository: "novox/mesh-catalog", Commit: "97b1b2b",
|
|
||||||
Created: time.Now().UTC(), State: inventory.PlanRolling, Tier: 1, Tiers: [][]string{{"a"}, {"b"}},
|
|
||||||
Modules: map[string]*inventory.PlanModule{"a": {State: "built"}, "b": {}}}
|
|
||||||
if err := open.inventory.SavePlan(ctx, &stuck); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := plansCommand(ctx, []string{"close", stuck.ID}); err == nil || !strings.Contains(err.Error(), "--why") {
|
|
||||||
t.Fatalf("a plan was closed by hand without saying why: %v", err)
|
|
||||||
}
|
|
||||||
if err := plansCommand(ctx, []string{"close", stuck.ID, "--why", "its report will not come"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
closed, err := open.inventory.PlanByID(ctx, stuck.ID)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if closed.State != inventory.PlanFailed || !strings.Contains(closed.Note, "closed by hand") ||
|
|
||||||
!strings.Contains(closed.Note, "its report will not come") {
|
|
||||||
t.Fatalf("the plan was left %s: %q", closed.State, closed.Note)
|
|
||||||
}
|
|
||||||
if err := plansCommand(ctx, []string{"close", stuck.ID, "--why", "again"}); err == nil {
|
|
||||||
t.Fatal("a plan already closed was closed again")
|
|
||||||
}
|
|
||||||
if argv, err := argvFor("plans", map[string]any{"close": stuck.ID, "why": "w"}); err != nil ||
|
|
||||||
!reflect.DeepEqual(argv, []string{"plans", "close", stuck.ID, "--why", "w"}) {
|
|
||||||
t.Fatalf("the seat's verb does not close a plan: %v %v", argv, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+301
-177
@@ -12,6 +12,7 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/builder"
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
@@ -297,42 +298,20 @@ func notNow(err error) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// SourceMoved is the forge announcing a merge: every module recorded as built from that
|
// SourceMoved is the forge announcing a merge. It is put into the open batch (novox/hq ADR 0276), which
|
||||||
// repository and branch is marked as moved to the merge commit, and built — bases first, so a
|
// becomes one walk when its merge window closes: hearMerge, and cutBatchesHeld in batches.go.
|
||||||
// module that stands on another's artifact is built after it and not against the old one
|
|
||||||
// (novox/hq 04-ISSUES/131). Nothing is pushed here: what a finished build does to the machines
|
|
||||||
// running the module is the upgrade's decision, taken when the catalogue announces it.
|
|
||||||
func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
||||||
// One merge acted on at a time, whoever hands it over: the bus, or the catch-up that reads back
|
return f.hearMerge(ctx, m, time.Now().UTC())
|
||||||
// what the bus did not hand over (novox/hq issue 266). Each judges against what the other wrote.
|
|
||||||
actingOnMerges.Lock()
|
|
||||||
defer actingOnMerges.Unlock()
|
|
||||||
|
|
||||||
inv := f.open.inventory
|
|
||||||
entries, err := inv.Catalogued(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return notNow(err)
|
|
||||||
}
|
|
||||||
read, err := inv.ReadRepositories(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return notNow(err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// **A merge older than the newest planned merge of its branch is planned at that merge's commit**
|
// movesOfMerge is what one merge moves, acted on: every module recorded as built from that repository and
|
||||||
// (novox/hq issue 349): the catch-up acts on a merge the bus did not hand over after the merges that
|
// branch is marked as moved to the merge commit; a module the merge deleted is forgotten or said; a new module
|
||||||
// came after it, and planned at its own commit it built what a later merge had fixed — the forge's
|
// is asked for and sent nowhere. The names returned are what the walk builds, bases first along the graph
|
||||||
// security fix, on 2026-10-09 — from the commit before it, dependents and all. The later commit contains
|
// (novox/hq 04-ISSUES/131). Nothing is built or pushed here: the walk asks its tiers. Called when a batch is
|
||||||
// this merge's change. Planned there, with that merge's time, a module the later merge already looked at
|
// cut, once per repository, with the latest merge of the repository's branch and every file the batch's
|
||||||
// reads as history and is not built again; the rest are built from the newest commit.
|
// merges of it changed.
|
||||||
newest, known, err := inv.NewestMergeOf(ctx, m.Owner+"/"+m.Repo, m.Base)
|
func movesOfMerge(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, entries []inventory.Entry,
|
||||||
if err != nil {
|
read map[string][]inventory.ReadRepository) ([]string, error) {
|
||||||
return notNow(err)
|
|
||||||
}
|
|
||||||
if known && laterOnTheBranch(m, newest) {
|
|
||||||
fmt.Printf(" %s/%s %.8s was merged before %.8s (%s, %s): what it moved is built from %.8s, which "+
|
|
||||||
"contains it\n", m.Owner, m.Repo, m.Commit, newest.Commit, newest.ID, newest.State, newest.Commit)
|
|
||||||
m.Commit, m.MergedAt = newest.Commit, newest.Merged.Format(time.RFC3339Nano)
|
|
||||||
}
|
|
||||||
from, packaging, already := mergeCandidates(m, entries, read)
|
from, packaging, already := mergeCandidates(m, entries, read)
|
||||||
if len(from) == 0 && len(packaging) == 0 {
|
if len(from) == 0 && len(packaging) == 0 {
|
||||||
// "Already built from it" and "nothing reads it" are different facts, and reading the first
|
// "Already built from it" and "nothing reads it" are different facts, and reading the first
|
||||||
@@ -340,17 +319,11 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
|||||||
if already > 0 {
|
if already > 0 {
|
||||||
fmt.Printf("%s/%s merged into %s (%.8s); %d module(s) the mesh holds are already built "+
|
fmt.Printf("%s/%s merged into %s (%.8s); %d module(s) the mesh holds are already built "+
|
||||||
"from it\n", m.Owner, m.Repo, m.Base, m.Commit, already)
|
"from it\n", m.Owner, m.Repo, m.Base, m.Commit, already)
|
||||||
return nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
fmt.Printf("%s/%s merged into %s (%.8s); nothing the mesh holds reads it\n",
|
fmt.Printf("%s/%s merged into %s (%.8s); nothing the mesh holds reads it\n",
|
||||||
m.Owner, m.Repo, m.Base, m.Commit)
|
m.Owner, m.Repo, m.Base, m.Commit)
|
||||||
return nil
|
return nil, nil
|
||||||
}
|
|
||||||
// The same judgement for the packaging kind, against the newest look at that repository by
|
|
||||||
// anything built from it: they keep no record of it themselves, and a replayed old merge should
|
|
||||||
// not rebuild them either.
|
|
||||||
if isHistory(m.MergedAt, lastLookAt(entries, m)) {
|
|
||||||
packaging = nil
|
|
||||||
}
|
}
|
||||||
// Said, never silent (novox/hq 04-ISSUES/215): a module built from this repository that follows
|
// Said, never silent (novox/hq 04-ISSUES/215): a module built from this repository that follows
|
||||||
// another branch is not part of this merge, and whoever is waiting for its change should read why.
|
// another branch is not part of this merge, and whoever is waiting for its change should read why.
|
||||||
@@ -360,7 +333,7 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
|||||||
e.Manifest.Module, m.Owner, m.Repo, e.Source.Ref, m.Base)
|
e.Manifest.Module, m.Owner, m.Repo, e.Source.Ref, m.Base)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
touched, added, _ := touchedBy(from, entries, m)
|
touched, added, _ := touchedBy(from, entries, m, read)
|
||||||
// **A module the merge deleted is not built** (novox/hq ADR 0236): its manifest is gone, so the build
|
// **A module the merge deleted is not built** (novox/hq ADR 0236): its manifest is gone, so the build
|
||||||
// seat finds nothing saying what it is, and the plan failed on it (`has no module.json at …`) with
|
// seat finds nothing saying what it is, and the plan failed on it (`has no module.json at …`) with
|
||||||
// every other module of its tier left unsent. It is forgotten where nothing holds it, said otherwise.
|
// every other module of its tier left unsent. It is forgotten where nothing holds it, said otherwise.
|
||||||
@@ -370,7 +343,7 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
|||||||
}
|
}
|
||||||
for _, e := range touched {
|
for _, e := range touched {
|
||||||
if err := inv.SourceMoved(ctx, e.Manifest.Module, m.Commit); err != nil {
|
if err := inv.SourceMoved(ctx, e.Manifest.Module, m.Commit); err != nil {
|
||||||
return notNow(err)
|
return nil, notNow(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// **A new module is built and registered, and sent nowhere** (novox/hq issue 300): the delivery plan
|
// **A new module is built and registered, and sent nowhere** (novox/hq issue 300): the delivery plan
|
||||||
@@ -382,117 +355,25 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
|||||||
fmt.Printf("%s/%s merged into %s (%.8s); it changed no module the mesh holds, and adds %s: "+
|
fmt.Printf("%s/%s merged into %s (%.8s); it changed no module the mesh holds, and adds %s: "+
|
||||||
"built, registered when the build lands, and sent nowhere\n", m.Owner, m.Repo, m.Base, m.Commit,
|
"built, registered when the build lands, and sent nowhere\n", m.Owner, m.Repo, m.Base, m.Commit,
|
||||||
strings.Join(built, ", "))
|
strings.Join(built, ", "))
|
||||||
return nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
fmt.Printf("%s/%s merged into %s (%.8s); it changed nothing any module the mesh holds is "+
|
fmt.Printf("%s/%s merged into %s (%.8s); it changed nothing any module the mesh holds is "+
|
||||||
"built from\n", m.Owner, m.Repo, m.Base, m.Commit)
|
"built from\n", m.Owner, m.Repo, m.Base, m.Commit)
|
||||||
return nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
// A merge produces a plan the mesh keeps (novox/hq ADR 0162): what moved and everything that
|
// Why each is in it (issue 363): the files of its build source the merge changed, or why it is read whole.
|
||||||
// depends on it, along the catalogue's one dependency relation, sorted into tiers. The plan is
|
|
||||||
// written before any build is asked; the first tier is asked; this returns. Outcomes advance it.
|
|
||||||
edges, err := inv.Dependencies(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return notNow(err)
|
|
||||||
}
|
|
||||||
var movedNames []string
|
|
||||||
for _, e := range moved {
|
for _, e := range moved {
|
||||||
movedNames = append(movedNames, e.Manifest.Module)
|
fmt.Printf(" %s: %s\n", e.Manifest.Module, whyMoved(e, read[e.Manifest.Module], m))
|
||||||
}
|
}
|
||||||
// Written and its first tier asked as one act on the plans (novox/hq issue 213): a timer on
|
|
||||||
// another controller reading it between the two would ask the tier again.
|
|
||||||
release, err := inv.HoldPlans(ctx, true)
|
|
||||||
if err != nil {
|
|
||||||
return notNow(err)
|
|
||||||
}
|
|
||||||
defer release()
|
|
||||||
plan := planOfMerge(m, movedNames, edges)
|
|
||||||
// **A newer plan supersedes the older open plans of this repository and branch** (novox/hq issue
|
|
||||||
// 254, ADR 0218): what they had not built is planned here again, and they are closed, so one plan
|
|
||||||
// works a repository's modules at a time and a stuck one ends at the next merge.
|
|
||||||
working, err := inv.OpenPlans(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return notNow(err)
|
|
||||||
}
|
|
||||||
rollsOut := func(module string) bool {
|
|
||||||
u, err := inv.UpgradeOf(ctx, module)
|
|
||||||
return err == nil && u.RollOut
|
|
||||||
}
|
|
||||||
folded, superseded := supersededBy(plan, working, rollsOut)
|
|
||||||
if len(folded) > 0 {
|
|
||||||
held := map[string]bool{}
|
|
||||||
for _, e := range entries {
|
|
||||||
held[e.Manifest.Module] = true
|
|
||||||
}
|
|
||||||
names := map[string]bool{}
|
|
||||||
for _, name := range movedNames {
|
|
||||||
names[name] = true
|
|
||||||
}
|
|
||||||
var also []string
|
|
||||||
for _, name := range folded {
|
|
||||||
// One the catalogue no longer holds would fail the newer plan's ask; it is not this
|
|
||||||
// merge's to build.
|
|
||||||
if held[name] && !names[name] {
|
|
||||||
names[name] = true
|
|
||||||
movedNames = append(movedNames, name)
|
|
||||||
also = append(also, name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(also) > 0 {
|
|
||||||
again := planOfMerge(m, movedNames, edges)
|
|
||||||
again.ID, again.Created = plan.ID, plan.Created
|
|
||||||
plan = again
|
|
||||||
fmt.Printf(" %s, left unbuilt by an older plan of %s, are planned here again\n",
|
|
||||||
strings.Join(also, ", "), plan.Repository)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// **Whether it waits for its delivery's word** (novox/hq ADR 0239): while the mesh-delivery seat has a
|
|
||||||
// holder on record, a walk that moves no module on the controller's own path is opened and waits.
|
|
||||||
plan.Delivery = awaitsFor(entries, movedNames)
|
|
||||||
if hasCycle(plan.Tiers, edges) {
|
|
||||||
fmt.Printf(" the last tier depends on itself: %s — built together, in no order\n",
|
|
||||||
strings.Join(plan.Tiers[len(plan.Tiers)-1], ", "))
|
|
||||||
}
|
|
||||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
|
||||||
return notNow(err)
|
|
||||||
}
|
|
||||||
// Closed after the newer plan is kept, never before: a controller replaced between the two leaves
|
|
||||||
// both open, which the next merge settles, rather than neither.
|
|
||||||
for _, old := range superseded {
|
|
||||||
if err := inv.SavePlan(ctx, &old); err != nil {
|
|
||||||
return notNow(err)
|
|
||||||
}
|
|
||||||
fmt.Printf(" %s (%s at %s) is %s\n", old.ID, old.Repository, short(old.Commit), old.Note)
|
|
||||||
}
|
|
||||||
var tiers []string
|
|
||||||
for i, t := range plan.Tiers {
|
|
||||||
tiers = append(tiers, fmt.Sprintf("%d: %s", i, strings.Join(t, ", ")))
|
|
||||||
}
|
|
||||||
fmt.Printf("%s/%s merged into %s (%.8s); plan %s, %d module(s) in %d tier(s)\n %s\n",
|
|
||||||
m.Owner, m.Repo, m.Base, m.Commit, plan.ID, len(plan.Modules), len(plan.Tiers), strings.Join(tiers, "\n "))
|
|
||||||
if len(packaging) > 0 {
|
|
||||||
var also []string
|
|
||||||
for _, e := range packaging {
|
for _, e := range packaging {
|
||||||
also = append(also, e.Manifest.Module)
|
fmt.Printf(" %s reads %s/%s through its build context: its own source record is left where it is\n",
|
||||||
|
e.Manifest.Module, m.Owner, m.Repo)
|
||||||
}
|
}
|
||||||
fmt.Printf(" %s package source from it, so they are rebuilt and their own source record "+
|
var names []string
|
||||||
"is left where it is\n", strings.Join(also, ", "))
|
for _, e := range moved {
|
||||||
|
names = append(names, e.Manifest.Module)
|
||||||
}
|
}
|
||||||
if plan.Waiting() {
|
return names, nil
|
||||||
plan.Note = waitingNote(plan)
|
|
||||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
|
||||||
return notNow(err)
|
|
||||||
}
|
|
||||||
fmt.Printf(" %s waits for %s's word before its first tier is asked\n", plan.ID, plan.Delivery.Awaits)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if err := askTier(ctx, inv, &plan); err != nil {
|
|
||||||
return notNow(err)
|
|
||||||
}
|
|
||||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
|
||||||
return notNow(err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// askNewModules asks the build seat for every module a merge adds to the repository — a directory holding a
|
// askNewModules asks the build seat for every module a merge adds to the repository — a directory holding a
|
||||||
@@ -570,25 +451,119 @@ func mergeCandidates(m link.SourceMoved, entries []inventory.Entry,
|
|||||||
}
|
}
|
||||||
from = append(from, e)
|
from = append(from, e)
|
||||||
case readsFrom(read[e.Manifest.Module], m):
|
case readsFrom(read[e.Manifest.Module], m):
|
||||||
|
// **A merge older than the module's last look is history for it** (novox/hq ADR 0267): a
|
||||||
|
// build or plan of it after the merge already read the repository with the merge in it. Per
|
||||||
|
// module, since a merge that moved only the module built from the repository says nothing
|
||||||
|
// about the ones packaging it.
|
||||||
|
// Judged with a margin for the forge's clock running behind the store's: too late a look
|
||||||
|
// rebuilds once more, too early one would miss the merge.
|
||||||
|
if lookedAtCommit(read[e.Manifest.Module], m.Commit) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if looked := lookedOf(read[e.Manifest.Module]); !looked.IsZero() &&
|
||||||
|
isHistory(m.MergedAt, looked.Add(-historyMargin)) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
packaging = append(packaging, e)
|
packaging = append(packaging, e)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return from, packaging, already
|
return from, packaging, already
|
||||||
}
|
}
|
||||||
|
|
||||||
// wouldMove is the modules built from the merged repository that acting on this merge would mark as
|
// lookedAtCommit is whether a plan that built a module, or is building it, answered this merge commit.
|
||||||
// moved and rebuild — SourceMoved's judgement, made without acting (novox/hq issue 266). Empty for a
|
func lookedAtCommit(read []inventory.ReadRepository, commit string) bool {
|
||||||
// merge already acted on: acting marks each of them as looked at, so the merge then reads as history.
|
for _, r := range read {
|
||||||
|
if slices.Contains(r.LookedAt, commit) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// historyMargin is how far a packaging module's last look is taken back before a merge is history for it.
|
||||||
|
const historyMargin = time.Minute
|
||||||
|
|
||||||
|
// whyMoved is why a merge moves a module, as a plan says it (novox/hq ADR 0267, issue 363): the changed
|
||||||
|
// files in its build source, or why it is read whole. For a module built from the merged repository or one
|
||||||
|
// whose build context is that repository; a dependent is in a plan for what it stands on.
|
||||||
|
func whyMoved(e inventory.Entry, read []inventory.ReadRepository, m link.SourceMoved) string {
|
||||||
|
if len(m.Paths) == 0 || m.PathsTruncated {
|
||||||
|
return "read whole: the merge's changed files were not all said"
|
||||||
|
}
|
||||||
|
whole := "no build source recorded"
|
||||||
|
for _, r := range read {
|
||||||
|
if r.Own && r.Whole != "" {
|
||||||
|
whole = r.Whole
|
||||||
|
}
|
||||||
|
}
|
||||||
|
held := func(paths []string) []string {
|
||||||
|
var in []string
|
||||||
|
for _, p := range m.Paths {
|
||||||
|
if builder.SourceHolds(paths, p) {
|
||||||
|
in = append(in, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return in
|
||||||
|
}
|
||||||
|
changed := func(where string, in []string) string {
|
||||||
|
return fmt.Sprintf("its build source%s changed: %d changed file(s) in it, e.g. %s", where, len(in), in[0])
|
||||||
|
}
|
||||||
|
if sameRepository(e.Source.Repository, m) {
|
||||||
|
if own := ownSource(read); own != nil {
|
||||||
|
if in := held(own); len(in) > 0 {
|
||||||
|
return changed("", in)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dir := strings.Trim(e.Source.Path, "/")
|
||||||
|
if dir == "" {
|
||||||
|
return "read whole: " + whole + ", so every file of its repository is its build source"
|
||||||
|
}
|
||||||
|
for _, p := range m.Paths {
|
||||||
|
if inside(p, dir) {
|
||||||
|
return "read whole: " + whole + ", so its directory is its build source; e.g. " + p
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "read whole: " + whole
|
||||||
|
}
|
||||||
|
for _, r := range read {
|
||||||
|
if r.Own || !sameRepository(r.Repository, m) || (r.Ref != "" && r.Ref != m.Base) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(r.Paths) > 0 {
|
||||||
|
if in := held(r.Paths); len(in) > 0 {
|
||||||
|
return changed(" in "+m.Owner+"/"+m.Repo, in)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return "read whole: " + whole + ", so every file of " + m.Owner + "/" + m.Repo + ", which its build context is, is its build source"
|
||||||
|
}
|
||||||
|
return "read whole: " + whole
|
||||||
|
}
|
||||||
|
|
||||||
|
// lookedOf is when a module packaging another repository was last looked at, as readForPlanning says.
|
||||||
|
func lookedOf(read []inventory.ReadRepository) time.Time {
|
||||||
|
var at time.Time
|
||||||
|
for _, r := range read {
|
||||||
|
if r.Looked.After(at) {
|
||||||
|
at = r.Looked
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return at
|
||||||
|
}
|
||||||
|
|
||||||
|
// wouldMove is the modules acting on this merge would move and rebuild — SourceMoved's judgement, made
|
||||||
|
// without acting (novox/hq issue 266). Empty for a merge already acted on: acting marks each module built
|
||||||
|
// from the repository as looked at, so the merge then reads as history for it.
|
||||||
//
|
//
|
||||||
// **Only the modules built from it, never the ones that merely package source from it.** Acting
|
// **The ones packaging source from it too** (novox/hq ADR 0267): with a module moved only by the files of
|
||||||
// records nothing about those, so a merge acted on would go on reading as unacted for them, and be
|
// its build source, a merge can move a packaging module and nothing built from the repository, and a missed
|
||||||
// acted on again on every look. A merge that moves both is caught by the first kind, and acting on it
|
// one of those was never acted on. A packaging module's look is its newest build or plan (lookedAt), so a
|
||||||
// rebuilds the second as well.
|
// merge acted on for it reads as history once its plan is made.
|
||||||
func wouldMove(m link.SourceMoved, entries []inventory.Entry,
|
func wouldMove(m link.SourceMoved, entries []inventory.Entry,
|
||||||
read map[string][]inventory.ReadRepository) []inventory.Entry {
|
read map[string][]inventory.ReadRepository) []inventory.Entry {
|
||||||
from, _, _ := mergeCandidates(m, entries, read)
|
from, packaging, _ := mergeCandidates(m, entries, read)
|
||||||
touched, _ := splitDeleted(whatTheMergeTouched(from, entries, m), m)
|
touched, _ := splitDeleted(whatTheMergeTouched(from, entries, m, read), m)
|
||||||
return touched
|
return append(touched, packaging...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// splitDeleted parts the modules a merge touched into those it changed and those whose manifest it
|
// splitDeleted parts the modules a merge touched into those it changed and those whose manifest it
|
||||||
@@ -672,34 +647,171 @@ func sameRepository(repository string, m link.SourceMoved) bool {
|
|||||||
(m.CloneURL != "" && repo == strings.ToLower(strings.TrimSuffix(m.CloneURL, ".git")))
|
(m.CloneURL != "" && repo == strings.ToLower(strings.TrimSuffix(m.CloneURL, ".git")))
|
||||||
}
|
}
|
||||||
|
|
||||||
// readsFrom is whether a module's build read the repository a merge names: the second repository its
|
// readsFrom is whether a merge changed what a module's build read in another repository: the second
|
||||||
// recipe packages source from. Its ref must be the branch that moved, or unset — the same rule a
|
// repository its recipe packages source from. Its ref must be the branch that moved, or unset — the same
|
||||||
// module's own source follows.
|
// rule a module's own source follows.
|
||||||
|
//
|
||||||
|
// **Only a changed file in what the build read there** (novox/hq ADR 0267 rule 2): where the module's
|
||||||
|
// newest trunk build said its build source in that repository, a merge touching none of it is no change
|
||||||
|
// to the module (issue 338: every merge to the controller's repository moved the route proxy and the
|
||||||
|
// build seat's holder). Where it said none, or the merge's files are not all said, the whole repository is
|
||||||
|
// read, as before.
|
||||||
func readsFrom(read []inventory.ReadRepository, m link.SourceMoved) bool {
|
func readsFrom(read []inventory.ReadRepository, m link.SourceMoved) bool {
|
||||||
for _, r := range read {
|
for _, r := range read {
|
||||||
if sameRepository(r.Repository, m) && (r.Ref == "" || r.Ref == m.Base) {
|
if r.Own || !sameRepository(r.Repository, m) || (r.Ref != "" && r.Ref != m.Base) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(r.Paths) == 0 || len(m.Paths) == 0 || m.PathsTruncated {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
for _, p := range m.Paths {
|
||||||
|
if builder.SourceHolds(r.Paths, p) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// lastLookAt is the most recent look at this repository by anything built from it.
|
// ownSource is the build source a module's newest trunk build said it read in its own repository; nil
|
||||||
func lastLookAt(entries []inventory.Entry, m link.SourceMoved) time.Time {
|
// when it said none, and the module's own directory — or, built from the root, its whole repository — is
|
||||||
var newest time.Time
|
// its build source, as before (novox/hq ADR 0267).
|
||||||
for _, e := range entries {
|
func ownSource(read []inventory.ReadRepository) []string {
|
||||||
if sameRepository(e.Source.Repository, m) && e.Source.Seen.After(newest) {
|
for _, r := range read {
|
||||||
newest = e.Source.Seen
|
if r.Own && len(r.Paths) > 0 {
|
||||||
|
return r.Paths
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return newest
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// readsFile is whether a module built from the merged repository reads one of its changed files: in its
|
||||||
|
// build source where its newest trunk build said one, else anywhere in its directory, or anywhere at all
|
||||||
|
// for a module built from the repository's root.
|
||||||
|
func readsFile(e inventory.Entry, read []inventory.ReadRepository, p string) bool {
|
||||||
|
if own := ownSource(read); own != nil {
|
||||||
|
return builder.SourceHolds(own, p)
|
||||||
|
}
|
||||||
|
return strings.Trim(e.Source.Path, "/") == "" || inside(p, e.Source.Path)
|
||||||
|
}
|
||||||
|
|
||||||
|
// staleIn is the modules whose recorded build source a plan has overtaken (novox/hq ADR 0267): a plan still
|
||||||
|
// working that has yet to build one, or a plan made after that build which never built it — failed, stopped
|
||||||
|
// or superseded. What such a module is built from is changing, or changed without a build to say so: a merge
|
||||||
|
// that added an import to it, and a later one changing only what that import names, would otherwise move
|
||||||
|
// nothing. Each is read whole, as before, until a build of it works again.
|
||||||
|
//
|
||||||
|
// Each is answered with the plan that overtook it, for saying why it is read whole.
|
||||||
|
func staleIn(read map[string][]inventory.ReadRepository, plans []inventory.Plan) map[string]string {
|
||||||
|
stale := map[string]string{}
|
||||||
|
for name, rs := range read {
|
||||||
|
var since time.Time
|
||||||
|
for _, r := range rs {
|
||||||
|
if r.Built.After(since) {
|
||||||
|
since = r.Built
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, p := range plans {
|
||||||
|
s, in := p.Modules[name]
|
||||||
|
if !in || (s != nil && (s.State == "built" || s.State == planDeleted)) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if p.Open() || p.Created.After(since) {
|
||||||
|
stale[name] = p.ID
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return stale
|
||||||
|
}
|
||||||
|
|
||||||
|
// lookedAt is when a merge was last acted on for a module that packages another repository's source: its
|
||||||
|
// newest build, or the newest plan that built it or is still building it, whichever is later. A build asked
|
||||||
|
// after a merge clones that repository with the merge in it, so an older merge is history for it; a plan
|
||||||
|
// that closed without building it looked at nothing.
|
||||||
|
func lookedAt(name string, read []inventory.ReadRepository, plans []inventory.Plan) time.Time {
|
||||||
|
var at time.Time
|
||||||
|
for _, r := range read {
|
||||||
|
if r.Looked.After(at) {
|
||||||
|
at = r.Looked
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, p := range plans {
|
||||||
|
s, in := p.Modules[name]
|
||||||
|
if in && (p.Open() || (s != nil && s.State == "built")) && p.Created.After(at) {
|
||||||
|
at = p.Created
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return at
|
||||||
|
}
|
||||||
|
|
||||||
|
// readForPlanning is what each module's build read, as the planner maps a change onto it — for a merge
|
||||||
|
// acting now, the merge gate, a pull request's check, a delivery's order and the what-if alike, so planning
|
||||||
|
// and gating cannot disagree (novox/hq ADR 0238): the build sources the newest trunk builds said, but for
|
||||||
|
// the modules a plan has overtaken (staleIn), and with when each was last looked at (lookedAt).
|
||||||
|
func readForPlanning(ctx context.Context, inv *inventory.Inventory) (map[string][]inventory.ReadRepository, error) {
|
||||||
|
read, err := inv.ReadRepositories(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// Every plan since the oldest build whose source is recorded: one made after a module's build can have
|
||||||
|
// overtaken it, however long ago, so no window of recent plans would do.
|
||||||
|
var oldest time.Time
|
||||||
|
for _, rs := range read {
|
||||||
|
for _, r := range rs {
|
||||||
|
if !r.Built.IsZero() && (oldest.IsZero() || r.Built.Before(oldest)) {
|
||||||
|
oldest = r.Built
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
plans, err := inv.PlansSince(ctx, oldest)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return planningView(read, plans), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// planningView is readForPlanning over what was read, so a test can hand it records.
|
||||||
|
func planningView(read map[string][]inventory.ReadRepository, plans []inventory.Plan) map[string][]inventory.ReadRepository {
|
||||||
|
stale := staleIn(read, plans)
|
||||||
|
out := make(map[string][]inventory.ReadRepository, len(read))
|
||||||
|
for name, rs := range read {
|
||||||
|
looked := lookedAt(name, rs, plans)
|
||||||
|
var commits []string
|
||||||
|
for _, p := range plans {
|
||||||
|
if st, in := p.Modules[name]; in && p.Commit != "" && (p.Open() || (st != nil && st.State == "built")) {
|
||||||
|
// Every commit the walk carries (novox/hq ADR 0276): a batch's walk answers one per repository.
|
||||||
|
for _, c := range p.Carried() {
|
||||||
|
commits = append(commits, c.Commit)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var kept []inventory.ReadRepository
|
||||||
|
overtaken, isStale := stale[name]
|
||||||
|
for _, r := range rs {
|
||||||
|
if isStale {
|
||||||
|
if r.Own {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
r.Paths = nil
|
||||||
|
}
|
||||||
|
r.Looked, r.LookedAt = looked, commits
|
||||||
|
kept = append(kept, r)
|
||||||
|
}
|
||||||
|
if isStale {
|
||||||
|
kept = append(kept, inventory.ReadRepository{Own: true, Whole: "plan " + overtaken + " has not built it yet",
|
||||||
|
Looked: looked, LookedAt: commits})
|
||||||
|
}
|
||||||
|
out[name] = kept
|
||||||
|
}
|
||||||
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// whatTheMergeTouched narrows the modules built from a repository to the ones the merge changed: **a
|
// whatTheMergeTouched narrows the modules built from a repository to the ones the merge changed: **a
|
||||||
// changed file touches exactly the modules whose build reads it** (novox/hq issue 280, ADR 0238). It is
|
// changed file touches exactly the modules whose build reads it** (novox/hq issue 280, ADR 0238). It is
|
||||||
// touchedBy's first answer; touchedBy is the one place the mesh maps a changed file onto its modules.
|
// touchedBy's first answer; touchedBy is the one place the mesh maps a changed file onto its modules.
|
||||||
func whatTheMergeTouched(candidates, known []inventory.Entry, m link.SourceMoved) []inventory.Entry {
|
func whatTheMergeTouched(candidates, known []inventory.Entry, m link.SourceMoved,
|
||||||
touched, _, _ := touchedBy(candidates, known, m)
|
read map[string][]inventory.ReadRepository) []inventory.Entry {
|
||||||
|
touched, _, _ := touchedBy(candidates, known, m, read)
|
||||||
return touched
|
return touched
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -707,8 +819,11 @@ func whatTheMergeTouched(candidates, known []inventory.Entry, m link.SourceMoved
|
|||||||
// merge handler, the release planner's what-if, the merge gate and a pull request's check alike (novox/hq
|
// merge handler, the release planner's what-if, the merge gate and a pull request's check alike (novox/hq
|
||||||
// ADR 0238), so planning and gating cannot disagree about what a change touches.
|
// ADR 0238), so planning and gating cannot disagree about what a change touches.
|
||||||
//
|
//
|
||||||
// **A changed file touches exactly the modules whose build reads it.** What a build reads is the module's
|
// **A changed file touches exactly the modules whose build reads it.** What a build reads is its build
|
||||||
// own directory — the builder clones the repository and builds within that directory alone: the manifest,
|
// source, where the module's newest trunk build said one (novox/hq ADR 0267): a Go program's import closure,
|
||||||
|
// an archive's directory, a recipe, its manifest — so a README at the root of a repository whose module is
|
||||||
|
// built from its root, or another program's package beside it, touches nothing. Where none was said, it is
|
||||||
|
// the module's own directory — the builder clones the repository and builds within that directory alone: the manifest,
|
||||||
// the recipes, the bundles' sources, the Docker context — or the whole repository for a module built from
|
// the recipes, the bundles' sources, the Docker context — or the whole repository for a module built from
|
||||||
// its root. A second repository a recipe packages (an artifact's `context`) is read too; that is the build
|
// its root. A second repository a recipe packages (an artifact's `context`) is read too; that is the build
|
||||||
// record's `read`, answered by readsFrom in mergeCandidates. So a changed file inside a module's directory
|
// record's `read`, answered by readsFrom in mergeCandidates. So a changed file inside a module's directory
|
||||||
@@ -728,7 +843,8 @@ func whatTheMergeTouched(candidates, known []inventory.Entry, m link.SourceMoved
|
|||||||
//
|
//
|
||||||
// Nothing said about the files, or not all of them said, is still everything: what is not known cannot
|
// Nothing said about the files, or not all of them said, is still everything: what is not known cannot
|
||||||
// be narrowed.
|
// be narrowed.
|
||||||
func touchedBy(candidates, known []inventory.Entry, m link.SourceMoved) (touched []inventory.Entry, added, unread []string) {
|
func touchedBy(candidates, known []inventory.Entry, m link.SourceMoved,
|
||||||
|
read map[string][]inventory.ReadRepository) (touched []inventory.Entry, added, unread []string) {
|
||||||
knownDirs := map[string]bool{}
|
knownDirs := map[string]bool{}
|
||||||
for _, e := range known {
|
for _, e := range known {
|
||||||
if !e.Provided && sameRepository(e.Source.Repository, m) {
|
if !e.Provided && sameRepository(e.Source.Repository, m) {
|
||||||
@@ -763,19 +879,27 @@ func touchedBy(candidates, known []inventory.Entry, m link.SourceMoved) (touched
|
|||||||
return candidates, added, nil
|
return candidates, added, nil
|
||||||
}
|
}
|
||||||
for _, e := range candidates {
|
for _, e := range candidates {
|
||||||
if strings.Trim(e.Source.Path, "/") == "" || anyInside(m.Paths, e.Source.Path) {
|
for _, p := range m.Paths {
|
||||||
|
if readsFile(e, read[e.Manifest.Module], p) {
|
||||||
touched = append(touched, e)
|
touched = append(touched, e)
|
||||||
|
break
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, p := range m.Paths {
|
for _, p := range m.Paths {
|
||||||
read := newDir["."]
|
isRead := newDir["."]
|
||||||
for _, e := range candidates {
|
for _, e := range candidates {
|
||||||
read = read || strings.Trim(e.Source.Path, "/") == "" || inside(p, e.Source.Path)
|
isRead = isRead || readsFile(e, read[e.Manifest.Module], p)
|
||||||
}
|
}
|
||||||
for d := range newDir {
|
for d := range newDir {
|
||||||
read = read || inside(p, d)
|
isRead = isRead || inside(p, d)
|
||||||
}
|
}
|
||||||
if !read {
|
// A file a module packages from this repository is read too, by that module's build.
|
||||||
|
for _, e := range known {
|
||||||
|
isRead = isRead || readsFrom(read[e.Manifest.Module], link.SourceMoved{Owner: m.Owner, Repo: m.Repo,
|
||||||
|
Base: m.Base, CloneURL: m.CloneURL, Paths: []string{p}})
|
||||||
|
}
|
||||||
|
if !isRead {
|
||||||
unread = append(unread, p)
|
unread = append(unread, p)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -834,7 +958,7 @@ func (r mergeReach) Dependents() []string {
|
|||||||
func reachOfMerge(m link.SourceMoved, entries []inventory.Entry, read map[string][]inventory.ReadRepository,
|
func reachOfMerge(m link.SourceMoved, entries []inventory.Entry, read map[string][]inventory.ReadRepository,
|
||||||
edges []inventory.Edge) mergeReach {
|
edges []inventory.Edge) mergeReach {
|
||||||
from, packaging, already := mergeCandidates(m, entries, read)
|
from, packaging, already := mergeCandidates(m, entries, read)
|
||||||
touched, added, unread := touchedBy(from, entries, m)
|
touched, added, unread := touchedBy(from, entries, m, read)
|
||||||
kept, deleted := splitDeleted(touched, m)
|
kept, deleted := splitDeleted(touched, m)
|
||||||
r := mergeReach{Touched: kept, Deleted: deleted, Packaging: packaging, Already: already, Added: added, Unread: unread}
|
r := mergeReach{Touched: kept, Deleted: deleted, Packaging: packaging, Already: already, Added: added, Unread: unread}
|
||||||
var building []string
|
var building []string
|
||||||
|
|||||||
@@ -0,0 +1,384 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A provider waiting for the operator holds its consumers, and a wait beside another wait is said (novox/hq
|
||||||
|
// issue 405, found in the review of ADR 0283's controller change).
|
||||||
|
//
|
||||||
|
// The shapes are those of issue 386 (mounts waiting for smb-password-games: waitingResource, passwordWait) and of
|
||||||
|
// the openrazer wait on the workstation of 2026-10-11 (relogin, userUnit): an account in its group whose session
|
||||||
|
// began before it was, and its unit failed in that account's own service manager.
|
||||||
|
|
||||||
|
// waitingDatabase is a provider whose only part not healthy waits for the operator's secret: a database's
|
||||||
|
// process stated waiting, as the node-engine states a tool check answering waits (ADR 0283 decision 2). Its wait
|
||||||
|
// names the part that waits by the provision it gives.
|
||||||
|
func waitingDatabase() inventory.ResourceHealth {
|
||||||
|
return waitingDatabaseFor(inventory.Wait{Part: "postgres-database", Secret: "licence",
|
||||||
|
What: "the licence key of the database"})
|
||||||
|
}
|
||||||
|
|
||||||
|
func waitingDatabaseFor(waits ...inventory.Wait) inventory.ResourceHealth {
|
||||||
|
return inventory.ResourceHealth{Module: "db", Resource: "db.server", Kind: "process", Target: "db.service",
|
||||||
|
State: link.StateWaiting, Check: "tool", Reason: "the database waits for its licence", Waits: waits}
|
||||||
|
}
|
||||||
|
|
||||||
|
// backupsWait is a wait of the same provider for another part than the one its consumers need.
|
||||||
|
var backupsWait = inventory.Wait{Part: "the nightly backups", Secret: "backup-key", What: "the key of the backups"}
|
||||||
|
|
||||||
|
// failingConsumer is a consumer whose check that needs the database fails.
|
||||||
|
func failingConsumer(module string) inventory.ResourceHealth {
|
||||||
|
return inventory.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module,
|
||||||
|
State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"}
|
||||||
|
}
|
||||||
|
|
||||||
|
// dbSecrets is what the database's waits name: own secrets issued outside the mesh, so a wait for one checks out
|
||||||
|
// while nobody gave it (ADR 0283 decision 3, novox/hq issue 450).
|
||||||
|
var dbSecrets = catalogue.OwnSecrets{
|
||||||
|
"licence": {Path: "/s/licence", IssuedBy: catalogue.IssuedOutside},
|
||||||
|
"backup-key": {Path: "/s/backup-key", IssuedBy: catalogue.IssuedOutside},
|
||||||
|
}
|
||||||
|
|
||||||
|
// holdingOf is one reading of the record without a store: the newest statements, the open conditions, the
|
||||||
|
// catalogue, what was given on the provider's machine (nothing), and each consumer's provider already looked up,
|
||||||
|
// as providerFor memoises it.
|
||||||
|
func holdingOf(open []conditions.Condition, healths map[string]inventory.NodeHealth, bound map[[3]string]catalogue.Chosen) *holding {
|
||||||
|
h := &holding{ctx: context.Background(), healths: healths, open: open, providers: map[string]providerLookup{},
|
||||||
|
shelf: map[string]catalogue.Manifest{"db": {Module: "db", Version: "1", OwnSecrets: dbSecrets,
|
||||||
|
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}}},
|
||||||
|
waits: operatorWaitFacts{given: map[string]map[string]time.Time{"db@anchor": {}}}}
|
||||||
|
for k, p := range bound {
|
||||||
|
h.providers[k[0]+"\x00"+k[1]+"\x00"+k[2]] = providerLookup{p, true}
|
||||||
|
}
|
||||||
|
return h
|
||||||
|
}
|
||||||
|
|
||||||
|
var theDatabase = catalogue.Chosen{Node: "anchor", Module: "db"}
|
||||||
|
|
||||||
|
var shopOnTheDatabase = map[[3]string]catalogue.Chosen{{"laptop", "shop", "postgres-database"}: theDatabase}
|
||||||
|
|
||||||
|
func shopFailingBeside(provider ...inventory.ResourceHealth) map[string]inventory.NodeHealth {
|
||||||
|
return map[string]inventory.NodeHealth{
|
||||||
|
"anchor": {Node: "anchor", Resources: provider},
|
||||||
|
"laptop": {Node: "laptop", Resources: []inventory.ResourceHealth{failingConsumer("shop")}},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// (1) A provider whose only part not healthy waits for the operator holds the findings of the consumers of the
|
||||||
|
// waiting part under it (ADR 0240 rule 5); a consumer of another part, or one whose part cannot be matched, is
|
||||||
|
// its own (ADR 0283 decision 1: a module that waits says nothing else of it is wrong).
|
||||||
|
func TestAProviderWaitingForTheOperatorHoldsOnlyTheConsumersOfTheWaitingPart(t *testing.T) {
|
||||||
|
shop := []inventory.ResourceHealth{failingConsumer("shop")}
|
||||||
|
unhealthyDB := waitingDatabase()
|
||||||
|
unhealthyDB.State, unhealthyDB.Waits, unhealthyDB.Reason = link.StateUnhealthy, nil, "its tool check: refused"
|
||||||
|
healthyDB := waitingDatabase()
|
||||||
|
healthyDB.State, healthyDB.Waits = link.StateHealthy, nil
|
||||||
|
byCredential := holdingOf(nil, shopFailingBeside(waitingDatabaseFor(inventory.Wait{Part: "the server",
|
||||||
|
Secret: "licence", What: "the licence key"})), shopOnTheDatabase)
|
||||||
|
byCredential.shelf["db"] = catalogue.Manifest{Module: "db", Version: "1", OwnSecrets: dbSecrets, Provides: []catalogue.Offer{{
|
||||||
|
Name: "postgres-database", Credential: &catalogue.OfferCredential{Own: "licence"}}}}
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
hold *holding
|
||||||
|
held bool
|
||||||
|
onlyWaits bool
|
||||||
|
uncovered bool
|
||||||
|
}{
|
||||||
|
{"the waiting part is the provision", holdingOf(nil, shopFailingBeside(waitingDatabase()), shopOnTheDatabase), true, true, false},
|
||||||
|
{"the wait is for the provision's shared credential", byCredential, true, true, false},
|
||||||
|
{"the wait is for another part", holdingOf(nil, shopFailingBeside(waitingDatabaseFor(backupsWait)), shopOnTheDatabase), false, false, true},
|
||||||
|
{"only the needs-operator condition, its parts not said", holdingOf(
|
||||||
|
[]conditions.Condition{{Key: needsOperatorKey("db", "anchor"), Kind: kindNeedsOperator}},
|
||||||
|
shopFailingBeside(), shopOnTheDatabase), false, false, true},
|
||||||
|
{"an unhealthy provider holds as before", holdingOf(nil, shopFailingBeside(unhealthyDB), shopOnTheDatabase), true, false, false},
|
||||||
|
{"a healthy provider holds nothing", holdingOf(nil, shopFailingBeside(healthyDB), shopOnTheDatabase), false, false, false},
|
||||||
|
} {
|
||||||
|
by, held := c.hold.heldWith("laptop", "shop", shop)
|
||||||
|
if held != c.held || (held && by.provider != theDatabase) || (len(by.waits) > 0) != c.onlyWaits {
|
||||||
|
t.Errorf("%s: held %v under %v with waits %v; want held %v, only waits %v", c.name, held, by.provider,
|
||||||
|
by.waits, c.held, c.onlyWaits)
|
||||||
|
}
|
||||||
|
if _, uncovered := c.hold.waitingUncovered("laptop", "shop", shop); uncovered != c.uncovered {
|
||||||
|
t.Errorf("%s: said as a provider waiting for another part %v; want %v", c.name, uncovered, c.uncovered)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// (1) The consumer's first-node gate under a provider that only waits for the operator passes as a wait for a
|
||||||
|
// person (ADR 0254), carrying the wait (ADR 0283 decision 4); under an unhealthy provider it waits, as before.
|
||||||
|
func TestAConsumersGateUnderAWaitingProviderPassesCarryingTheWait(t *testing.T) {
|
||||||
|
now := time.Now()
|
||||||
|
since := now.Add(-time.Minute)
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
provider inventory.ResourceHealth
|
||||||
|
want health
|
||||||
|
says []string
|
||||||
|
}{
|
||||||
|
{"a provider that only waits", waitingDatabase(), healthPerson,
|
||||||
|
[]string{"waits on db on anchor, which waits for you", "the licence key of the database", "nox secret ask anchor db licence"}},
|
||||||
|
{"an unhealthy provider", func() inventory.ResourceHealth {
|
||||||
|
r := waitingDatabase()
|
||||||
|
r.State, r.Waits, r.Reason = link.StateUnhealthy, nil, "its tool check: refused"
|
||||||
|
return r
|
||||||
|
}(), healthWaiting, []string{"waits on db on anchor, which is unhealthy"}},
|
||||||
|
} {
|
||||||
|
healths := shopFailingBeside(c.provider)
|
||||||
|
for m, h := range healths {
|
||||||
|
h.HeardAt = now
|
||||||
|
healths[m] = h
|
||||||
|
}
|
||||||
|
f := gateFacts{now: now, health: healths, heldOn: heldReadings(holdingOf(nil, healths, shopOnTheDatabase))}
|
||||||
|
h, why := moduleHealthWord("shop", "laptop", since, f)
|
||||||
|
if h != c.want {
|
||||||
|
t.Errorf("%s: the consumer's gate reads %v %q; want %v", c.name, h, why, c.want)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, s := range c.says {
|
||||||
|
if !strings.Contains(why, s) {
|
||||||
|
t.Errorf("%s: the gate's reading %q does not say %q", c.name, why, s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// judgeBoth judges the provider's statement and then the consumer's, two looks each, over a record that changes
|
||||||
|
// as the statements do.
|
||||||
|
func judgeBoth(t *testing.T, k *conditions.Keeper, provider []inventory.ResourceHealth,
|
||||||
|
byProvider, byConsumer map[string]inventory.NodeHealth) []conditions.Condition {
|
||||||
|
t.Helper()
|
||||||
|
ctx := t.Context()
|
||||||
|
was := readHoldingFor
|
||||||
|
t.Cleanup(func() { readHoldingFor = was })
|
||||||
|
healths := byProvider
|
||||||
|
readHoldingFor = func(_ context.Context, _ *inventory.Inventory, open []conditions.Condition) (*holding, error) {
|
||||||
|
return holdingOf(open, healths, shopOnTheDatabase), nil
|
||||||
|
}
|
||||||
|
for look := 1; look <= 2; look++ {
|
||||||
|
healths = byProvider
|
||||||
|
if err := judgeModuleHealth(ctx, nil, k, "anchor", map[string][]inventory.ResourceHealth{"db": provider},
|
||||||
|
map[string]int{"db": look}, time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for look := 1; look <= 2; look++ {
|
||||||
|
healths = byConsumer
|
||||||
|
if err := judgeModuleHealth(ctx, nil, k, "laptop", map[string][]inventory.ResourceHealth{"shop": {failingConsumer("shop")}},
|
||||||
|
map[string]int{"shop": look}, time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
open, err := k.Open(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return open
|
||||||
|
}
|
||||||
|
|
||||||
|
func conditionOf(open []conditions.Condition, key string) *conditions.Condition {
|
||||||
|
for i, c := range open {
|
||||||
|
if c.Key == key {
|
||||||
|
return &open[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// (1) The consumer raises nothing of its own; the provider's needs-operator condition lists who waits on it and
|
||||||
|
// stays a warning (ADR 0283 decision 5: never escalated). The consumer's statement arrives after the provider's,
|
||||||
|
// so it is the consumer's judging (sayWaiters) that lists it at the provider — no store involved.
|
||||||
|
func TestAWaitingProvidersConditionListsWhoWaitsOnItAndStaysAWarning(t *testing.T) {
|
||||||
|
k, _ := withConditionsInMemory(t)
|
||||||
|
open := judgeBoth(t, k, []inventory.ResourceHealth{waitingDatabase()},
|
||||||
|
map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Resources: []inventory.ResourceHealth{waitingDatabase()}}}, shopFailingBeside(waitingDatabase()))
|
||||||
|
provider := conditionOf(open, needsOperatorKey("db", "anchor"))
|
||||||
|
if len(open) != 1 || provider == nil {
|
||||||
|
t.Fatalf("a provider waiting for the operator and a consumer of its waiting part raised %v; want the "+
|
||||||
|
"provider's needs-operator alone", openKeysOf(open))
|
||||||
|
}
|
||||||
|
if said := provider.Evidence[0].Said; !strings.Contains(said, "shop on laptop") {
|
||||||
|
t.Fatalf("the provider's needs-operator does not list shop on laptop as waiting on it: %s", said)
|
||||||
|
}
|
||||||
|
if provider.Severity != conditions.Warning {
|
||||||
|
t.Fatalf("the provider's needs-operator became %s with a consumer waiting; it stays a warning", provider.Severity)
|
||||||
|
}
|
||||||
|
if provider.Resolver != conditions.ResolverOperator || !strings.Contains(provider.Needs, "desk prompt") {
|
||||||
|
t.Fatalf("the provider's condition: %+v", provider)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// (1) A consumer of another part than the one waiting is raised on its own, and says its provider waits.
|
||||||
|
func TestAConsumerOfAnotherPartIsRaisedOnItsOwn(t *testing.T) {
|
||||||
|
k, _ := withConditionsInMemory(t)
|
||||||
|
backups := waitingDatabaseFor(backupsWait)
|
||||||
|
open := judgeBoth(t, k, []inventory.ResourceHealth{backups}, shopFailingBeside(backups), shopFailingBeside(backups))
|
||||||
|
consumer := conditionOf(open, moduleUnhealthyKey("shop", "laptop"))
|
||||||
|
if consumer == nil || conditionOf(open, needsOperatorKey("db", "anchor")) == nil {
|
||||||
|
t.Fatalf("raised %v; want shop's own unhealthy beside db's needs-operator", openKeysOf(open))
|
||||||
|
}
|
||||||
|
if said := consumer.Evidence[0].Said; !strings.Contains(said, "db on anchor waits for you, but not for anything shop is known to need, so shop's fault is said on its own") {
|
||||||
|
t.Fatalf("the consumer's condition does not say its provider waits: %s", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// relogin and userUnit are person_wait_test.go's; mounts is said here with the same account and unit beside its
|
||||||
|
// wait for the password.
|
||||||
|
func reloginBesideAWait() []inventory.ResourceHealth {
|
||||||
|
return []inventory.ResourceHealth{relogin("mounts", "operator"), userUnit("mounts", "operator"),
|
||||||
|
waitingResource(passwordWait)}
|
||||||
|
}
|
||||||
|
|
||||||
|
// (2) A module with a checked wait beside a relogin-needed account says both: the new login, and the act the
|
||||||
|
// operator owes it.
|
||||||
|
func TestAWaitBesideAReloginIsSaid(t *testing.T) {
|
||||||
|
k, _ := withConditionsInMemory(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
for look := 1; look <= 2; look++ {
|
||||||
|
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": reloginBesideAWait()},
|
||||||
|
map[string]int{"mounts": look}, time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
got, open := needsOperatorOpen(t, k)
|
||||||
|
if got == nil {
|
||||||
|
t.Fatalf("a wait for the operator beside a relogin is not said: %v", openKeysOf(open))
|
||||||
|
}
|
||||||
|
if !strings.Contains(got.Summary, "smb-password-games") || got.Severity != conditions.Warning {
|
||||||
|
t.Fatalf("the needs-operator beside the relogin: %+v", got)
|
||||||
|
}
|
||||||
|
relogged := false
|
||||||
|
for _, c := range open {
|
||||||
|
relogged = relogged || c.Key == reloginKey("mounts", "workstation")
|
||||||
|
}
|
||||||
|
if !relogged {
|
||||||
|
t.Fatalf("the relogin is no longer said beside the wait: %v", openKeysOf(open))
|
||||||
|
}
|
||||||
|
// The login done, the wait stays said and the relogin clears.
|
||||||
|
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}},
|
||||||
|
map[string]int{"mounts": 3}, time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, open = needsOperatorOpen(t, k)
|
||||||
|
if got == nil || len(open) != 1 {
|
||||||
|
t.Fatalf("after the new login: %v", openKeysOf(open))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// (2) The same beside a directory used as found.
|
||||||
|
func TestAWaitBesideADirectoryUsedAsFoundIsSaid(t *testing.T) {
|
||||||
|
k, _ := withConditionsInMemory(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
found := foundDirectory("mounts", time.Now().Add(-time.Hour))
|
||||||
|
for look := 1; look <= 2; look++ {
|
||||||
|
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{
|
||||||
|
"mounts": {found, waitingResource(passwordWait)}}, map[string]int{"mounts": look}, time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
got, open := needsOperatorOpen(t, k)
|
||||||
|
if got == nil {
|
||||||
|
t.Fatalf("a wait for the operator beside a directory used as found is not said: %v", openKeysOf(open))
|
||||||
|
}
|
||||||
|
asFound := false
|
||||||
|
for _, c := range open {
|
||||||
|
asFound = asFound || c.Key == usedAsFoundKey("mounts", "workstation")
|
||||||
|
}
|
||||||
|
if !asFound {
|
||||||
|
t.Fatalf("the directory used as found is no longer said beside the wait: %v", openKeysOf(open))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// (2) Beside a fault of its own, the wait is said too, and the fault's words do not count the waiting part among
|
||||||
|
// what fails.
|
||||||
|
func TestAWaitBesideAFaultIsSaidAndTheFaultIsTheFaultAlone(t *testing.T) {
|
||||||
|
k, _ := withConditionsInMemory(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
down := inventory.ResourceHealth{Module: "mounts", Resource: "mounts.apply", Kind: "process",
|
||||||
|
Target: "mesh-mounts-apply.service", State: link.StateUnhealthy, Reason: "down"}
|
||||||
|
for look := 1; look <= 2; look++ {
|
||||||
|
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{
|
||||||
|
"mounts": {down, waitingResource(passwordWait)}}, map[string]int{"mounts": look}, time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
got, open := needsOperatorOpen(t, k)
|
||||||
|
if got == nil {
|
||||||
|
t.Fatalf("a wait for the operator beside a fault is not said: %v", openKeysOf(open))
|
||||||
|
}
|
||||||
|
var fault *conditions.Condition
|
||||||
|
for i, c := range open {
|
||||||
|
if c.Key == moduleUnhealthyKey("mounts", "workstation") {
|
||||||
|
fault = &open[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if fault == nil {
|
||||||
|
t.Fatalf("the fault is not said: %v", openKeysOf(open))
|
||||||
|
}
|
||||||
|
if strings.Contains(fault.Summary, "mounts.watch") {
|
||||||
|
t.Fatalf("the fault's summary counts the waiting part as failing: %q", fault.Summary)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func openKeysOf(cs []conditions.Condition) []string {
|
||||||
|
var out []string
|
||||||
|
for _, c := range cs {
|
||||||
|
out = append(out, c.Key)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// A consumer raised on its own, whose provider then waits for the operator for the part it needs, is cleared saying
|
||||||
|
// which the provider is: it waits for you, not that it is unhealthy (novox/hq issue 405 review).
|
||||||
|
func TestAConsumerHeldOnceItsProviderWaitsSaysWhichItIs(t *testing.T) {
|
||||||
|
k, _ := withConditionsInMemory(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
start := time.Now().Add(-time.Second)
|
||||||
|
healthy := waitingDatabase()
|
||||||
|
healthy.State, healthy.Waits = link.StateHealthy, nil
|
||||||
|
healths := shopFailingBeside(healthy)
|
||||||
|
was := readHoldingFor
|
||||||
|
t.Cleanup(func() { readHoldingFor = was })
|
||||||
|
readHoldingFor = func(_ context.Context, _ *inventory.Inventory, open []conditions.Condition) (*holding, error) {
|
||||||
|
return holdingOf(open, healths, shopOnTheDatabase), nil
|
||||||
|
}
|
||||||
|
shop := map[string][]inventory.ResourceHealth{"shop": {failingConsumer("shop")}}
|
||||||
|
for look := 1; look <= 2; look++ {
|
||||||
|
if err := judgeModuleHealth(ctx, nil, k, "laptop", shop, map[string]int{"shop": look}, time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if open, _ := k.Open(ctx); conditionOf(open, moduleUnhealthyKey("shop", "laptop")) == nil {
|
||||||
|
t.Fatalf("shop beside a healthy provider is not raised: %v", openKeysOf(open))
|
||||||
|
}
|
||||||
|
healths = shopFailingBeside(waitingDatabase())
|
||||||
|
if err := judgeModuleHealth(ctx, nil, k, "laptop", shop, map[string]int{"shop": 3}, time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var why string
|
||||||
|
for deadline := time.Now().Add(2 * time.Second); why == "" && time.Now().Before(deadline); {
|
||||||
|
events, err := k.HistorySince(ctx, start)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, e := range events {
|
||||||
|
if e.Key == moduleUnhealthyKey("shop", "laptop") && e.Change == conditions.ChangeCleared {
|
||||||
|
why = e.Why
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if why == "" {
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !strings.Contains(why, "waits on db on anchor, which waits for you") {
|
||||||
|
t.Fatalf("shop's clearing says %q; want it to say db on anchor waits for you", why)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,130 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// A module's act on the operator's warrant, recorded in the hand-act log (novox/hq ADR 0274, ADR 0259 §6).
|
||||||
|
//
|
||||||
|
// mesh-controller hand-act warrant --asker <module> --ask <id>
|
||||||
|
//
|
||||||
|
// The verb `warranted` runs it. A module that asks the operator (an asker) acts on the warrant with its own grants;
|
||||||
|
// the controller's log is where a person's decisions are read back, so the module asks the controller to record
|
||||||
|
// it. **What is recorded is the router's word, never the caller's**: the controller reads the router's own record
|
||||||
|
// of that asker's ask — the bus lets only the router write it — and records who chose, through which channel, with
|
||||||
|
// which proofs, and which answer. The caller gives nothing but which ask: a word of its own, recorded first under
|
||||||
|
// the one id, would stand for every node's (the review of 2026-10-10). Recorded once per
|
||||||
|
// ask, under an id the ask decides, however many of the module's instances ask; an ask still open, ended without
|
||||||
|
// a choice, or another asker's is refused and nothing is written.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
var askerModule = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`)
|
||||||
|
|
||||||
|
// warrantedID is the one entry an ask's warrant is recorded under.
|
||||||
|
func warrantedID(asker, ask string) string { return "warrant-" + asker + "-" + ask }
|
||||||
|
|
||||||
|
// warrantedAct is the entry for an asker's act on the warrant the router recorded for its ask (state, w), or why
|
||||||
|
// none is written.
|
||||||
|
func warrantedAct(asker, ask, caller, state string, w *asks.Warrant) (link.HandAct, error) {
|
||||||
|
switch {
|
||||||
|
case !askerModule.MatchString(asker):
|
||||||
|
return link.HandAct{}, fmt.Errorf("%q is not a module's name", asker)
|
||||||
|
case asker == askerName:
|
||||||
|
return link.HandAct{}, errors.New("the controller records its own acts on a warrant as it performs them")
|
||||||
|
case !asks.UsableID(ask):
|
||||||
|
return link.HandAct{}, fmt.Errorf("%q is not an ask's id", ask)
|
||||||
|
case state == "" || w == nil:
|
||||||
|
return link.HandAct{}, fmt.Errorf("the router holds no closed record of %s's ask %s", asker, ask)
|
||||||
|
case state == "open":
|
||||||
|
return link.HandAct{}, fmt.Errorf("%s's ask %s is still open: nobody has answered it", asker, ask)
|
||||||
|
case w.Asker != asker || w.Ask != ask:
|
||||||
|
return link.HandAct{}, fmt.Errorf("the router's record is for %s's ask %s", w.Asker, w.Ask)
|
||||||
|
case w.Outcome != asks.OutcomeChosen || w.By == nil:
|
||||||
|
return link.HandAct{}, fmt.Errorf("%s's ask %s ended %s: no person chose, so there is no warrant to record", asker, ask, w.Outcome)
|
||||||
|
case w.AskDigest == "":
|
||||||
|
return link.HandAct{}, fmt.Errorf("the router's warrant for %s's ask %s names no ask digest", asker, ask)
|
||||||
|
}
|
||||||
|
return link.HandAct{ID: warrantedID(asker, ask), Verb: handActWarrant, Args: []string{fmt.Sprintf("the operator chose %s on %s's ask %s", w.Label, asker, ask)},
|
||||||
|
Why: fmt.Sprintf("%s (ask %s of %s)", w.Says(), ask, asker), By: byWords(*w), Cause: conditions.CauseOperatorAnswer,
|
||||||
|
Via: viaWords(*w), Ask: ask, Proofs: w.Proofs, RequestedBy: asker + ", recorded at the word of " + caller,
|
||||||
|
Outcome: "chosen; what " + asker + " did with it is in its own record", At: w.At.UTC()}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// readRouterRecord reads the router's record of one asker's ask: its state and warrant, or "" when there is none.
|
||||||
|
// The controller's grant reaches the JetStream API whole (`$JS.API.>`), so it reads any asker's record.
|
||||||
|
func readRouterRecord(ctx context.Context, conn *nats.Conn, bucket, asker, ask string) (string, *asks.Warrant, error) {
|
||||||
|
reply, err := conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET.KV_"+bucket+".$KV."+bucket+"."+asker+"."+ask, nil)
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, err
|
||||||
|
}
|
||||||
|
if status := reply.Header.Get("Status"); status != "" {
|
||||||
|
if status == "404" {
|
||||||
|
return "", nil, nil
|
||||||
|
}
|
||||||
|
return "", nil, fmt.Errorf("the router's record could not be read: %s %s", status, reply.Header.Get("Description"))
|
||||||
|
}
|
||||||
|
var rec struct {
|
||||||
|
State string `json:"state"`
|
||||||
|
Warrant *asks.Warrant `json:"warrant"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(reply.Data, &rec); err != nil {
|
||||||
|
return "", nil, fmt.Errorf("the router's record of %s's ask %s cannot be read: %w", asker, ask, err)
|
||||||
|
}
|
||||||
|
return rec.State, rec.Warrant, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func handActWarrantCommand(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("hand-act warrant", flag.ContinueOnError)
|
||||||
|
asker := set.String("asker", "", "the module that asked")
|
||||||
|
ask := set.String("ask", "", "its ask's id")
|
||||||
|
positionals, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if *asker == "" || *ask == "" || len(positionals) > 0 {
|
||||||
|
return errors.New("hand-act warrant --asker <module> --ask <id>: what is recorded is the router's record, and nothing else")
|
||||||
|
}
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
bucket, err := asksRecords(ctx, open.inventory)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if bucket == "" {
|
||||||
|
return errors.New("no module declares the operator channel's records, so no warrant can be read")
|
||||||
|
}
|
||||||
|
return onTheBus(func(conn *nats.Conn) error {
|
||||||
|
state, w, err := readRouterRecord(ctx, conn, bucket, *asker, *ask)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
act, err := warrantedAct(*asker, *ask, link.Caller(), state, w)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("%w. Nothing was recorded", err)
|
||||||
|
}
|
||||||
|
written, err := link.RecordHandActOnce(ctx, conn, act)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the warrant could not be recorded: %w", err)
|
||||||
|
}
|
||||||
|
if !written {
|
||||||
|
fmt.Printf("already recorded as %s: %s\n", act.ID, act.Why)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
fmt.Printf("recorded as %s: %s, through %s\n", act.ID, act.Why, act.Via)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
)
|
||||||
|
|
||||||
|
func chosenWarrant() *asks.Warrant {
|
||||||
|
return &asks.Warrant{Ask: "instr-1", Asker: "claude-code", Outcome: asks.OutcomeChosen, Option: "approve",
|
||||||
|
Label: "Approve", Level: asks.Approve, Channel: "telegram", Proofs: []string{"P1"},
|
||||||
|
By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"},
|
||||||
|
At: time.Date(2026, 10, 10, 4, 0, 0, 0, time.UTC), AskDigest: "sha256:ab"}
|
||||||
|
}
|
||||||
|
|
||||||
|
// What is recorded of a module's act on a warrant is the router's word (novox/hq ADR 0274): who chose, how and
|
||||||
|
// with which proofs; the caller gives only what it did. Nothing is recorded without a person's choice.
|
||||||
|
func TestAWarrantIsRecordedFromTheRoutersRecordAlone(t *testing.T) {
|
||||||
|
act, err := warrantedAct("claude-code", "instr-1", "node-tools.shanks", "chosen", chosenWarrant())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if act.ID != "warrant-claude-code-instr-1" || act.Verb != handActWarrant || act.Cause != conditions.CauseOperatorAnswer ||
|
||||||
|
act.By != "the operator, as telegram identity 42" || act.Ask != "instr-1" || !slices.Equal(act.Proofs, []string{"P1"}) ||
|
||||||
|
!strings.Contains(act.Why, "the operator, via telegram (user id verified), chose Approve") ||
|
||||||
|
!strings.Contains(act.RequestedBy, "node-tools.shanks") ||
|
||||||
|
!slices.Equal(act.Args, []string{"the operator chose Approve on claude-code's ask instr-1"}) {
|
||||||
|
t.Fatalf("recorded as %+v", act)
|
||||||
|
}
|
||||||
|
for name, c := range map[string]struct {
|
||||||
|
asker, ask, state string
|
||||||
|
w func() *asks.Warrant
|
||||||
|
}{
|
||||||
|
"no record": {"claude-code", "instr-1", "", func() *asks.Warrant { return nil }},
|
||||||
|
"still open": {"claude-code", "instr-1", "open", chosenWarrant},
|
||||||
|
"another asker's": {"messenger", "instr-1", "chosen", chosenWarrant},
|
||||||
|
"another ask's": {"claude-code", "instr-2", "chosen", chosenWarrant},
|
||||||
|
"the controller's": {"mesh-controller", "instr-1", "chosen", chosenWarrant},
|
||||||
|
"not a module": {"Claude Code", "instr-1", "chosen", chosenWarrant},
|
||||||
|
"expired": {"claude-code", "instr-1", "expired", func() *asks.Warrant {
|
||||||
|
w := chosenWarrant()
|
||||||
|
w.Outcome, w.By = asks.OutcomeExpired, nil
|
||||||
|
return w
|
||||||
|
}},
|
||||||
|
"no digest": {"claude-code", "instr-1", "chosen", func() *asks.Warrant {
|
||||||
|
w := chosenWarrant()
|
||||||
|
w.AskDigest = ""
|
||||||
|
return w
|
||||||
|
}},
|
||||||
|
} {
|
||||||
|
if _, err := warrantedAct(c.asker, c.ask, "x", c.state, c.w()); err == nil {
|
||||||
|
t.Errorf("%s: recorded", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheWarrantedVerbRunsTheWarrantLineWithoutAWhy(t *testing.T) {
|
||||||
|
if _, err := argvFor("warranted", map[string]any{"asker": "claude-code", "ask": "instr-1", "what": "a word of the caller's"}); err == nil {
|
||||||
|
t.Error("the caller's own words were taken into the record")
|
||||||
|
}
|
||||||
|
argv, err := argvFor("warranted", map[string]any{"asker": "claude-code", "ask": "instr-1"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !slices.Equal(argv, []string{"hand-act", "warrant", "--asker", "claude-code", "--ask", "instr-1"}) {
|
||||||
|
t.Fatalf("%v", argv)
|
||||||
|
}
|
||||||
|
if repairingCommand(argv) != "" {
|
||||||
|
t.Error("recording a person's answer is taken for a repair")
|
||||||
|
}
|
||||||
|
if terminalOnly(argv) != nil {
|
||||||
|
t.Error("the verb is kept for the terminal")
|
||||||
|
}
|
||||||
|
if _, err := argvFor("warranted", map[string]any{"asker": "claude-code"}); err == nil {
|
||||||
|
t.Error("a call naming no ask was taken")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -53,6 +53,12 @@ type signalFacts struct {
|
|||||||
plansErr error
|
plansErr error
|
||||||
// waits are the walks waiting for their delivery's word (S16, novox/hq ADR 0239).
|
// waits are the walks waiting for their delivery's word (S16, novox/hq ADR 0239).
|
||||||
waits []waitFacts
|
waits []waitFacts
|
||||||
|
// batches are the batches not yet cut (S18, S19, novox/hq ADR 0276).
|
||||||
|
batches []batchFacts
|
||||||
|
// refusedSends are the sends a machine refused within the hour for the generation they were composed
|
||||||
|
// from, each naming its sender (S20, novox/hq issue 234).
|
||||||
|
refusedSends []inventory.Send
|
||||||
|
refusedSendsErr error
|
||||||
|
|
||||||
loop loopFacts
|
loop loopFacts
|
||||||
loopErr error
|
loopErr error
|
||||||
@@ -78,6 +84,9 @@ type signalFacts struct {
|
|||||||
// deadLetters are how many messages DEAD_LETTERS holds per consumer, by `<stream>.<consumer>`
|
// deadLetters are how many messages DEAD_LETTERS holds per consumer, by `<stream>.<consumer>`
|
||||||
// (novox/hq issue 330): each consumer's max-deliveries condition is open while it holds any.
|
// (novox/hq issue 330): each consumer's max-deliveries condition is open while it holds any.
|
||||||
deadLetters map[string]int
|
deadLetters map[string]int
|
||||||
|
// deadLettersNewest is when DEAD_LETTERS kept the newest message it holds for each of those
|
||||||
|
// consumers: the condition counts the messages given up on, not the looks (novox/hq issue 440).
|
||||||
|
deadLettersNewest map[string]time.Time
|
||||||
advisoriesErr error
|
advisoriesErr error
|
||||||
|
|
||||||
selfCheck selfCheckFacts
|
selfCheck selfCheckFacts
|
||||||
@@ -159,6 +168,8 @@ type waitFacts struct {
|
|||||||
since time.Time
|
since time.Time
|
||||||
// modules are the modules the walk moves, as a person names the delivery.
|
// modules are the modules the walk moves, as a person names the delivery.
|
||||||
modules []string
|
modules []string
|
||||||
|
// merges are the merges it answers (novox/hq ADR 0276), as the deliveries to read are found.
|
||||||
|
merges []inventory.PlanMerge
|
||||||
}
|
}
|
||||||
|
|
||||||
type loopFacts struct {
|
type loopFacts struct {
|
||||||
@@ -326,6 +337,9 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
|
|||||||
f.machines, f.machinesErr = w.gatherMachines(ctx, inv, now)
|
f.machines, f.machinesErr = w.gatherMachines(ctx, inv, now)
|
||||||
f.bus, f.busErr = gatherBus(ctx, inv)
|
f.bus, f.busErr = gatherBus(ctx, inv)
|
||||||
f.plans, f.waits, f.plansErr = gatherPlans(ctx, inv, now, f.bus.heldByTheBus())
|
f.plans, f.waits, f.plansErr = gatherPlans(ctx, inv, now, f.bus.heldByTheBus())
|
||||||
|
if f.plansErr == nil {
|
||||||
|
f.batches, f.plansErr = gatherBatches(ctx, inv, now)
|
||||||
|
}
|
||||||
f.loop, f.loopErr = w.gatherLoop()
|
f.loop, f.loopErr = w.gatherLoop()
|
||||||
f.mergesPassed, f.merges, f.mergesErr = watchedMerges.last()
|
f.mergesPassed, f.merges, f.mergesErr = watchedMerges.last()
|
||||||
if f.mergesErr == nil && !f.mergesPassed.IsZero() && now.Sub(f.mergesPassed) > 3*mergeCatchUpEvery {
|
if f.mergesErr == nil && !f.mergesPassed.IsZero() && now.Sub(f.mergesPassed) > 3*mergeCatchUpEvery {
|
||||||
@@ -343,7 +357,18 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
|
|||||||
if f.advisoriesErr == nil && w.js != nil {
|
if f.advisoriesErr == nil && w.js != nil {
|
||||||
f.deadLetters, f.advisoriesErr = link.HeldDeadLetters(w.js.Context())
|
f.deadLetters, f.advisoriesErr = link.HeldDeadLetters(w.js.Context())
|
||||||
}
|
}
|
||||||
|
if f.advisoriesErr == nil && len(f.deadLetters) > 0 {
|
||||||
|
f.deadLettersNewest, f.advisoriesErr = link.NewestDeadLetters(w.js.Context(), f.deadLetters)
|
||||||
|
for key := range f.deadLetters {
|
||||||
|
if _, ok := f.deadLettersNewest[key]; !ok && f.advisoriesErr == nil {
|
||||||
|
// Delivered again or dropped between the two reads: left out of this look, rather than
|
||||||
|
// observed without a time and counted as a look (novox/hq issue 440).
|
||||||
|
delete(f.deadLetters, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
f.handActs, f.handActsErr = w.gatherHandActs(ctx, now)
|
f.handActs, f.handActsErr = w.gatherHandActs(ctx, now)
|
||||||
|
f.refusedSends, f.refusedSendsErr = inv.RefusedSendsSince(ctx, now.Add(-generationRefusalsSaid))
|
||||||
f.facts.taken, _, f.facts.began, f.facts.err = exportedFacts.last()
|
f.facts.taken, _, f.facts.began, f.facts.err = exportedFacts.last()
|
||||||
return f
|
return f
|
||||||
}
|
}
|
||||||
@@ -480,7 +505,12 @@ func gatherPlans(ctx context.Context, inv *inventory.Inventory, now time.Time, b
|
|||||||
// S16's, whatever mesh-delivery says or does not say.
|
// S16's, whatever mesh-delivery says or does not say.
|
||||||
if p.Waiting() {
|
if p.Waiting() {
|
||||||
waits = append(waits, waitFacts{id: p.ID, repository: p.Repository, commit: p.Commit,
|
waits = append(waits, waitFacts{id: p.ID, repository: p.Repository, commit: p.Commit,
|
||||||
awaits: p.Delivery.Awaits, since: p.Created, modules: planModules(p)})
|
awaits: p.Delivery.Awaits, since: p.Created, modules: planModules(p), merges: p.Delivery.Merges})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// A walk let go and waiting for the walk before it to end (ADR 0276) is no tier late either: the walk
|
||||||
|
// before it is the one S3 watches.
|
||||||
|
if deferred(p) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
_, paused := pausedWaiting(p, pause, now)
|
_, paused := pausedWaiting(p, pause, now)
|
||||||
|
|||||||
@@ -19,10 +19,12 @@ func TestTheBuildSeatsHolderFollowsTheControllerThatDefinesItsWorker(t *testing.
|
|||||||
{From: "route-proxy", To: "mesh-controller", Kind: inventory.EdgePackages},
|
{From: "route-proxy", To: "mesh-controller", Kind: inventory.EdgePackages},
|
||||||
{From: "route-proxy", To: "build-agent", Kind: inventory.EdgeBuiltBy},
|
{From: "route-proxy", To: "build-agent", Kind: inventory.EdgeBuiltBy},
|
||||||
}
|
}
|
||||||
set := reachableFrom([]string{"mesh-controller"}, edges)
|
// A packages edge recorded before novox/hq ADR 0267 widens nothing: the controller moved alone moves
|
||||||
if len(set) != 3 {
|
// alone, and a change to a package all three build from moves all three, each by its own build source.
|
||||||
t.Fatalf("the controller, what packages it, and nothing more: %v", set)
|
if alone := reachableFrom([]string{"mesh-controller"}, edges); len(alone) != 1 {
|
||||||
|
t.Fatalf("the controller alone, whatever packages its repository: %v", alone)
|
||||||
}
|
}
|
||||||
|
set := reachableFrom([]string{"mesh-controller", "build-agent", "route-proxy"}, edges)
|
||||||
tiers := tiersOf(set, edges)
|
tiers := tiersOf(set, edges)
|
||||||
pos := map[string]int{}
|
pos := map[string]int{}
|
||||||
for i, tier := range tiers {
|
for i, tier := range tiers {
|
||||||
|
|||||||
@@ -3,13 +3,14 @@ module github.com/novox/mesh-controller
|
|||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.10-0.20261009115850-16984aafc689
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c
|
||||||
github.com/jackc/pgx/v5 v5.10.0
|
github.com/jackc/pgx/v5 v5.10.0
|
||||||
github.com/nats-io/nats-server/v2 v2.11.17
|
github.com/nats-io/nats-server/v2 v2.11.17
|
||||||
github.com/nats-io/nats.go v1.54.0
|
github.com/nats-io/nats.go v1.54.0
|
||||||
github.com/novox/mesh-host v0.0.0
|
github.com/novox/mesh-host v0.0.0
|
||||||
golang.org/x/crypto v0.57.0
|
golang.org/x/crypto v0.57.0
|
||||||
golang.org/x/net v0.58.0
|
golang.org/x/net v0.58.0
|
||||||
|
golang.org/x/sys v0.48.0
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
@@ -24,7 +25,6 @@ require (
|
|||||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||||
github.com/nats-io/nuid v1.0.1 // indirect
|
github.com/nats-io/nuid v1.0.1 // indirect
|
||||||
golang.org/x/sync v0.23.0 // indirect
|
golang.org/x/sync v0.23.0 // indirect
|
||||||
golang.org/x/sys v0.48.0 // indirect
|
|
||||||
golang.org/x/text v0.42.0 // indirect
|
golang.org/x/text v0.42.0 // indirect
|
||||||
golang.org/x/time v0.15.0 // indirect
|
golang.org/x/time v0.15.0 // indirect
|
||||||
)
|
)
|
||||||
@@ -35,4 +35,4 @@ require (
|
|||||||
// committed. Every build (the build agent's `go build`, the Dockerfile) compiles from vendor/ and
|
// committed. Every build (the build agent's `go build`, the Dockerfile) compiles from vendor/ and
|
||||||
// fetches nothing; go refuses to build when vendor/ and this file disagree, so a pin moved without
|
// fetches nothing; go refuses to build when vendor/ and this file disagree, so a pin moved without
|
||||||
// `go mod vendor` fails loudly, at once, everywhere.
|
// `go mod vendor` fails loudly, at once, everywhere.
|
||||||
replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261009103656-1c61b72f354d
|
replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f
|
||||||
|
|||||||
@@ -1,25 +1,7 @@
|
|||||||
git.novox.be/novox/mesh-host v0.0.0-20261006095519-3e80b7ae325e h1:g9h4QRaAMg5yaJLwqtb0FoOs23DVGUYpW6qvnQ3oY5A=
|
git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f h1:8N5OW2mdTNIck2pe4EciTYX5NsrPsHrTLENGNIWYNTU=
|
||||||
git.novox.be/novox/mesh-host v0.0.0-20261006095519-3e80b7ae325e/go.mod h1:VlilMCRZ5yyNXg7SNigNBLr0Gt32jrGw5KSNq5JAVYs=
|
git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f/go.mod h1:tcTK4LMs1d6JpZUwy3hSHMFG/MIuDr/XFs7/nbeaW/c=
|
||||||
git.novox.be/novox/mesh-host v0.0.0-20261007120832-bdd44154ccac h1:KvnKtJ2rWeIE/t4GweK+JL0OjKSNxsrVP3/nMdpii8o=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s=
|
||||||
git.novox.be/novox/mesh-host v0.0.0-20261007120832-bdd44154ccac/go.mod h1:VlilMCRZ5yyNXg7SNigNBLr0Gt32jrGw5KSNq5JAVYs=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||||
git.novox.be/novox/mesh-host v0.0.0-20261007162834-56e2ebec4bac h1:yLtFS0pDCCqIE9Zx8hgXEFG9fUWzf8L9WQoKV+Amk1E=
|
|
||||||
git.novox.be/novox/mesh-host v0.0.0-20261007162834-56e2ebec4bac/go.mod h1:VlilMCRZ5yyNXg7SNigNBLr0Gt32jrGw5KSNq5JAVYs=
|
|
||||||
git.novox.be/novox/mesh-host v0.0.0-20261009081005-b28d7bbcbff4 h1:f4rBnKSemuN0Z9dTtRJMigIGfEs6ltFPOILJGHGab74=
|
|
||||||
git.novox.be/novox/mesh-host v0.0.0-20261009081005-b28d7bbcbff4/go.mod h1:72ZATZjxMLaJfWdvlSDJrygIoBzCmKIjCDMhEXxVzTo=
|
|
||||||
git.novox.be/novox/mesh-host v0.0.0-20261009101157-2673e7a2c95e h1:H7eVqDILL6e9cMbWSLHTbCqu9ZxDOmyeQhUmWl9QBV0=
|
|
||||||
git.novox.be/novox/mesh-host v0.0.0-20261009101157-2673e7a2c95e/go.mod h1:72ZATZjxMLaJfWdvlSDJrygIoBzCmKIjCDMhEXxVzTo=
|
|
||||||
git.novox.be/novox/mesh-host v0.0.0-20261009103656-1c61b72f354d h1:IrmJ+lz21n+eSqKrmXREtR/7raUCBJ+fZvs+BNhuXVI=
|
|
||||||
git.novox.be/novox/mesh-host v0.0.0-20261009103656-1c61b72f354d/go.mod h1:72ZATZjxMLaJfWdvlSDJrygIoBzCmKIjCDMhEXxVzTo=
|
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261008145004-62367ce15ad6 h1:JT7xM1bnLNInW7/oImV2OlXTrcQ4/GSM0Y8tAb+AhmY=
|
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261008145004-62367ce15ad6/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261008162031-55090da7e08f h1:BNvyWq899GwP7F3sY4ACieB5a5fnFAq+sJ9lP6HQ5qI=
|
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261008162031-55090da7e08f/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261009081503-d4077b473ea8 h1:soqhLNpEXThdq6PdiPy6ExxjJ+yjhh1N1n9E3j1CtrM=
|
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261009081503-d4077b473ea8/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261009095928-76902998cd39 h1:WHW6CgbuTxP7M+qRBOgzsiG9vT49xdkZ/rarc9/vKMA=
|
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261009095928-76902998cd39/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.10-0.20261009115850-16984aafc689 h1:Ti2P9nwders7YQ/hq3X/dPo+CXMj5pdUcfA5P/c12CU=
|
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.10-0.20261009115850-16984aafc689/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
@@ -56,8 +38,6 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
|
|||||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||||
go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
|
|
||||||
go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
|
|
||||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||||
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
|
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
|
||||||
|
|||||||
@@ -0,0 +1,95 @@
|
|||||||
|
// Package beside is where a test finds another repository of the mesh it reads: the catalogue's manifests,
|
||||||
|
// the node-engine's genesis template (novox/hq issue 432), the SDK's conformance fixtures (issue 449).
|
||||||
|
//
|
||||||
|
// A test used to read the checkout beside this one, `../../../mesh-catalog`, so its verdict depended on
|
||||||
|
// whatever sat on the machine running it: a stale or dirty checkout failed it on a desktop, and where none
|
||||||
|
// was beside it skipped and said nothing. Now there are two inputs, both named, and no third:
|
||||||
|
//
|
||||||
|
// - In a merge check, the build seat clones each core repository beside the one checked (the catalogue
|
||||||
|
// at its main, the node-engine at the commit the mesh runs) and says where in MESH_CHECK_BESIDE. A
|
||||||
|
// test judges against those clones, so agreement with the other repository is checked where
|
||||||
|
// `mesh/repo-check` runs; a repository missing there fails the test, never skips it. Which clones a
|
||||||
|
// check gets is chosen from the inventory: mesh-catalog by the source of the `nats` module, mesh-host
|
||||||
|
// by the source of `mesh-host`, and mesh-sdk as the controller's sibling at the commit the controller's
|
||||||
|
// go.mod pins. In a mesh where either module has no source repository nothing is
|
||||||
|
// cloned, and these tests fail loudly with "not beside this check": a cause in the setup, not in the
|
||||||
|
// change. And in a delivery group that holds a mesh-catalog pull request, these tests read the
|
||||||
|
// catalogue's main, not the group's head.
|
||||||
|
// - Anywhere else, the test judges against the copy captured in this repository's testdata/beside, at the
|
||||||
|
// commit testdata/beside/CAPTURED names. Never against a developer's own checkout: to judge one, set
|
||||||
|
// MESH_CHECK_BESIDE to the directory holding it, as the check does.
|
||||||
|
//
|
||||||
|
// The captured manifests are named module.json.captured, and put back as module.json in a directory of
|
||||||
|
// the test's own: a module.json anywhere in this repository is a module of it to the forge's announcer and
|
||||||
|
// the planner, and a merge would build and register a hundred copies of the catalogue's.
|
||||||
|
package beside
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Env is where a merge check says the repositories cloned beside the one checked are (internal/builder's
|
||||||
|
// EnvBeside, repeated here so a test does not import the builder).
|
||||||
|
const Env = "MESH_CHECK_BESIDE"
|
||||||
|
|
||||||
|
// CapturedSuffix is what a captured file's name carries that the repository's own does not.
|
||||||
|
const CapturedSuffix = ".captured"
|
||||||
|
|
||||||
|
// Dir is the named repository a test reads — the clone beside a merge check, or the captured copy — and
|
||||||
|
// says which in the test's log.
|
||||||
|
func Dir(t testing.TB, repository string) string {
|
||||||
|
t.Helper()
|
||||||
|
if root := os.Getenv(Env); root != "" {
|
||||||
|
dir := filepath.Join(root, repository)
|
||||||
|
if _, err := os.Stat(dir); err != nil {
|
||||||
|
t.Fatalf("%s is not beside this check in %s=%s, so its agreement with this repository cannot be "+
|
||||||
|
"judged here: %v", repository, Env, root, err)
|
||||||
|
}
|
||||||
|
t.Logf("judged against %s as cloned beside this check", dir)
|
||||||
|
return dir
|
||||||
|
}
|
||||||
|
from := filepath.Join(Captured(), repository)
|
||||||
|
if _, err := os.Stat(from); err != nil {
|
||||||
|
t.Fatalf("no captured copy of %s at %s: %v", repository, from, err)
|
||||||
|
}
|
||||||
|
dir := filepath.Join(t.TempDir(), repository)
|
||||||
|
err := filepath.WalkDir(from, func(path string, d fs.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
rel, _ := filepath.Rel(from, path)
|
||||||
|
into := filepath.Join(dir, strings.TrimSuffix(rel, CapturedSuffix))
|
||||||
|
if d.IsDir() {
|
||||||
|
return os.MkdirAll(into, 0o755)
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.WriteFile(into, raw, 0o644)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("the captured copy of %s could not be laid out: %v", repository, err)
|
||||||
|
}
|
||||||
|
t.Logf("judged against the copy of %s captured in testdata/beside (see CAPTURED); a merge check "+
|
||||||
|
"judges it against the repository's own clone", repository)
|
||||||
|
return dir
|
||||||
|
}
|
||||||
|
|
||||||
|
// Catalogue is the catalogue's modules directory, as Dir finds the catalogue.
|
||||||
|
func Catalogue(t testing.TB) string {
|
||||||
|
t.Helper()
|
||||||
|
return filepath.Join(Dir(t, "mesh-catalog"), "modules")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Captured is this repository's testdata/beside, found from this file rather than from the working
|
||||||
|
// directory, so a test in any package reaches it.
|
||||||
|
func Captured() string {
|
||||||
|
_, file, _, _ := runtime.Caller(0)
|
||||||
|
return filepath.Join(filepath.Dir(file), "..", "..", "testdata", "beside")
|
||||||
|
}
|
||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/beside"
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -74,10 +75,10 @@ func TestTheAgreementCheckCatchesASubscriptionThatMatchesNothing(t *testing.T) {
|
|||||||
|
|
||||||
func theCataloguesEvents(t *testing.T) ([]AnEmitter, []AConsumer, []DeclaredSeat) {
|
func theCataloguesEvents(t *testing.T) ([]AnEmitter, []AConsumer, []DeclaredSeat) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
root := filepath.Join("..", "..", "..", "mesh-catalog", "modules")
|
root := beside.Catalogue(t)
|
||||||
entries, err := os.ReadDir(root)
|
entries, err := os.ReadDir(root)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Skipf("catalogue sibling not present: %v", err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
var emitters []AnEmitter
|
var emitters []AnEmitter
|
||||||
var consumers []AConsumer
|
var consumers []AConsumer
|
||||||
@@ -119,7 +120,7 @@ func theCataloguesEvents(t *testing.T) ([]AnEmitter, []AConsumer, []DeclaredSeat
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(emitters) == 0 {
|
if len(emitters) == 0 {
|
||||||
t.Skip("no manifests found beside this checkout")
|
t.Fatalf("no module under %s emits anything, so this proved nothing", root)
|
||||||
}
|
}
|
||||||
return emitters, consumers, seats
|
return emitters, consumers, seats
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,117 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A tool call names its caller (novox/hq issue 365, by ADR 0259 §3's precedent for asks).
|
||||||
|
//
|
||||||
|
// mesh.mod.<module>.call.<tool>[.<node>].<caller>
|
||||||
|
// mesh.seat.<seat>.call.<verb>[.<node>].<caller>
|
||||||
|
//
|
||||||
|
// The last token is the bus user that published the call, and each user is granted these subjects with its own
|
||||||
|
// name there and no other — the caller is a fact the server enforces, and the tool runtime hands it to the
|
||||||
|
// module from the subject a call arrived on (mesh-tools node-tools internal/bus caller.go holds the same shape).
|
||||||
|
//
|
||||||
|
// **A kind of its own, `call`, not the `tool` subject with a token appended.** Every grant to call a tool is a
|
||||||
|
// wildcard over the `tool` kind — `.tool.<t>.*` for the instance on any machine, `mesh.mod.*.tool.>` for every
|
||||||
|
// tool — and either would match a `tool` subject with any caller appended, so a caller could name another.
|
||||||
|
// Under `call` nothing is granted but the caller-named subjects. No stream's filter covers it: a tool call is
|
||||||
|
// never persisted (design 25 §3).
|
||||||
|
//
|
||||||
|
// **Derived from the `tool` grants, in one place** (callerNamed): wherever a principal may call or answer a
|
||||||
|
// tool, it may call it in its own name, or answer it naming any caller — so no kind of principal is left
|
||||||
|
// unable to call in its own name, and a new grant to call is one in both shapes by construction.
|
||||||
|
//
|
||||||
|
// The `tool` grants stay beside these for one release, so every caller and every runtime moves without a gap:
|
||||||
|
// their retirement is hq issue 464.
|
||||||
|
const CallKind = "call"
|
||||||
|
|
||||||
|
var userName = regexp.MustCompile(`^[A-Za-z0-9_-]+(\.[A-Za-z0-9_-]+)*$`)
|
||||||
|
|
||||||
|
// CallerToken is a bus user's name as the last token of a call it publishes: each dot written `~`, which no part
|
||||||
|
// of a user's name may hold (safeSubject), so the token names that user and no other. "" for a name that is not
|
||||||
|
// a user's.
|
||||||
|
func CallerToken(user string) string {
|
||||||
|
if !userName.MatchString(user) {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return strings.ReplaceAll(user, ".", "~")
|
||||||
|
}
|
||||||
|
|
||||||
|
// toolGrant splits a grant on the `tool` kind — `mesh.mod.<m>.tool.<rest>` or `mesh.seat.<s>.tool.<rest>`, any
|
||||||
|
// part possibly a wildcard — at the kind; ok is false for any other subject.
|
||||||
|
func toolGrant(subject string) (head, rest string, ok bool) {
|
||||||
|
parts := strings.SplitN(subject, ".", 5)
|
||||||
|
if len(parts) != 5 || parts[0] != "mesh" || (parts[1] != "mod" && parts[1] != "seat") || parts[3] != "tool" ||
|
||||||
|
parts[2] == "" || parts[4] == "" {
|
||||||
|
return "", "", false
|
||||||
|
}
|
||||||
|
return parts[0] + "." + parts[1] + "." + parts[2], parts[4], true
|
||||||
|
}
|
||||||
|
|
||||||
|
// CalledSubject is where a call to a tool subject goes naming its caller; "" when the subject is not a tool's
|
||||||
|
// or the user is not a bus user.
|
||||||
|
func CalledSubject(subject, user string) string {
|
||||||
|
head, rest, ok := toolGrant(subject)
|
||||||
|
token := CallerToken(user)
|
||||||
|
if !ok || token == "" || strings.ContainsAny(rest, "*>") {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return head + "." + CallKind + "." + rest + "." + token
|
||||||
|
}
|
||||||
|
|
||||||
|
// CalledPattern is what a holder answering a tool subject also subscribes, to hear the calls that name their
|
||||||
|
// caller: the same address under the `call` kind, any caller last. "" for a subject that is not a tool's.
|
||||||
|
func CalledPattern(subject string) string {
|
||||||
|
head, rest, ok := toolGrant(subject)
|
||||||
|
if !ok || strings.ContainsAny(rest, "*>") {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return head + "." + CallKind + "." + rest + ".*"
|
||||||
|
}
|
||||||
|
|
||||||
|
// calledPublish is a grant to call on the `tool` kind, as the same grant in the caller's own name: its last
|
||||||
|
// token the caller's, and nothing that reaches past it. A `>` is every tail a call carries — the tool alone or
|
||||||
|
// the tool and the machine — so it becomes both, each ending in the caller.
|
||||||
|
func calledPublish(grant, token string) []string {
|
||||||
|
head, rest, ok := toolGrant(grant)
|
||||||
|
if !ok || token == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
base := head + "." + CallKind + "."
|
||||||
|
switch {
|
||||||
|
case rest == ">":
|
||||||
|
return []string{base + "*." + token, base + "*.*." + token}
|
||||||
|
case strings.HasSuffix(rest, ".>"):
|
||||||
|
return []string{base + strings.TrimSuffix(rest, ">") + "*." + token}
|
||||||
|
}
|
||||||
|
return []string{base + rest + "." + token}
|
||||||
|
}
|
||||||
|
|
||||||
|
// calledSubscribe is a grant to answer on the `tool` kind, as the same grant for the calls naming any caller.
|
||||||
|
func calledSubscribe(grant string) []string {
|
||||||
|
head, rest, ok := toolGrant(grant)
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
base := head + "." + CallKind + "."
|
||||||
|
if strings.HasSuffix(rest, ">") {
|
||||||
|
return []string{base + rest}
|
||||||
|
}
|
||||||
|
return []string{base + rest + ".*"}
|
||||||
|
}
|
||||||
|
|
||||||
|
// callerNamed adds, beside a principal's grants on the `tool` kind, the same grants under `call`: to publish in
|
||||||
|
// its own name, to subscribe naming anybody.
|
||||||
|
func callerNamed(user string, pub, sub []string) ([]string, []string) {
|
||||||
|
token := CallerToken(user)
|
||||||
|
for _, g := range pub {
|
||||||
|
pub = append(pub, calledPublish(g, token)...)
|
||||||
|
}
|
||||||
|
for _, g := range sub {
|
||||||
|
sub = append(sub, calledSubscribe(g)...)
|
||||||
|
}
|
||||||
|
return unique(pub), unique(sub)
|
||||||
|
}
|
||||||
@@ -0,0 +1,100 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats-server/v2/server"
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"golang.org/x/crypto/bcrypt"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **On a real server, as composed** (novox/hq issue 365): a machine's runtime calls in its own name, and the
|
||||||
|
// server refuses it a call naming another — the grant, not the runtime, is what makes the caller a fact.
|
||||||
|
func TestAServerComposedFromTheGrantsRefusesACallNamingAnother(t *testing.T) {
|
||||||
|
hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ledger := Seat{Name: "issue-tracker", Scope: "mesh", Serves: []string{"open"}}
|
||||||
|
accounts, err := ComposeAccounts([]Principal{
|
||||||
|
{Kind: KindNodeTools, Node: "novox", Module: RuntimeModule, PasswordHash: string(hash),
|
||||||
|
Carries: []Declared{{Module: "mesh-issues", Holds: []Seat{ledger}}}},
|
||||||
|
{Kind: KindNodeTools, Node: "shanks", Module: RuntimeModule, PasswordHash: string(hash)},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
conf := filepath.Join(t.TempDir(), "bus.conf")
|
||||||
|
if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+
|
||||||
|
`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
opts, err := server.ProcessConfigFile(conf)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("the composed accounts do not parse: %v", err)
|
||||||
|
}
|
||||||
|
opts.NoLog, opts.NoSigs = true, true
|
||||||
|
s, err := server.NewServer(opts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
go s.Start()
|
||||||
|
if !s.ReadyForConnections(10 * time.Second) {
|
||||||
|
t.Fatal("the bus did not come up")
|
||||||
|
}
|
||||||
|
defer s.Shutdown()
|
||||||
|
|
||||||
|
holder, err := nats.Connect(s.ClientURL(), nats.UserInfo("novox.node-tools", "pw"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer holder.Close()
|
||||||
|
heard := make(chan string, 4)
|
||||||
|
sub, err := holder.Subscribe("mesh.seat.issue-tracker.call.open.*", func(m *nats.Msg) {
|
||||||
|
heard <- m.Subject
|
||||||
|
_ = m.Respond([]byte(`{"result":{}}`))
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = holder.Flush()
|
||||||
|
if !sub.IsValid() {
|
||||||
|
t.Fatal("the holder may not hear the caller-named calls to its seat")
|
||||||
|
}
|
||||||
|
|
||||||
|
refusals := make(chan error, 4)
|
||||||
|
caller, err := nats.Connect(s.ClientURL(), nats.UserInfo("shanks.node-tools", "pw"),
|
||||||
|
nats.CustomInboxPrefix("_INBOX.shanks.node-tools"),
|
||||||
|
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refusals <- err }))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer caller.Close()
|
||||||
|
if _, err := caller.Request("mesh.seat.issue-tracker.call.open.shanks~node-tools", []byte(`{}`), 3*time.Second); err != nil {
|
||||||
|
t.Fatalf("a call in the caller's own name was not answered: %v", err)
|
||||||
|
}
|
||||||
|
if got := <-heard; got != "mesh.seat.issue-tracker.call.open.shanks~node-tools" {
|
||||||
|
t.Fatalf("heard %s", got)
|
||||||
|
}
|
||||||
|
if err := caller.Publish("mesh.seat.issue-tracker.call.open.novox~node-tools", []byte(`{}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = caller.Flush()
|
||||||
|
select {
|
||||||
|
case err := <-refusals:
|
||||||
|
if !errors.Is(err, nats.ErrPermissionViolation) {
|
||||||
|
t.Errorf("the server said %v, want a permissions violation", err)
|
||||||
|
}
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
t.Error("the server did not refuse a call naming another caller")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case got := <-heard:
|
||||||
|
t.Errorf("a call naming another reached the holder: %s", got)
|
||||||
|
case <-time.After(200 * time.Millisecond):
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,119 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **A tool call names its caller, and the bus lets each user name itself alone** (novox/hq issue 365, by ADR
|
||||||
|
// 0259 §3's precedent for asks): wherever a principal may call a tool, it may call it on the caller-named
|
||||||
|
// subject — kind `call`, its own bus user last, dots written `~` — and on no subject naming anybody else.
|
||||||
|
func TestEachCredentialMayCallOnlyInItsOwnName(t *testing.T) {
|
||||||
|
ledger := Seat{Name: "node-desk", Scope: "node", Serves: []string{"who"}}
|
||||||
|
tracker := Seat{Name: "issue-tracker", Scope: "mesh", Serves: []string{"open"}}
|
||||||
|
for _, c := range []struct {
|
||||||
|
p Principal
|
||||||
|
may []string
|
||||||
|
mayNot []string
|
||||||
|
subject []string // what it subscribes, to answer calls naming any caller
|
||||||
|
}{
|
||||||
|
{p: Principal{Kind: KindPerson, Module: "jochen", Invokes: []string{"*"}},
|
||||||
|
may: []string{"mesh.mod.ledger.call.who.person~jochen", "mesh.mod.ledger.call.who.anchor.person~jochen",
|
||||||
|
"mesh.seat.issue-tracker.call.open.person~jochen", "mesh.seat.node-desk.call.who.anchor.person~jochen"},
|
||||||
|
mayNot: []string{"mesh.mod.ledger.call.who.shanks~node-tools", "mesh.mod.ledger.call.who.anchor.controller",
|
||||||
|
"mesh.seat.issue-tracker.call.open.person~somebody", "mesh.mod.ledger.call.who.person"}},
|
||||||
|
{p: Principal{Kind: KindNodeTools, Node: "shanks", Module: RuntimeModule,
|
||||||
|
Carries: []Declared{{Module: "ledger", Holds: []Seat{ledger, tracker}}}},
|
||||||
|
may: []string{"mesh.mod.ledger.call.who.shanks~node-tools", "mesh.seat.issue-tracker.call.open.shanks~node-tools"},
|
||||||
|
mayNot: []string{"mesh.mod.ledger.call.who.novox~node-tools", "mesh.seat.issue-tracker.call.open.controller"},
|
||||||
|
subject: []string{"mesh.mod.ledger.call.who.novox~node-tools", "mesh.seat.node-desk.call.who.shanks.person~jochen", "mesh.seat.issue-tracker.call.open.controller"}},
|
||||||
|
{p: Principal{Kind: KindModule, Node: "two", Module: "shop", Invokes: []string{"ledger.who", "seat:issue-tracker.open"}},
|
||||||
|
may: []string{"mesh.mod.ledger.call.who.two~shop", "mesh.mod.ledger.call.who.anchor.two~shop",
|
||||||
|
"mesh.seat.issue-tracker.call.open.two~shop"},
|
||||||
|
mayNot: []string{"mesh.mod.ledger.call.other.two~shop", "mesh.mod.ledger.call.who.one~shop",
|
||||||
|
"mesh.seat.issue-tracker.call.open.one~telegram"}},
|
||||||
|
{p: Principal{Kind: KindNode, Node: "one", Checks: []string{"ledger.health"}},
|
||||||
|
may: []string{"mesh.mod.ledger.call.health.one.node~one"},
|
||||||
|
mayNot: []string{"mesh.mod.ledger.call.health.two.node~one", "mesh.mod.ledger.call.health.one.node~two"}},
|
||||||
|
{p: Principal{Kind: KindModule, Node: "one", Module: "ledger", Holds: []Seat{ledger, tracker}},
|
||||||
|
subject: []string{"mesh.mod.ledger.call.who.person~jochen", "mesh.mod.ledger.call.who.one.controller",
|
||||||
|
"mesh.seat.node-desk.call.who.one.two~shop", "mesh.seat.issue-tracker.call.open.two~shop"}},
|
||||||
|
} {
|
||||||
|
perms, err := PermissionsFor(c.p)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: %v", c.p.Username(), err)
|
||||||
|
}
|
||||||
|
for _, s := range c.may {
|
||||||
|
if !MayPublish(perms, s) {
|
||||||
|
t.Errorf("%s may not call %s, in its own name", c.p.Username(), s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, s := range c.mayNot {
|
||||||
|
if MayPublish(perms, s) {
|
||||||
|
t.Errorf("%s may call %s, naming somebody else", c.p.Username(), s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, s := range c.subject {
|
||||||
|
if !MaySubscribe(perms, s) {
|
||||||
|
t.Errorf("%s does not hear %s, a call to what it serves", c.p.Username(), s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The subjects that name no caller stay granted beside the caller-named ones for one release, so a caller
|
||||||
|
// moves over without a gap (their retirement: hq issue 464).
|
||||||
|
func TestTheSubjectsThatNameNoCallerStayGrantedForOneRelease(t *testing.T) {
|
||||||
|
perms, err := PermissionsFor(Principal{Kind: KindModule, Node: "two", Module: "shop", Invokes: []string{"ledger.who"}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, s := range []string{"mesh.mod.ledger.tool.who", "mesh.mod.ledger.tool.who.anchor"} {
|
||||||
|
if !MayPublish(perms, s) {
|
||||||
|
t.Errorf("the old subject %s is no longer granted", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The desk's hidden prompt is the controller's alone on the caller-named subjects too (the review of
|
||||||
|
// 2026-10-09, M4): a grant of every tool does not reach it there either.
|
||||||
|
func TestTheDesksPromptIsTheControllersAloneUnderCallToo(t *testing.T) {
|
||||||
|
for _, p := range []Principal{{Kind: KindPerson, Module: "jochen", Invokes: []string{"*"}},
|
||||||
|
{Kind: KindNodeTools, Node: "shanks", Module: RuntimeModule}} {
|
||||||
|
perms, err := PermissionsFor(p)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
token := CallerToken(p.Username())
|
||||||
|
for _, s := range []string{"mesh.seat.node-launcher.call.secret.shanks." + token,
|
||||||
|
"mesh.mod.shell.call.node-launcher.secret.shanks." + token, "mesh.mod.shell.call.node-launcher.secret." + token} {
|
||||||
|
if MayPublish(perms, s) {
|
||||||
|
t.Errorf("%s may publish %s, the desk's hidden prompt", p.Username(), s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The controller's grants are the installer's first user list too, so they move with hq issue 464: this release
|
||||||
|
// it calls and serves on the subjects that name no caller alone, and nobody may call in its name.
|
||||||
|
func TestTheControllerKeepsTheSubjectsThatNameNoCallerThisRelease(t *testing.T) {
|
||||||
|
perms, err := PermissionsFor(Principal{Kind: KindController})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, s := range append(perms.Publish, perms.Subscribe...) {
|
||||||
|
if strings.Contains(s, "."+CallKind+".") {
|
||||||
|
t.Errorf("the controller is granted %s, which the installer's first user list does not carry", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, p := range []Principal{{Kind: KindPerson, Module: "jochen", Invokes: []string{"*"}},
|
||||||
|
{Kind: KindNodeTools, Node: "shanks", Module: RuntimeModule}} {
|
||||||
|
perms, err := PermissionsFor(p)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if MayPublish(perms, "mesh.mod.ledger.call.who.controller") || MayPublish(perms, "mesh.seat.mesh-controller.call.status.controller") {
|
||||||
|
t.Errorf("%s may call in the controller's name", p.Username())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -10,6 +10,8 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"golang.org/x/crypto/bcrypt"
|
"golang.org/x/crypto/bcrypt"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/beside"
|
||||||
)
|
)
|
||||||
|
|
||||||
// **The first user list the installer carries must be the one the controller would compose.**
|
// **The first user list the installer carries must be the one the controller would compose.**
|
||||||
@@ -76,13 +78,14 @@ func theCarriedAccounts(t *testing.T) string {
|
|||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// theTemplate is the installer's bundle, as resources.
|
// theTemplate is the installer's bundle, as resources: the node-engine's, as a merge check clones it at
|
||||||
|
// the commit the mesh runs, or as captured in testdata/beside (internal/beside, novox/hq issue 432).
|
||||||
func theTemplate(t *testing.T) []map[string]any {
|
func theTemplate(t *testing.T) []map[string]any {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
path := filepath.Join("..", "..", "..", "mesh-host", "examples", "foundation-first-node-nats.lock")
|
path := filepath.Join(beside.Dir(t, "mesh-host"), "examples", "foundation-first-node-nats.lock")
|
||||||
raw, err := os.ReadFile(path)
|
raw, err := os.ReadFile(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Skipf("the host's checkout is not beside this one: %v", err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
// The template is JSON with line comments, which is how every one of them is written.
|
// The template is JSON with line comments, which is how every one of them is written.
|
||||||
var lines []string
|
var lines []string
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user