Compare commits
73
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fe2e5e91b5 | ||
|
|
51db0b5273 | ||
|
|
5c4fa43f8b | ||
|
|
c494ed03a7 | ||
|
|
02c61b983c | ||
|
|
3b6b54a501 | ||
|
|
83ce19b9c0 | ||
|
|
7758301444 | ||
|
|
094d3d5bc6 | ||
|
|
e7bcc107c8 | ||
|
|
ebca7816bf | ||
|
|
9bed7d6398 | ||
|
|
0a2e58c070 | ||
|
|
1747e33923 | ||
|
|
9b6acf0ef5 | ||
|
|
45b4ae92bc | ||
|
|
4f5fab81fe | ||
|
|
4632b6a508 | ||
|
|
9d6a12eb53 | ||
|
|
984d85865d | ||
|
|
ed45cc6415 | ||
|
|
ef551fdfb6 | ||
|
|
7493bd8f18 | ||
|
|
8305e4e3d1 | ||
|
|
fbc7bc976b | ||
|
|
f510b46319 | ||
|
|
926fde2fda | ||
|
|
d5cf3b42fe | ||
|
|
47c7877e8d | ||
|
|
68fbd99e89 | ||
|
|
9a37c143e4 | ||
|
|
eb72075104 | ||
|
|
d6b867a87a | ||
|
|
1dc9961c43 | ||
|
|
11ffab887b | ||
|
|
cfbbad8209 | ||
|
|
d2e9dc6159 | ||
|
|
5557a01f06 | ||
|
|
5bddb16514 | ||
|
|
671e350f56 | ||
|
|
f83fcdc15f | ||
|
|
585caa4371 | ||
|
|
f8d08b0637 | ||
|
|
dc62fd0075 | ||
|
|
525b2c1a11 | ||
|
|
c11222026a | ||
|
|
360c318e85 | ||
|
|
f008fab9d8 | ||
|
|
15a9919df5 | ||
|
|
1390836b73 | ||
|
|
dac7e5f7f6 | ||
|
|
c06a2cd972 | ||
|
|
d5f6b67b94 | ||
|
|
12d4025d30 | ||
|
|
1ca4d8ce60 | ||
|
|
a4f93b52a8 | ||
|
|
a2a671adb7 | ||
|
|
412f11b079 | ||
|
|
5d4eb974ab | ||
|
|
6805e2bcb3 | ||
|
|
c640341c50 | ||
|
|
690b75f659 | ||
|
|
d52de218c8 | ||
|
|
6188e6b1da | ||
|
|
798738cc12 | ||
|
|
babfef4f3a | ||
|
|
96b0966ad8 | ||
|
|
4d30b5de13 | ||
|
|
c97942d591 | ||
|
|
9c69b9da17 | ||
|
|
16362e1bbd | ||
|
|
3496b58f66 | ||
|
|
04fcce2fcc |
@@ -99,16 +99,22 @@ proxy-image:
|
||||
@echo
|
||||
@docker image inspect $(PROXY_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||
|
||||
# The whole gate. Raises a database, runs everything against it, and takes it down again --
|
||||
# including when the tests fail, which is why the teardown is not conditional.
|
||||
# The whole check. Raises a database, runs the repository's own check against it -- merge-check.sh,
|
||||
# the very script `mesh/repo-check` runs -- and takes the database down again, including when the
|
||||
# check fails, which is why the teardown is not conditional and the script's exit status is the
|
||||
# target's.
|
||||
#
|
||||
# **Packages in parallel, under the race detector, each test on a bus of its own** (internal/testbus).
|
||||
# It was one package at a time against one shared bus, because the live tests assert, read and remove
|
||||
# the mesh's own objects by their fixed names, and two packages at once deleted what the other read; the
|
||||
# suite was red run as Go runs it and read as noise. A bus per test, of the release the mesh runs, made
|
||||
# it the same in any order. The timeout bounds a hang to a failure with a stack, never a stalled gate.
|
||||
check: fmt vet postgres
|
||||
@go test -race -timeout 15m ./... ; status=$$? ; $(MAKE) postgres-stop ; exit $$status
|
||||
# **It calls the script rather than restating it** (novox/hq issue 431): this target used to list its
|
||||
# own steps, its formatting step walked vendor/ where the script's does not, and it failed on
|
||||
# third-party code no change could fix -- so a developer's check and the gate disagreed, and the steps
|
||||
# after formatting never ran through it. The script is the one list of steps; this target only gives it
|
||||
# a database (MESH_TEST_POSTGRES, exported above).
|
||||
#
|
||||
# **Packages in parallel, under the race detector, each test on a bus of its own** (internal/testbus),
|
||||
# as the script runs them: a bus per test, of the release the mesh runs, makes the suite the same in any
|
||||
# order, and the timeout bounds a hang to a failure with a stack, never a stalled gate.
|
||||
check: postgres
|
||||
@sh merge-check.sh ; status=$$? ; docker rm -f $(PG_CONTAINER) >/dev/null 2>&1 || true ; exit $$status
|
||||
|
||||
# Without a database the store's tests skip rather than fail, so this is the honest subset and not
|
||||
# the gate.
|
||||
@@ -118,8 +124,10 @@ test:
|
||||
vet:
|
||||
go vet ./...
|
||||
|
||||
# Quick steps for a developer, not part of `check`. The directories gofmt reads must be the ones
|
||||
# merge-check.sh lists, never `.`, which walks vendor/ (novox/hq issue 431).
|
||||
fmt:
|
||||
@unformatted=$$(gofmt -l . 2>/dev/null) ; \
|
||||
@unformatted=$$(gofmt -l cmd internal examples) ; \
|
||||
if [ -n "$$unformatted" ] ; then echo "not gofmt'd:" ; echo "$$unformatted" ; exit 1 ; fi
|
||||
|
||||
postgres:
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A refusal the bus said once is counted once** (novox/hq issue 402). The bus refused the controller
|
||||
// one publish at 18:52:59 UTC on 2026-10-10 and never again, yet S9 looked at the one remembered
|
||||
// advisory every half minute for the hour it is kept, and the condition said "observed 15 time(s), last
|
||||
// 13s ago" with a refusal in its evidence every 30 seconds: two of us read it as a refusal going on and
|
||||
// went looking for a missing grant. The condition counts what the bus said, with when it last said it,
|
||||
// never how often the controller looked.
|
||||
func TestARefusalSaidOnceIsCountedOnceHoweverOftenItIsLookedAt(t *testing.T) {
|
||||
refused := time.Date(2026, 10, 10, 18, 52, 59, 0, time.UTC)
|
||||
now := refused.Add(10 * time.Second)
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
||||
t.Cleanup(func() { k.Close(t.Context()) })
|
||||
|
||||
// The words the live condition bus.controller.refused carried on 2026-10-10.
|
||||
said := "the bus refused the controller: nats: permissions violation: Permissions Violation for " +
|
||||
`Publish to "mesh.seat.mesh-delivery.tool.times"`
|
||||
advisory := link.Advisory{Kind: link.AdvisoryRefused, ID: "controller", Said: said,
|
||||
First: refused, Last: refused, Count: 1}
|
||||
look := func() conditions.Condition {
|
||||
t.Helper()
|
||||
f := &signalFacts{now: now, host: "novox", advisories: []link.Advisory{advisory}}
|
||||
if err := k.Reconcile(t.Context(), "S9", watchAdvisories(f)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, found, err := conditions.ReadOne(t.Context(), store, "bus.controller.refused")
|
||||
if err != nil || !found {
|
||||
t.Fatalf("bus.controller.refused: found %v, %v", found, err)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
var c conditions.Condition
|
||||
for range 15 {
|
||||
c = look()
|
||||
now = now.Add(30 * time.Second)
|
||||
}
|
||||
if c.Observations != 1 || len(c.Evidence) != 1 {
|
||||
t.Fatalf("one refusal, looked at 15 times, reads as observed %d time(s) with %d evidence lines: %+v",
|
||||
c.Observations, len(c.Evidence), c.Evidence)
|
||||
}
|
||||
if !c.LastObserved.Equal(refused) || !c.Evidence[0].At.Equal(refused) {
|
||||
t.Fatalf("last observed %s, evidence at %s: the refusal was at %s", c.LastObserved, c.Evidence[0].At, refused)
|
||||
}
|
||||
|
||||
// The bus refuses it three times more between two looks: the condition counts four refusals, says
|
||||
// the newest, and adds one line of evidence for the look that saw them.
|
||||
again := now.Add(-5 * time.Second)
|
||||
advisory.Last, advisory.Count = again, 4
|
||||
c = look()
|
||||
if c.Observations != 4 || len(c.Evidence) != 2 || !c.LastObserved.Equal(again) || !c.Evidence[0].At.Equal(again) {
|
||||
t.Fatalf("four refusals read as observed %d time(s), last %s, evidence %+v", c.Observations, c.LastObserved, c.Evidence)
|
||||
}
|
||||
if !strings.Contains(c.Summary, "(4 times since 18:52 UTC)") {
|
||||
t.Fatalf("summary %q", c.Summary)
|
||||
}
|
||||
now = now.Add(30 * time.Second)
|
||||
if c = look(); c.Observations != 4 || len(c.Evidence) != 2 {
|
||||
t.Fatalf("a look with nothing new counted: observed %d time(s), evidence %+v", c.Observations, c.Evidence)
|
||||
}
|
||||
}
|
||||
|
||||
// **A condition that looks at the present still counts its looks** (novox/hq issue 402 changes only
|
||||
// what reads a record): a machine silent for three looks was observed three times, and says so.
|
||||
func TestAConditionOfThePresentCountsItsLooks(t *testing.T) {
|
||||
now := time.Date(2026, 10, 10, 19, 0, 0, 0, time.UTC)
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
||||
t.Cleanup(func() { k.Close(t.Context()) })
|
||||
var c conditions.Condition
|
||||
for range 3 {
|
||||
var err error
|
||||
if c, err = k.Observe(t.Context(), conditions.Observation{Scope: conditions.ScopeMachine, ID: "ace",
|
||||
Kind: "silent", Machine: "ace", Severity: conditions.Warning, Summary: "ace has not been heard from",
|
||||
Source: "S1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now = now.Add(30 * time.Second)
|
||||
}
|
||||
if c.Observations != 3 || len(c.Evidence) != 3 || !c.LastObserved.Equal(now.Add(-30*time.Second)) {
|
||||
t.Fatalf("observed %d time(s), %d evidence lines, last %s", c.Observations, len(c.Evidence), c.LastObserved)
|
||||
}
|
||||
}
|
||||
|
||||
// **One key, one watcher** (novox/hq issue 440). A consumer's max-deliveries condition is said from
|
||||
// DEAD_LETTERS while it holds a message the consumer gave up on (issue 330). A max-deliveries advisory
|
||||
// without a token names the same key; read beside it, each look added an observation and a line of
|
||||
// evidence through the dead-letter half, and the two summaries overwrote each other on every look. The
|
||||
// dead-letter row alone says that key, and counts the messages given up on, never the looks.
|
||||
func TestAHeldDeadLetterIsCountedByWhatWasGivenUpNotByTheLooks(t *testing.T) {
|
||||
gaveUp := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
|
||||
now := gaveUp.Add(20 * time.Second)
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
||||
t.Cleanup(func() { k.Close(t.Context()) })
|
||||
|
||||
const key = "bus.EVENTS.media_sonarr.max-deliveries"
|
||||
held := map[string]int{"EVENTS.media_sonarr": 1}
|
||||
newest := map[string]time.Time{"EVENTS.media_sonarr": gaveUp}
|
||||
advisory := link.Advisory{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.media_sonarr", Stream: "EVENTS",
|
||||
Consumer: "media_sonarr", Said: "sonarr on media handed message 7 over 5 times and gave up on it",
|
||||
First: gaveUp, Last: gaveUp, Count: 1}
|
||||
look := func() conditions.Condition {
|
||||
t.Helper()
|
||||
f := &signalFacts{now: now, host: "novox", advisories: []link.Advisory{advisory},
|
||||
deadLetters: held, deadLettersNewest: newest}
|
||||
if err := k.Reconcile(t.Context(), "S9", watchAdvisories(f)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, found, err := conditions.ReadOne(t.Context(), store, key)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("%s: found %v, %v", key, found, err)
|
||||
}
|
||||
if !strings.Contains(c.Summary, "dead-letters verb") {
|
||||
t.Fatalf("the summary is not the dead-letter row's: %q", c.Summary)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
var c conditions.Condition
|
||||
for range 5 {
|
||||
c = look()
|
||||
now = now.Add(30 * time.Second)
|
||||
}
|
||||
if c.Observations != 1 || len(c.Evidence) != 1 || !c.LastObserved.Equal(gaveUp) {
|
||||
t.Fatalf("one message given up on, looked at five times, reads as observed %d time(s), last %s, "+
|
||||
"with %d evidence lines: %+v", c.Observations, c.LastObserved, len(c.Evidence), c.Evidence)
|
||||
}
|
||||
|
||||
// The consumer gives up on a second message between two looks: two given up on, one more line.
|
||||
again := now.Add(-10 * time.Second)
|
||||
held["EVENTS.media_sonarr"], newest["EVENTS.media_sonarr"] = 2, again
|
||||
c = look()
|
||||
if c.Observations != 2 || len(c.Evidence) != 2 || !c.LastObserved.Equal(again) {
|
||||
t.Fatalf("two given up on read as observed %d time(s), last %s, evidence %+v", c.Observations,
|
||||
c.LastObserved, c.Evidence)
|
||||
}
|
||||
now = now.Add(30 * time.Second)
|
||||
if c = look(); c.Observations != 2 || len(c.Evidence) != 2 {
|
||||
t.Fatalf("a look with nothing new counted: observed %d time(s), evidence %+v", c.Observations, c.Evidence)
|
||||
}
|
||||
}
|
||||
|
||||
// The advisory still says the max-deliveries it alone knows: a message given up on that could not be
|
||||
// kept, under a key of its own (issue 330), which issue 440's change leaves as it was.
|
||||
func TestAMessageThatCouldNotBeKeptIsStillSaidFromTheAdvisory(t *testing.T) {
|
||||
at := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
|
||||
f := &signalFacts{now: at, host: "novox", advisories: []link.Advisory{{Kind: link.AdvisoryMaxDeliveries,
|
||||
ID: "EVENTS.media_sonarr", Stream: "EVENTS", Consumer: "media_sonarr", Token: link.AdvisoryNotKept,
|
||||
Said: "sonarr on media gave up on message 7, and it could not be kept", First: at, Last: at, Count: 1}}}
|
||||
said := watchAdvisories(f)
|
||||
if len(said) != 1 || said[0].Key() != "bus.EVENTS.media_sonarr.not-kept" {
|
||||
t.Fatalf("%+v", said)
|
||||
}
|
||||
}
|
||||
|
||||
// A max-deliveries advisory without a token names a consumer's dead-letter key, which only DEAD_LETTERS
|
||||
// says (novox/hq issues 330 and 440): with nothing held, the advisory alone raises nothing.
|
||||
func TestAMaxDeliveriesAdvisoryAloneRaisesNothing(t *testing.T) {
|
||||
at := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
|
||||
f := &signalFacts{now: at, host: "novox", advisories: []link.Advisory{{Kind: link.AdvisoryMaxDeliveries,
|
||||
ID: "EVENTS.media_sonarr", Stream: "EVENTS", Consumer: "media_sonarr",
|
||||
Said: "sonarr on media handed message 7 over 5 times and gave up on it", First: at, Last: at, Count: 1}}}
|
||||
if said := watchAdvisories(f); len(said) != 0 {
|
||||
t.Fatalf("said %+v", said)
|
||||
}
|
||||
}
|
||||
+169
-60
@@ -38,8 +38,9 @@ import (
|
||||
// by that one.
|
||||
//
|
||||
// A merge on the controller's own path (a module whose walk waits for nobody's word) never shares a batch with
|
||||
// one that waits for mesh-delivery's: each kind has a batch of its own, so no catalogue delivery skips its turn
|
||||
// behind a controller merge (decided during the build, 2026-10-10).
|
||||
// one that waits for mesh-delivery's: each kind has a batch of its own, an own-path batch is cut first and folds
|
||||
// no waiting catalogue walk, and a catalogue walk let go while another walk runs starts once it ended — so no
|
||||
// catalogue batch's walk starts without the word (decided during the build, 2026-10-10).
|
||||
//
|
||||
// The batch, its merges and their times are in the store (batched_merge, and the batch's own plan record in
|
||||
// the state `assembling` or `queued`), so a restarted controller resumes the window where it stood.
|
||||
@@ -181,7 +182,12 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
|
||||
if _, known, err := inv.MergeOf(ctx, repository, m.Commit); err != nil || known {
|
||||
return notNow(err)
|
||||
}
|
||||
event, err := json.Marshal(m)
|
||||
var moves []string
|
||||
for _, e := range touches {
|
||||
moves = append(moves, e.Manifest.Module)
|
||||
}
|
||||
sort.Strings(moves)
|
||||
event, err := json.Marshal(keptMerge{SourceMoved: m, Moves: moves})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -193,7 +199,7 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
|
||||
// word** (decided during the build, 2026-10-10): batched together, the catalogue's deliveries would start
|
||||
// with the controller's and skip their turn. Each kind has a batch of its own.
|
||||
own := ownPath(touches)
|
||||
if p, ok, err := answeredByALaterMerge(ctx, inv, heard, touches, own); err != nil {
|
||||
if p, ok, err := answeredByALaterMerge(ctx, inv, heard, touches, own, deliverySeatHeld(entries)); err != nil {
|
||||
return notNow(err)
|
||||
} else if ok {
|
||||
heard.Plan = p.ID
|
||||
@@ -251,7 +257,7 @@ func owedLate(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved,
|
||||
// later merge, which contains it. A failed or stopped walk answers nothing more, and a walk folded into
|
||||
// another is followed to that one. False when none does: the merge joins the next batch.
|
||||
func answeredByALaterMerge(ctx context.Context, inv *inventory.Inventory, heard inventory.BatchedMerge,
|
||||
moves []inventory.Entry, own bool) (inventory.Plan, bool, error) {
|
||||
moves []inventory.Entry, own, held bool) (inventory.Plan, bool, error) {
|
||||
later, err := inv.LaterMergesOf(ctx, heard.Repository, heard.Branch, heard.Merged)
|
||||
if err != nil || len(later) == 0 {
|
||||
return inventory.Plan{}, false, err
|
||||
@@ -268,7 +274,11 @@ func answeredByALaterMerge(ctx context.Context, inv *inventory.Inventory, heard
|
||||
return p, true, nil
|
||||
}
|
||||
case p.Open() || p.State == inventory.PlanDone:
|
||||
if buildsEvery(p, moves) {
|
||||
// A walk that waited for nobody's word is not a catalogue merge's to be answered by while the delivery
|
||||
// seat has a holder: its delivery would skip its turn (decided during the build, 2026-10-10). With no
|
||||
// holder on record nothing waits, and any walk that built it answers.
|
||||
waited := p.Delivery != nil && p.Delivery.Awaits != ""
|
||||
if buildsEvery(p, moves) && (own || waited || !held) {
|
||||
return p, true, nil
|
||||
}
|
||||
}
|
||||
@@ -431,7 +441,7 @@ func carriedOf(merges []inventory.BatchedMerge) ([]inventory.PlanCommit, []inven
|
||||
if repo == "" {
|
||||
repo = m.Repository
|
||||
}
|
||||
n := inventory.PlanMerge{Repository: repo, Commit: m.Commit}
|
||||
n := namedOf(m, repo)
|
||||
if l := latest[k]; l.Commit != m.Commit {
|
||||
n.Carried = l.Commit
|
||||
}
|
||||
@@ -443,15 +453,35 @@ func carriedOf(merges []inventory.BatchedMerge) ([]inventory.PlanCommit, []inven
|
||||
return commits, named
|
||||
}
|
||||
|
||||
// keptMerge is a merge as the record keeps it: the forge's announcement, and the modules it moved when it was
|
||||
// heard. The announcement reads back as a SourceMoved alone, which ignores the rest.
|
||||
type keptMerge struct {
|
||||
link.SourceMoved
|
||||
Moves []string `json:"moves,omitempty"`
|
||||
}
|
||||
|
||||
// announcedOf is a kept merge as the forge announced it, and what it moved; empty where the record says none.
|
||||
func announcedOf(m inventory.BatchedMerge) keptMerge {
|
||||
var k keptMerge
|
||||
_ = json.Unmarshal(m.Event, &k)
|
||||
return k
|
||||
}
|
||||
|
||||
// spelledOf is a kept merge's repository as the forge spelled it; empty when its announcement does not say.
|
||||
func spelledOf(m inventory.BatchedMerge) string {
|
||||
var e link.SourceMoved
|
||||
if json.Unmarshal(m.Event, &e) == nil && e.Owner != "" {
|
||||
if e := announcedOf(m); e.Owner != "" {
|
||||
return e.Owner + "/" + e.Repo
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// namedOf is one merge as a walk or batch names it: its commit, and its pull request and moves as announced.
|
||||
func namedOf(m inventory.BatchedMerge, repository string) inventory.PlanMerge {
|
||||
k := announcedOf(m)
|
||||
return inventory.PlanMerge{Repository: repository, Commit: m.Commit, Number: k.Number, Title: k.Title, Moves: k.Moves,
|
||||
Merged: m.Merged, Heard: m.Heard}
|
||||
}
|
||||
|
||||
// laterMerge says a was merged after b: by the forge's merge time, then by when each was heard.
|
||||
func laterMerge(a, b inventory.BatchedMerge) bool {
|
||||
if !a.Merged.Equal(b.Merged) {
|
||||
@@ -480,10 +510,11 @@ func nameMerges(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan
|
||||
// The walk's own commits stand: a merge heard late is carried by the one of its repository's branch.
|
||||
var named []inventory.PlanMerge
|
||||
for _, m := range merges {
|
||||
n := inventory.PlanMerge{Repository: m.Repository, Commit: m.Commit}
|
||||
repo := m.Repository
|
||||
if e := spelledOf(m); e != "" {
|
||||
n.Repository = e
|
||||
repo = e
|
||||
}
|
||||
n := namedOf(m, repo)
|
||||
if c := p.CommitOn(m.Repository, m.Branch); c != "" && c != m.Commit {
|
||||
n.Carried = c
|
||||
}
|
||||
@@ -579,18 +610,18 @@ func cutBatchesHeld(ctx context.Context, open *stores, now time.Time) error {
|
||||
if err := keepAll(""); err != nil {
|
||||
return err
|
||||
}
|
||||
// The oldest batch whose window closed is cut; one of each kind at most is open (theOpenBatch).
|
||||
// The oldest batch whose window closed is cut, an own-path one before a catalogue one (it goes first, and
|
||||
// the catalogue batch queues behind it rather than being cut and overtaken); one of each kind at most is
|
||||
// open (theOpenBatch).
|
||||
sort.SliceStable(batches, func(i, j int) bool { return batches[i].OwnPath() && !batches[j].OwnPath() })
|
||||
for i := range batches {
|
||||
b := &batches[i]
|
||||
if !windowClosed(*b, now) {
|
||||
continue
|
||||
}
|
||||
if b.OwnPath() {
|
||||
// A walk on the controller's own path folds no walk waiting for its word: those merges are batched
|
||||
// again, behind it (decided during the build, 2026-10-10).
|
||||
if err := rebatchWaiting(ctx, inv, waiting, b.ID, now); err != nil {
|
||||
return err
|
||||
}
|
||||
// A walk on the controller's own path folds no walk waiting for its word (decided during the build,
|
||||
// 2026-10-10): that walk keeps waiting beside it, and starts once this one ended (advanceOnce).
|
||||
waiting = nil
|
||||
}
|
||||
if err := cutBatch(ctx, open, b, waiting, now); err != nil {
|
||||
@@ -608,46 +639,6 @@ func cutBatchesHeld(ctx context.Context, open *stores, now time.Time) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// rebatchWaiting puts the merges of the walks waiting for their word into the open batch of their kind — a new
|
||||
// one when none is — queued behind the walk about to be cut, and closes the walks as taken over by that batch:
|
||||
// the batch keeps its id when it is cut, so the delivery's owner follows them to the walk that answers them.
|
||||
func rebatchWaiting(ctx context.Context, inv *inventory.Inventory, waiting []inventory.Plan, behind string, now time.Time) error {
|
||||
for i := range waiting {
|
||||
w := waiting[i]
|
||||
merges, err := inv.MergesOf(ctx, w.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(merges) == 0 {
|
||||
continue // nothing of the record's: left waiting
|
||||
}
|
||||
batch, err := theOpenBatch(ctx, inv, merges[0], now, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, m := range merges {
|
||||
if err := inv.AnswerMerge(ctx, m.Repository, m.Commit, batch.ID, m.Alone); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := keepBatch(ctx, inv, &batch, now, behind); err != nil {
|
||||
return err
|
||||
}
|
||||
w.State = inventory.PlanSuperseded
|
||||
if w.Delivery == nil {
|
||||
w.Delivery = &inventory.PlanDelivery{}
|
||||
}
|
||||
w.Delivery.TakenOverBy = batch.ID
|
||||
w.Note = fmt.Sprintf("superseded at tier %d by %s before it started: a walk on the controller's own path "+
|
||||
"(%s) goes first, and that batch answers its merges after it", w.Tier, batch.ID, behind)
|
||||
if err := inv.SavePlan(ctx, &w); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf(" %s is %s\n", w.ID, w.Note)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// cutBatch makes a closed batch one walk, folding in the walks that wait for their word: it keeps its id, and
|
||||
// the folded walks name it as the walk that took them over.
|
||||
func cutBatch(ctx context.Context, open *stores, batch *inventory.Plan, waiting []inventory.Plan, now time.Time) error {
|
||||
@@ -754,6 +745,9 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
|
||||
}
|
||||
}
|
||||
}
|
||||
// **Whether it waits for its delivery's word** is read from what its merges moved (novox/hq ADR 0239), never
|
||||
// from what a folded walk carried along, which holds dependents too.
|
||||
awaits := awaitsFor(entries, moved)
|
||||
held := map[string]bool{}
|
||||
for _, e := range entries {
|
||||
held[e.Manifest.Module] = true
|
||||
@@ -773,14 +767,14 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
|
||||
plan.Commits = commits
|
||||
newest := newestCommit(commits)
|
||||
plan.Repository, plan.Branch, plan.Commit, plan.Merged = newest.Repository, newest.Branch, newest.Commit, newest.Merged
|
||||
// **Whether it waits for its delivery's word** (novox/hq ADR 0239): while the mesh-delivery seat has a
|
||||
// holder on record, a walk that moves no module on the controller's own path is opened and waits.
|
||||
plan.Delivery = awaitsFor(entries, moved)
|
||||
plan.Delivery = awaits
|
||||
if plan.Delivery == nil {
|
||||
plan.Delivery = &inventory.PlanDelivery{}
|
||||
}
|
||||
plan.Delivery.Merges = named
|
||||
plan.Delivery.Alone = batch.Delivery != nil && batch.Delivery.Alone
|
||||
// **Its moments and its class** (novox/hq ADR 0282 decision 6): measured, never acted on.
|
||||
plan.Times = walkTimesAtCut(*batch, plan, entries, now)
|
||||
if len(moved) == 0 {
|
||||
plan.State = inventory.PlanDone
|
||||
plan.Tiers = [][]string{}
|
||||
@@ -829,6 +823,50 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
|
||||
return nil
|
||||
}
|
||||
|
||||
// walkTimesAtCut is a walk's own moments as it is cut (novox/hq ADR 0282 decision 6): when its batch's window
|
||||
// closed — no merge for the window's length, or its maximum, whichever came first — when it was cut, and its
|
||||
// class. A merge walked alone had no window.
|
||||
func walkTimesAtCut(batch, walk inventory.Plan, entries []inventory.Entry, now time.Time) *inventory.PlanTimes {
|
||||
cut := now
|
||||
t := &inventory.PlanTimes{Cut: &cut, Class: classOf(walk, entries)}
|
||||
if batch.Delivery != nil && batch.Delivery.Batch != nil {
|
||||
w := batch.Delivery.Batch
|
||||
closed := w.ClosesAt
|
||||
if !w.AtMost.IsZero() && w.AtMost.Before(closed) {
|
||||
closed = w.AtMost
|
||||
}
|
||||
if !closed.IsZero() {
|
||||
if closed.After(now) {
|
||||
closed = now
|
||||
}
|
||||
t.WindowClosed = &closed
|
||||
}
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// resolverSeat is the seat of the mesh's resolver: a module claiming it is a core module (ADR 0282 decision 1).
|
||||
const resolverSeat = "mesh-dns-resolver"
|
||||
|
||||
// classOf is a walk's class (novox/hq ADR 0282 decision 1): core when it walks a module on the controller's own
|
||||
// path or one holding the mesh's resolver, leaf otherwise.
|
||||
func classOf(walk inventory.Plan, entries []inventory.Entry) string {
|
||||
resolvers := map[string]bool{}
|
||||
for _, e := range entries {
|
||||
for _, c := range e.Manifest.Claims {
|
||||
if c.Name == resolverSeat {
|
||||
resolvers[e.Manifest.Module] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
for m := range walk.Modules {
|
||||
if _, own := onTheControllersPath[m]; own || resolvers[m] {
|
||||
return inventory.ClassCore
|
||||
}
|
||||
}
|
||||
return inventory.ClassLeaf
|
||||
}
|
||||
|
||||
// combinedMerges is a batch's merges as one merge per repository's branch: the latest, with every file the
|
||||
// merges of it changed and said to be a module's — on a linear trunk the latest contains the others. A file is
|
||||
// removed when the last merge of the batch that changed it removed it. merges are in the order they were made.
|
||||
@@ -1088,6 +1126,77 @@ func groupedWords(b inventory.Plan) string {
|
||||
return strings.Join(out, ", ")
|
||||
}
|
||||
|
||||
// batchLines is a batch's grouped list as `plans` prints it under its line (asked by the operator, 2026-10-10):
|
||||
// one line per repository, its pull request and title, the earlier merges it answers, and what its merges move.
|
||||
func batchLines(b inventory.Plan) []string {
|
||||
var out []string
|
||||
for _, c := range b.Carried() {
|
||||
var head *inventory.PlanMerge
|
||||
var answers []string
|
||||
moves := map[string]bool{}
|
||||
if b.Delivery != nil {
|
||||
for i := range b.Delivery.Merges {
|
||||
m := &b.Delivery.Merges[i]
|
||||
if !strings.EqualFold(m.Repository, c.Repository) {
|
||||
continue
|
||||
}
|
||||
for _, n := range m.Moves {
|
||||
moves[n] = true
|
||||
}
|
||||
switch {
|
||||
case m.Commit == c.Commit:
|
||||
head = m
|
||||
case m.Carried == c.Commit:
|
||||
answers = append(answers, pullWords(*m))
|
||||
}
|
||||
}
|
||||
}
|
||||
line := repoName(c.Repository) + " " + short(c.Commit)
|
||||
if head != nil && head.Number > 0 {
|
||||
line = repoName(c.Repository) + " " + pullWords(*head)
|
||||
}
|
||||
if len(answers) > 0 {
|
||||
line += " (answers " + strings.Join(answers, ", ") + ")"
|
||||
}
|
||||
if len(moves) > 0 {
|
||||
line += " · " + strings.Join(sortedKeysOf(boolsToStrings(moves)), ", ")
|
||||
}
|
||||
out = append(out, line)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// pullWords is a merge as its pull request: "#175 a tap shows its outcome", the title cut at fifty runes; the
|
||||
// commit where the announcement named no pull request.
|
||||
func pullWords(m inventory.PlanMerge) string {
|
||||
if m.Number == 0 {
|
||||
return short(m.Commit)
|
||||
}
|
||||
s := fmt.Sprintf("#%d", m.Number)
|
||||
if t := cutTitle(m.Title, 50); t != "" {
|
||||
s += " " + t
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// cutTitle is a title cut at n runes, with an ellipsis where it was cut.
|
||||
func cutTitle(title string, n int) string {
|
||||
r := []rune(strings.TrimSpace(title))
|
||||
if len(r) <= n {
|
||||
return string(r)
|
||||
}
|
||||
return strings.TrimSpace(string(r[:n-1])) + "…"
|
||||
}
|
||||
|
||||
// boolsToStrings is a set's members, for sortedKeysOf.
|
||||
func boolsToStrings(set map[string]bool) map[string]string {
|
||||
out := make(map[string]string, len(set))
|
||||
for k := range set {
|
||||
out[k] = ""
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// secondsWords is a short wait as a person reads it.
|
||||
func secondsWords(d time.Duration) string {
|
||||
if d < 2*time.Minute {
|
||||
|
||||
@@ -136,7 +136,7 @@ func TestTwoMergesSecondsApartAreOneWalkAtTheLaterCommit(t *testing.T) {
|
||||
}
|
||||
want := []inventory.PlanMerge{{Repository: "novox/mesh-catalog", Commit: claude.Commit, Carried: dunst.Commit},
|
||||
{Repository: "novox/mesh-catalog", Commit: dunst.Commit}}
|
||||
if w.Delivery == nil || !slices.Equal(w.Delivery.Merges, want) {
|
||||
if w.Delivery == nil || !slices.EqualFunc(w.Delivery.Merges, want, sameMerge) {
|
||||
t.Fatalf("the walk answers %+v, want %+v", w.Delivery, want)
|
||||
}
|
||||
if len(*asked) != 2 || (*asked)[0][2] != "main" || (*asked)[1][2] != "main" {
|
||||
@@ -229,17 +229,28 @@ func TestTheAssemblingBatchIsShown(t *testing.T) {
|
||||
inventory.PlanSaved = func(p inventory.Plan) { said = append(said, p) }
|
||||
t.Cleanup(func() { inventory.PlanSaved = was })
|
||||
now := time.Now().UTC()
|
||||
hear(t, open, catalogueMerge("553b7191claude", "app", now.Add(-20*time.Second)), now.Add(-19*time.Second))
|
||||
hear(t, open, catalogueMerge("48bda475dunst", "notes", now.Add(-2*time.Second)), now.Add(-time.Second))
|
||||
claude := catalogueMerge("553b7191claude", "app", now.Add(-20*time.Second))
|
||||
claude.Number, claude.Title = 174, "claude-code: an agent proposes a section"
|
||||
dunst := catalogueMerge("48bda475dunst", "notes", now.Add(-2*time.Second))
|
||||
dunst.Number, dunst.Title = 175, "dunst: the font the operator chose"
|
||||
hear(t, open, claude, now.Add(-19*time.Second))
|
||||
hear(t, open, dunst, now.Add(-time.Second))
|
||||
hear(t, open, repoMerge("one", "a6bc0931one", now), now)
|
||||
out := captured(t, func() error { return plansCommand(t.Context(), nil) })
|
||||
first := strings.SplitN(out, "\n", 2)[0]
|
||||
lines := strings.Split(out, "\n")
|
||||
first := lines[0]
|
||||
for _, want := range []string{"assembling: ", " s left (at the latest ", "grouped: novox/mesh-catalog@48bda475 " +
|
||||
"(answers 553b7191), novox/one@a6bc0931; plan not yet calculated"} {
|
||||
if !strings.Contains(first, want) {
|
||||
t.Fatalf("plans' first line %q does not say %q", first, want)
|
||||
}
|
||||
}
|
||||
// Under it, one line per repository: the pull request, what it answers, what it moves.
|
||||
if len(lines) < 3 || strings.TrimSpace(lines[1]) != "mesh-catalog #175 dunst: the font the operator chose (answers "+
|
||||
"#174 claude-code: an agent proposes a section) · app, notes" ||
|
||||
strings.TrimSpace(lines[2]) != "one a6bc0931 · one" {
|
||||
t.Fatalf("the grouped list reads %q", lines[1:4])
|
||||
}
|
||||
if len(said) == 0 || said[len(said)-1].State != inventory.PlanAssembling || said[len(said)-1].Delivery.Batch == nil ||
|
||||
len(said[len(said)-1].Delivery.Merges) != 3 {
|
||||
t.Fatalf("the batch was not said as assembling with its merges: %+v", said)
|
||||
@@ -364,7 +375,7 @@ func TestALateMergeIsAnsweredByTheWalkOfTheLaterOne(t *testing.T) {
|
||||
hear(t, open, catalogueMerge("553b7191early", "notes", t0), t0.Add(15*time.Minute))
|
||||
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
||||
if got.State != inventory.PlanDone || len(got.Delivery.Merges) != 2 ||
|
||||
got.Delivery.Merges[0] != (inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: "553b7191early",
|
||||
!sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: "553b7191early",
|
||||
Carried: later.Commit}) {
|
||||
t.Fatalf("the late merge is not named by the walk that carried it: %+v", got.Delivery.Merges)
|
||||
}
|
||||
@@ -404,7 +415,7 @@ func TestAFailedWalkWalksItsEarlierMergesAlone(t *testing.T) {
|
||||
ws, _ = walks(t, open)
|
||||
alone := ws[0]
|
||||
if alone.ID == failed.ID || alone.Commit != middle.Commit || len(alone.Delivery.Merges) != 1 ||
|
||||
alone.Delivery.Merges[0] != (inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: middle.Commit}) {
|
||||
!sameMerge(alone.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: middle.Commit}) {
|
||||
t.Fatalf("the newest earlier merge was not walked alone on its own commit: %+v", alone)
|
||||
}
|
||||
if _, in := alone.Modules["app"]; !in || (*asked)[len(*asked)-1] != [3]string{"novox/mesh-catalog", "modules/app",
|
||||
@@ -425,7 +436,7 @@ func TestAFailedWalkWalksItsEarlierMergesAlone(t *testing.T) {
|
||||
t.Fatalf("the search went on after a merge was delivered: %+v", ws[0])
|
||||
}
|
||||
done, _ := open.inventory.PlanByID(ctx, alone.ID)
|
||||
if len(done.Delivery.Merges) != 2 || done.Delivery.Merges[0] != (inventory.PlanMerge{Repository: "novox/mesh-catalog",
|
||||
if len(done.Delivery.Merges) != 2 || !sameMerge(done.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog",
|
||||
Commit: oldest.Commit, Carried: middle.Commit}) {
|
||||
t.Fatalf("the oldest merge is not answered by the walk that delivered the one after it: %+v", done.Delivery.Merges)
|
||||
}
|
||||
@@ -565,7 +576,7 @@ func TestALateMergeOfAOneModuleRepositoryIsNamed(t *testing.T) {
|
||||
}
|
||||
hear(t, open, repoMerge("one", "c1", t0.Add(-60*time.Second)), t0.Add(15*time.Minute))
|
||||
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
||||
if len(got.Delivery.Merges) != 2 || got.Delivery.Merges[0] != (inventory.PlanMerge{Repository: "novox/one",
|
||||
if len(got.Delivery.Merges) != 2 || !sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/one",
|
||||
Commit: "c1", Carried: "c2"}) {
|
||||
t.Fatalf("the late merge was not named by the walk that carried it: %+v", got.Delivery.Merges)
|
||||
}
|
||||
@@ -589,10 +600,11 @@ func TestALateMergeOfAOneModuleRepositoryIsNamed(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// One walk at a time holds against the delivery's word too: a waiting walk is not let go while another started.
|
||||
func TestAWaitingWalkIsNotLetGoBesideAStartedOne(t *testing.T) {
|
||||
// One walk at a time holds against the delivery's word too: a waiting walk let go beside a started one takes the
|
||||
// word and starts once that one ended, asking nothing before.
|
||||
func TestAWalkLetGoBesideAStartedOneStartsOnceItEnded(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
asked := asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
@@ -603,9 +615,37 @@ func TestAWaitingWalkIsNotLetGoBesideAStartedOne(t *testing.T) {
|
||||
if err := open.inventory.SavePlan(ctx, &waiting); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := letGo(ctx, open.inventory, waiting.ID, catalogue.DeliverySeat, "its turn"); err == nil ||
|
||||
!strings.Contains(err.Error(), started[0].ID) {
|
||||
t.Fatalf("a waiting walk was let go beside %s: %v", started[0].ID, err)
|
||||
if _, err := letGo(ctx, open.inventory, waiting.ID, catalogue.DeliverySeat, "its turn"); err != nil {
|
||||
t.Fatalf("the word was refused beside %s: %v", started[0].ID, err)
|
||||
}
|
||||
before := len(*asked)
|
||||
advanceHeld(ctx, open)
|
||||
got, _ := open.inventory.PlanByID(ctx, waiting.ID)
|
||||
if len(*asked) != before || !strings.Contains(got.Note, "starts once "+started[0].ID) {
|
||||
t.Fatalf("a walk let go beside a started one asked (%d → %d) or does not say it waits: %q", before, len(*asked), got.Note)
|
||||
}
|
||||
// Read as a wait, an hour on: its note on the line, never LATE, and no tier of it late for S3.
|
||||
later := time.Now().Add(time.Hour)
|
||||
if line := planLine(got, later); !strings.Contains(line, "starts once "+started[0].ID) || strings.Contains(line, "LATE") {
|
||||
t.Fatalf("a deferred walk reads %q", line)
|
||||
}
|
||||
facts, _, err := gatherPlans(ctx, open.inventory, later, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range facts {
|
||||
if f.id == got.ID {
|
||||
t.Fatalf("a deferred walk is watched as a tier running late: %+v", f)
|
||||
}
|
||||
}
|
||||
done := started[0]
|
||||
done.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) != before+1 {
|
||||
t.Fatalf("the walk did not start once the started one ended: asked %v", *asked)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -660,7 +700,7 @@ func TestTheCatchUpKeepsWhatACutMadeHistory(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
||||
if len(got.Delivery.Merges) != 2 || got.Delivery.Merges[0] != (inventory.PlanMerge{Repository: "novox/one",
|
||||
if len(got.Delivery.Merges) != 2 || !sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/one",
|
||||
Commit: "c1", Carried: "c2"}) {
|
||||
t.Fatalf("the earlier merge the catch-up handed over is not named by the walk that carried it: %+v",
|
||||
got.Delivery.Merges)
|
||||
@@ -706,50 +746,29 @@ func TestAMergeOnTheControllersPathNeverSharesABatch(t *testing.T) {
|
||||
if !strings.Contains(batchWords(ownBatch, t0.Add(30*time.Second)), "own path") {
|
||||
t.Fatalf("the own-path batch does not say so: %q", batchWords(ownBatch, t0.Add(30*time.Second)))
|
||||
}
|
||||
// The catalogue batch, the older, is cut first and waits for its word; the controller's batch is cut next:
|
||||
// the waiting walk is batched again behind it, not folded into it.
|
||||
// Both windows closed, the controller's batch is cut first and starts; the catalogue batch queues behind it,
|
||||
// is cut when the controller's walk ended, and waits for its word with both merges.
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 || !ws[0].Waiting() {
|
||||
t.Fatalf("the catalogue batch was not cut into a waiting walk: %+v", ws)
|
||||
}
|
||||
catalogueWalk := ws[0]
|
||||
cutAt(t, open, t0.Add(2*time.Minute+5*time.Second))
|
||||
ws, bs = walks(t, open)
|
||||
var ownWalk inventory.Plan
|
||||
for _, w := range ws {
|
||||
if w.Open() {
|
||||
ownWalk = w
|
||||
}
|
||||
}
|
||||
if ownWalk.ID == "" || ownWalk.Waiting() || ownWalk.CommitOf("novox/mesh-controller") != "k1" {
|
||||
t.Fatalf("the controller's batch was not cut into a started walk: %+v", ws)
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || ws[0].Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k1" {
|
||||
t.Fatalf("the controller's batch was not cut first into a started walk: %+v", ws)
|
||||
}
|
||||
ownWalk := ws[0]
|
||||
for _, m := range []string{"app", "notes"} {
|
||||
if _, in := ownWalk.Modules[m]; in {
|
||||
t.Fatalf("the controller's walk builds %s, a catalogue module: it skipped its turn", m)
|
||||
}
|
||||
}
|
||||
for _, a := range *asked {
|
||||
if a[1] == "modules/app" || a[1] == "modules/notes" {
|
||||
t.Fatalf("a catalogue module was asked without the word: %v", *asked)
|
||||
}
|
||||
if len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != ownWalk.ID || bs[0].OwnPath() {
|
||||
t.Fatalf("the catalogue batch does not queue behind the controller's walk: %+v", bs)
|
||||
}
|
||||
folded, _ := open.inventory.PlanByID(ctx, catalogueWalk.ID)
|
||||
if folded.State != inventory.PlanSuperseded || len(bs) != 1 || folded.Delivery.TakenOverBy != bs[0].ID ||
|
||||
bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != ownWalk.ID || bs[0].OwnPath() ||
|
||||
len(bs[0].Delivery.Merges) != 2 {
|
||||
t.Fatalf("the waiting catalogue walk is %s (taken over by %q); batches %+v", folded.State,
|
||||
folded.Delivery.TakenOverBy, bs)
|
||||
}
|
||||
// The controller's walk done, the catalogue batch is cut and waits for its word with both merges.
|
||||
ownWalk.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &ownWalk); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(3*time.Minute))
|
||||
ws, bs = walks(t, open)
|
||||
if len(bs) != 0 || ws[0].ID != folded.Delivery.TakenOverBy || !ws[0].Waiting() || len(ws[0].Delivery.Merges) != 2 {
|
||||
if len(bs) != 0 || ws[0].ID != catalogueBatch.ID || !ws[0].Waiting() || len(ws[0].Delivery.Merges) != 2 {
|
||||
t.Fatalf("the catalogue batch was not cut into a waiting walk once the controller's ended: %+v %+v", ws, bs)
|
||||
}
|
||||
for _, m := range []string{"app", "notes"} {
|
||||
@@ -757,4 +776,127 @@ func TestAMergeOnTheControllersPathNeverSharesABatch(t *testing.T) {
|
||||
t.Fatalf("the catalogue walk does not build %s: %v", m, ws[0].Modules)
|
||||
}
|
||||
}
|
||||
for _, a := range *asked {
|
||||
if a[1] == "modules/app" || a[1] == "modules/notes" {
|
||||
t.Fatalf("a catalogue module was asked without the word: %v", *asked)
|
||||
}
|
||||
}
|
||||
|
||||
// A catalogue walk waiting for its word when an own-path batch is cut keeps waiting beside the own-path
|
||||
// walk, is not folded into it, and its word is taken meanwhile: it starts once the own-path walk ended.
|
||||
catalogueWalk := ws[0]
|
||||
hear(t, open, repoMerge("mesh-controller", "k2", t0.Add(4*time.Minute)), t0.Add(4*time.Minute))
|
||||
cutAt(t, open, t0.Add(6*time.Minute))
|
||||
ws, _ = walks(t, open)
|
||||
kept, _ := open.inventory.PlanByID(ctx, catalogueWalk.ID)
|
||||
if !kept.Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k2" || ws[0].Waiting() {
|
||||
t.Fatalf("the waiting catalogue walk was not kept waiting beside the controller's walk: %s %+v", kept.State, ws)
|
||||
}
|
||||
if _, in := ws[0].Modules["app"]; in {
|
||||
t.Fatalf("the controller's walk folded the waiting catalogue walk in: %v", ws[0].Modules)
|
||||
}
|
||||
if _, err := letGo(ctx, open.inventory, catalogueWalk.ID, catalogue.DeliverySeat, "its turn"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := len(*asked)
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) != before {
|
||||
t.Fatalf("the catalogue walk started beside the controller's: asked %v", (*asked)[before:])
|
||||
}
|
||||
own2 := ws[0]
|
||||
own2.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &own2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) < before+1 || (*asked)[before][1] != "modules/app" {
|
||||
t.Fatalf("the catalogue walk did not start once the controller's ended: %v", (*asked)[before:])
|
||||
}
|
||||
}
|
||||
|
||||
// `plans` names a walk by what it moves (asked by the operator, 2026-10-10): the repository's pull request and
|
||||
// title, the modules it moves and the machines running them, then the state words; a record naming no pull
|
||||
// request is named by its commit, and a title is cut at fifty runes.
|
||||
func TestAPlanLineNamesWhatItMoves(t *testing.T) {
|
||||
now := time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)
|
||||
p := inventory.Plan{ID: "plan-1", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1c1c1c1c1",
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"messenger", "telegram"}}, TierEntered: now.Add(-2 * time.Minute),
|
||||
Modules: map[string]*inventory.PlanModule{"messenger": {State: "asked"}, "telegram": {State: "asked"}},
|
||||
Commits: []inventory.PlanCommit{{Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1c1c1c1c1"}},
|
||||
Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{{Repository: "novox/mesh-catalog",
|
||||
Commit: "c1c1c1c1c1", Number: 175, Title: "a tap shows its outcome", Moves: []string{"telegram", "messenger"}}}}}
|
||||
running := func(module string) []string {
|
||||
if module == "messenger" || module == "telegram" {
|
||||
return []string{"novox"}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if got, want := planLineOn(p, now, pauseView{}, tierAtLeast, running),
|
||||
"mesh-catalog #175 a tap shows its outcome · messenger, telegram → novox · tier 1 of 1, building for 2m0s"; got != want {
|
||||
t.Fatalf("the line reads %q, want %q", got, want)
|
||||
}
|
||||
if got := planLine(p, now); !strings.HasPrefix(got, "mesh-catalog #175 a tap shows its outcome · messenger, telegram · tier") {
|
||||
t.Fatalf("without the machines the line reads %q", got)
|
||||
}
|
||||
old := p
|
||||
old.Commits, old.Delivery = nil, nil
|
||||
if got := planLine(old, now); !strings.HasPrefix(got, "mesh-catalog c1c1c1c1 · tier 1 of 1") {
|
||||
t.Fatalf("a record naming no pull request reads %q", got)
|
||||
}
|
||||
long := p
|
||||
long.Delivery.Merges[0].Title = strings.Repeat("abcdefghij", 6)
|
||||
if got := planHeadline(long, nil); !strings.Contains(got, "#175 "+strings.Repeat("abcdefghij", 4)+"abcdefghi…") {
|
||||
t.Fatalf("a long title is not cut at fifty runes: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// sameMerge compares what a walk names of a merge: its repository, commit and the commit carrying it.
|
||||
func sameMerge(a, b inventory.PlanMerge) bool {
|
||||
return a.Repository == b.Repository && a.Commit == b.Commit && a.Carried == b.Carried
|
||||
}
|
||||
|
||||
// A catalogue merge heard after a later merge of its branch was walked without the word (a merge that touched
|
||||
// the bus too, on the controller's own path) is not answered by that walk while the delivery seat has a holder:
|
||||
// its delivery would skip its turn. It joins the next catalogue batch.
|
||||
func TestALateCatalogueMergeIsNotAnsweredByAnOwnPathWalk(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
for _, m := range []struct {
|
||||
module string
|
||||
claims []catalogue.Claim
|
||||
}{
|
||||
{"nats", nil},
|
||||
{"mesh-delivery", []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
||||
} {
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1", Claims: m.claims},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + m.module, Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mixed := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "m1xed", MergedAt: t0.Format(time.RFC3339Nano),
|
||||
Paths: []string{"modules/nats/module.json", "modules/app/module.json"}, ModuleDirs: []string{"modules/nats", "modules/app"},
|
||||
ModuleDirsSaid: true}
|
||||
hear(t, open, mixed, t0.Add(time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 || ws[0].Waiting() {
|
||||
t.Fatalf("the mixed merge's walk waited, or was not cut: %+v", ws)
|
||||
}
|
||||
done := ws[0]
|
||||
done.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, catalogueMerge("ear1ier", "app", t0.Add(-30*time.Second)), t0.Add(3*time.Minute))
|
||||
got, _ := open.inventory.PlanByID(ctx, done.ID)
|
||||
_, bs := walks(t, open)
|
||||
if len(got.Delivery.Merges) != 1 || len(bs) != 1 || bs[0].OwnPath() || bs[0].Delivery.Merges[0].Commit != "ear1ier" {
|
||||
t.Fatalf("the late catalogue merge was answered by the own-path walk (%+v) rather than the next catalogue batch (%+v)",
|
||||
got.Delivery.Merges, bs)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,8 +6,10 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/beside"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// The check anybody can run is the check registration runs (novox/hq issue 148, ADR 0037): a manifest
|
||||
@@ -56,10 +58,7 @@ func TestModuleCheckJudgesBetweenTheManifestsGiven(t *testing.T) {
|
||||
|
||||
// The real catalogue passes the command, the way it passes the test that used to be the only check.
|
||||
func TestModuleCheckPassesTheCatalogue(t *testing.T) {
|
||||
root := filepath.Join("..", "..", "..", "mesh-catalog", "modules")
|
||||
if _, err := os.Stat(root); err != nil {
|
||||
t.Skipf("catalogue sibling not present: %v", err)
|
||||
}
|
||||
root := beside.Catalogue(t)
|
||||
paths, err := manifestsUnder(root)
|
||||
if err != nil || len(paths) == 0 {
|
||||
t.Fatalf("no manifests under %s: %v", root, err)
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path"
|
||||
"regexp"
|
||||
"runtime/debug"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -246,9 +248,11 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
|
||||
if dir == "mesh-controller" {
|
||||
beside["mesh-controller-main"] = link.CheckedOut{Repository: url, Ref: refs["mesh-controller-main"]}
|
||||
if e.Source.Seat != "" {
|
||||
if lab, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository,
|
||||
"mesh-lab")}); err == nil {
|
||||
beside["mesh-lab"] = link.CheckedOut{Repository: lab, Ref: refs["mesh-lab"]}
|
||||
for _, sibling := range []string{"mesh-lab", "mesh-sdk"} {
|
||||
if url, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository,
|
||||
sibling)}); err == nil {
|
||||
beside[sibling] = link.CheckedOut{Repository: url, Ref: refs[sibling]}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -271,17 +275,54 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
|
||||
// catalogue, whose checkout beside is what tests read its files from, at its main, what the next merge
|
||||
// builds from; and beside the controller its main, for a judge the running controller predates, and the
|
||||
// lab's main, whose replays every check runs. **One rule, read by the check the controller asks for and by
|
||||
// the facts snapshot** (Facts.Beside), so a check run by hand clones what the build seat clones.
|
||||
// the facts snapshot** (Facts.Beside), so a check run by hand clones what the build seat clones. Beside the
|
||||
// controller also the SDK, checked out at the commit the running controller's go.mod pins (sdkPinned), not
|
||||
// one a desktop holds (novox/hq issue 449). The checkout only places the clone: the conformance test reads the
|
||||
// fixtures at the pin of the tree under check, from the clone's history, so a pull request moving the SDK is
|
||||
// judged against the SDK it moves to (internal/link/conformance_test.go).
|
||||
func besideRefs(dir, running string) map[string]string {
|
||||
switch dir {
|
||||
case "mesh-catalog":
|
||||
return map[string]string{dir: "main"}
|
||||
case "mesh-controller":
|
||||
return map[string]string{dir: running, "mesh-controller-main": "main", "mesh-lab": "main"}
|
||||
return map[string]string{dir: running, "mesh-controller-main": "main", "mesh-lab": "main",
|
||||
"mesh-sdk": sdkPinned()}
|
||||
}
|
||||
return map[string]string{dir: running}
|
||||
}
|
||||
|
||||
// sdkModule is the Go module of the SDK the controller is built against.
|
||||
const sdkModule = "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
// pseudoCommit is the commit a Go pseudo-version names: v0.1.11-0.20261009143344-f047d0a4a970 → f047d0a4a970.
|
||||
var pseudoCommit = regexp.MustCompile(`-([0-9a-f]{12})$`)
|
||||
|
||||
// sdkPinned is the ref of the SDK repository this controller was built from, as its go.mod pins it and its
|
||||
// build records it: a pseudo-version's commit, or a release's tag (the SDK tags its Go module under go/).
|
||||
// "main" only for a binary that records no SDK version — a local replace — which a running controller is
|
||||
// not (novox/hq issue 449).
|
||||
func sdkPinned() string {
|
||||
info, ok := debug.ReadBuildInfo()
|
||||
if !ok {
|
||||
return "main"
|
||||
}
|
||||
for _, dep := range info.Deps {
|
||||
if dep.Path != sdkModule {
|
||||
continue
|
||||
}
|
||||
if dep.Replace != nil {
|
||||
dep = dep.Replace
|
||||
}
|
||||
if m := pseudoCommit.FindStringSubmatch(dep.Version); m != nil {
|
||||
return m[1]
|
||||
}
|
||||
if strings.HasPrefix(dep.Version, "v") {
|
||||
return "go/" + dep.Version
|
||||
}
|
||||
}
|
||||
return "main"
|
||||
}
|
||||
|
||||
// siblingOf is another repository of the same owner: novox/mesh-controller → novox/mesh-lab.
|
||||
func siblingOf(repository, name string) string {
|
||||
if cut := strings.LastIndex(repository, "/"); cut >= 0 {
|
||||
|
||||
@@ -169,6 +169,44 @@ func TestAShrinkOfMoreThanHalfIsUrgent(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// THE FALSE ALARM (issue 368), replayed through the store D13 reads: an agent's home moved from the
|
||||
// operator's own home (94.7 MB) to the agent account's fresh one (490 B), and `data-shrank` was raised
|
||||
// for data that was never lost. A moved item is read against its new path only, so nothing is raised —
|
||||
// and a genuine shrink at the new path, a week of history later, still is.
|
||||
func TestAMovedPathIsNoShrinkAndAShrinkThereStillIs(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
ctx := t.Context()
|
||||
shelf := shelfFor(t, houseManifest)
|
||||
declared := []inventory.DeclaredData{{Module: "house", Item: "config", Class: "irreplaceable", Owned: true}}
|
||||
start := time.Now().Add(-6 * time.Hour)
|
||||
measure := func(at time.Time, path string, size int64) []conditions.Observation {
|
||||
t.Helper()
|
||||
if _, err := inv.RecordData(ctx, "home", declared, map[string]map[string]inventory.Measurement{"house": {
|
||||
"config": {Path: path, Size: bytesOf(size), MeasuredAt: when(at), LastWrite: when(at),
|
||||
LastBackup: when(at)}}}, "", at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
records, err := inv.Data(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
peaks, err := inv.DataPeaks(ctx, at.Add(-shrinkWindow))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return dataFindings(records, peaks, shelf, nil, nil, at)
|
||||
}
|
||||
measure(start, "/home/operator/.claude", 94_700_000)
|
||||
if got := findingsByKind(measure(start.Add(10*time.Minute), "/home/agent/.claude", 490)); got[kindDataShrank].Kind != "" {
|
||||
t.Fatalf("a moved path raised a shrink: %+v", got[kindDataShrank])
|
||||
}
|
||||
measure(start.Add(2*time.Hour), "/home/agent/.claude", 300<<20)
|
||||
got := findingsByKind(measure(start.Add(4*time.Hour), "/home/agent/.claude", 1<<20))[kindDataShrank]
|
||||
if got.Severity != conditions.Urgent || !strings.Contains(got.Summary, "shrank") {
|
||||
t.Fatalf("a genuine shrink at the new path was not raised: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Data said to be written all the time and not written; data with no backup or an old one — urgent when
|
||||
// irreplaceable, a warning when valuable; and a new item given its bound before it is said.
|
||||
func TestQuietDataAndMissingBackupsAreSaidByClass(t *testing.T) {
|
||||
|
||||
@@ -144,11 +144,15 @@ func actOnDeadLetter(ctx context.Context, on *busHandles, act string, id uint64,
|
||||
}
|
||||
switch act {
|
||||
case "deliver":
|
||||
_, to, err := link.DeliverAgain(on.js, id)
|
||||
delivered, to, err := link.DeliverAgain(on.js, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
answer["delivered_on"] = to
|
||||
if delivered.Original != "" {
|
||||
// What became of the ask in its seat's queue (novox/hq issue 334): removed, or left, and why.
|
||||
answer["original"] = delivered.Original
|
||||
}
|
||||
answer["done"] = fmt.Sprintf("dead letter %d was delivered again to %s, and nobody else; it is no longer kept",
|
||||
id, consumerWho(d.Stream, d.Consumer))
|
||||
case "drop":
|
||||
|
||||
@@ -103,16 +103,8 @@ func letGo(ctx context.Context, inv *inventory.Inventory, id, by, why string) (i
|
||||
return p, fmt.Errorf("%s was let go by %s at %s already", p.ID, p.Delivery.By,
|
||||
p.Delivery.Go.Local().Format("15:04:05"))
|
||||
}
|
||||
// **One walk at a time** (novox/hq ADR 0276): a walk that started is open, so this one waits for its end.
|
||||
open, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return p, err
|
||||
}
|
||||
for _, q := range open {
|
||||
if q.ID != p.ID && q.Release == nil && !q.Waiting() {
|
||||
return p, fmt.Errorf("%s is open (%s): one walk at a time — %s is let go once it ended", q.ID, q.Named(), p.ID)
|
||||
}
|
||||
}
|
||||
// **One walk at a time** (novox/hq ADR 0276): the word is taken, and the walk starts once no other walk is
|
||||
// started (advanceOnce), so the delivery's owner says it once and is not refused.
|
||||
now := time.Now().UTC()
|
||||
p.Delivery.Go, p.Delivery.By, p.Delivery.Why = &now, by, why
|
||||
p.Note = "let go by " + by + "; its first tier is asked next"
|
||||
@@ -604,12 +596,81 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Each walk's phases, with the rest's reports (novox/hq ADR 0282 decision 6).
|
||||
now := time.Now()
|
||||
for i := range walks {
|
||||
walks[i].Phases = walks[i].WalkPhases(now, appliedFrom(ctx, inv))
|
||||
}
|
||||
return answer(map[string]any{"held": deliverySeatHeld(entries), "walks": walks,
|
||||
"own-path": sortedKeysOf(ownPathWords())})
|
||||
}
|
||||
return fmt.Errorf("delivery %s: plan, order, check, go, stop or walks", sub)
|
||||
}
|
||||
|
||||
// appliedFrom answers a machine's first report after a send from the controller's `apply` durations; a lookup
|
||||
// that fails is a report not read, which leaves the walk's end unknown rather than wrong.
|
||||
func appliedFrom(ctx context.Context, inv *inventory.Inventory) inventory.AppliedLookup {
|
||||
return func(node string, sent time.Time) (inventory.AppliedReport, bool) {
|
||||
r, ok, err := inv.FirstAppliedAfter(ctx, node, sent)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "the report of %s after %s could not be read: %v\n", node, sent.Format(time.RFC3339), err)
|
||||
return inventory.AppliedReport{}, false
|
||||
}
|
||||
return r, ok
|
||||
}
|
||||
}
|
||||
|
||||
// recordWalkPhases keeps, once, each phase of every walk ended lately whose end is known, as a duration of kind
|
||||
// walk-phase per class (novox/hq ADR 0282 decision 6): what `durations` summarises. A walk whose end is unknown
|
||||
// past ApplySilentAfter keeps its measured phases without its total.
|
||||
func recordWalkPhases(ctx context.Context, inv *inventory.Inventory, now time.Time) error {
|
||||
recent, err := inv.RecentPlans(ctx, 30)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, p := range recent {
|
||||
if p.State != inventory.PlanDone || p.Release != nil || now.Sub(p.Updated) > 2*time.Hour {
|
||||
continue
|
||||
}
|
||||
anyKept, totalKept, err := inv.WalkPhasesKept(ctx, p.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if totalKept {
|
||||
continue
|
||||
}
|
||||
ph := p.WalkPhases(now, appliedFrom(ctx, inv))
|
||||
if ph != nil && ph.End == nil && anyKept {
|
||||
continue // kept without its end; kept again only once its end is known
|
||||
}
|
||||
if ph == nil || (ph.End == nil && now.Sub(p.Updated) < inventory.ApplySilentAfter+time.Minute) {
|
||||
continue
|
||||
}
|
||||
class := ph.Class
|
||||
if class == "" {
|
||||
class = "unclassed"
|
||||
}
|
||||
for _, x := range ph.Phases {
|
||||
if x.State != inventory.PhaseMeasured || x.Start == nil {
|
||||
continue
|
||||
}
|
||||
if err := inv.RecordDuration(ctx, inventory.Duration{Kind: inventory.DurationWalkPhase,
|
||||
Subject: class + "/" + x.Name, Ref: fmt.Sprintf("%s/%s/%d", p.ID, x.Name, x.Tier), Started: *x.Start,
|
||||
Took: time.Duration(x.TookMS) * time.Millisecond, Detail: p.Named()}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if ph.End != nil && ph.From != nil {
|
||||
if err := inv.RecordDuration(ctx, inventory.Duration{Kind: inventory.DurationWalkPhase,
|
||||
Subject: class + "/total", Ref: p.ID + "/total", Started: *ph.From,
|
||||
Took: time.Duration(ph.TotalMS) * time.Millisecond, Detail: ph.Said}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ownPathWords is the controller's own path as words, for an answer.
|
||||
func ownPathWords() map[string]string { return onTheControllersPath }
|
||||
|
||||
@@ -736,6 +797,9 @@ func sayPlanMoved(ctx context.Context, bus link.Bus, p inventory.Plan) {
|
||||
}
|
||||
|
||||
func publishPlanMoved(ctx context.Context, bus link.Bus, p inventory.Plan) {
|
||||
// Its phases so far (novox/hq ADR 0282 decision 6): the rest's reports come after the walk ends, and are
|
||||
// read by whoever asks for the walk (`delivery walks`).
|
||||
p.Phases = p.WalkPhases(time.Now(), nil)
|
||||
body, err := json.Marshal(p)
|
||||
if err != nil {
|
||||
return
|
||||
|
||||
@@ -37,12 +37,34 @@ type stalledLine struct {
|
||||
// Number is the pull request's, for a line of a head the forge never announced (novox/hq issue 347).
|
||||
Number int `json:"number,omitempty"`
|
||||
State string `json:"state"`
|
||||
For string `json:"for"`
|
||||
Bound string `json:"bound"`
|
||||
H2 string `json:"h2"`
|
||||
Says string `json:"says"`
|
||||
// Waiting are the merge checks a line of the state `unanswered` waits on, as mesh-delivery says each:
|
||||
// "mesh/merge-gate pending since <UTC time>", "mesh/repo-check never set" (novox/hq issue 438).
|
||||
Waiting []string `json:"waiting,omitempty"`
|
||||
// Checks are the same merge checks as data, which the controller words in the operator's own time (novox/hq
|
||||
// issue 443): a time inside a finished sentence cannot be said again in another zone. A mesh-delivery from before
|
||||
// says none, and Waiting is said as it reads.
|
||||
Checks []waitingCheck `json:"checks,omitempty"`
|
||||
For string `json:"for"`
|
||||
Bound string `json:"bound"`
|
||||
H2 string `json:"h2"`
|
||||
Says string `json:"says"`
|
||||
}
|
||||
|
||||
// waitingCheck is one merge check a stalled line waits on, as mesh-delivery gives it: its context, its state —
|
||||
// "pending", or "never-set" — and, for a pending one, since when in RFC 3339, empty when the forge did not say.
|
||||
type waitingCheck struct {
|
||||
Context string `json:"context"`
|
||||
State string `json:"state"`
|
||||
Since string `json:"since,omitempty"`
|
||||
}
|
||||
|
||||
// wordsNow is the time the words are said at, which says whether a time needs its day; a seam a test replaces.
|
||||
var wordsNow = time.Now
|
||||
|
||||
// wordsZone is the zone a stalled line's times are said in: the controller's local zone, as every other time in its
|
||||
// messages. A seam a test replaces, so that no test writes time.Local, which every goroutine of the package reads.
|
||||
var wordsZone = func() *time.Location { return time.Local }
|
||||
|
||||
// operatorsOnly is whether the table leaves H2 nothing to do for the line: the state is the operator's.
|
||||
func (l stalledLine) operatorsOnly() bool { return l.H2 == "" || strings.HasPrefix(l.H2, "none") }
|
||||
|
||||
|
||||
@@ -15,16 +15,23 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/secrets"
|
||||
)
|
||||
|
||||
// A module's own secret given at the operator's desk (novox/hq ADR 0259 §10).
|
||||
// A module's own secret given at the operator's desk (novox/hq ADR 0259 §10, ADR 0277).
|
||||
//
|
||||
// mesh-controller secret ask <node> <module> <name> [--at <desk>]
|
||||
//
|
||||
// **The value never passes through whoever asked for it.** An agent, or the operator at the mesh MCP
|
||||
// server, calls `give` with the machine, the module, the secret's name and the desk — never a value. The
|
||||
// controller makes a sealing keypair for this one call, asks the desk's `node-launcher.secret` to prompt the
|
||||
// operator without showing what is typed, and is answered with what was typed **sealed to that key**: no
|
||||
// plaintext on the bus, in a runtime's log or in any call's record. It opens it here, seals it to the
|
||||
// server, calls `secret-ask` (or `give`) with the machine, the module, the secret's name and the desk — never
|
||||
// a value. The controller makes a sealing keypair for this one call, asks the desk's `node-launcher.secret` to
|
||||
// prompt the operator without showing what is typed, and is answered with what was typed **sealed to that
|
||||
// key**: no plaintext on the bus, in a runtime's log or in any call's record. It opens it here, seals it to the
|
||||
// module's machine exactly as `secret accept` does, and forgets it. What it answers says only that the
|
||||
// value was taken, or why not.
|
||||
//
|
||||
// **Bounded, and the prompt says who asked** (ADR 0277): one open prompt per secret and few an hour, read from
|
||||
// the store before the prompt opens (inventory.OpenSecretAsk), so an agent cannot keep a prompt in front of the
|
||||
// operator until they type. The prompt names the module, the secret, the machine and who asked — the caller as
|
||||
// the bus named it, never a word the caller chose — written by the desk's launcher from those names alone.
|
||||
//
|
||||
// **What remains** (ADR 0234's accepted residual risk): on an X11 desk any program of the operator's
|
||||
// account can read the keys as they are typed. And a program that calls the desk's prompt itself, with a
|
||||
// key of its own, is answered with what the operator typed into a prompt they did not ask for — as it could
|
||||
@@ -54,6 +61,37 @@ type deskGive struct {
|
||||
// announce raises the condition that says a module's own secret was given (secretGivenObservation), on
|
||||
// every channel; nil announces nothing (a test that does not look).
|
||||
announce func(node, module, name, how string) error
|
||||
// askedBy is who asked, as the bus named the caller: said in the prompt and recorded.
|
||||
askedBy string
|
||||
// open records the ask and holds the bounds (one open per secret, few an hour), answering the record's id;
|
||||
// nil keeps no record (a test that does not look). end closes it with how it ended.
|
||||
open func(node, module, name, desk string) (int64, error)
|
||||
end func(id int64, outcome string) error
|
||||
}
|
||||
|
||||
// askedByName is the caller as the prompt names it: the first clause of what the bus said, in the characters
|
||||
// a name has, at most 80 of them. The desk's launcher refuses anything else, so no words of the caller's own
|
||||
// reach the prompt.
|
||||
func askedByName(caller string) string {
|
||||
first, _, _ := strings.Cut(caller, ",")
|
||||
var b strings.Builder
|
||||
for _, r := range strings.TrimSpace(first) {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '.', r == '_', r == '/', r == '@',
|
||||
r == '-', r == ' ':
|
||||
b.WriteRune(r)
|
||||
default:
|
||||
b.WriteRune('-')
|
||||
}
|
||||
if b.Len() >= 80 {
|
||||
break
|
||||
}
|
||||
}
|
||||
name := strings.TrimSpace(b.String())
|
||||
if name == "" || strings.HasPrefix(name, "-") {
|
||||
return "an unnamed caller"
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// errNothingGiven is a prompt dismissed, or not answered in time: nothing changes.
|
||||
@@ -95,20 +133,37 @@ func (d deskGive) give(node, module, name, desk string) (string, error) {
|
||||
"bus, where an agent may answer first (novox/hq ADR 0259 §10)", module, name, node, module, name)
|
||||
}
|
||||
}
|
||||
// The bounds, read and kept before anybody is asked to type (novox/hq ADR 0277): one open prompt per secret,
|
||||
// few an hour. How the ask ends is recorded whatever happens below.
|
||||
outcome := "failed"
|
||||
if d.open != nil {
|
||||
id, err := d.open(node, module, name, desk)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("nobody was asked to type anything: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if d.end != nil {
|
||||
_ = d.end(id, outcome)
|
||||
}
|
||||
}()
|
||||
}
|
||||
public, private, err := secrets.Keypair()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("no key could be made to take the value: %w", err)
|
||||
}
|
||||
// By name, never by words: the holder writes the prompt from these, and says the controller asks, which
|
||||
// the bus alone makes true (broker.ControllerOnly).
|
||||
// the bus alone makes true (broker.ControllerOnly). Who asked is the bus's word on the caller, cut to a
|
||||
// name's characters — never an argument of the call.
|
||||
raw, err := d.ask(desk, map[string]any{
|
||||
"module": module,
|
||||
"secret": name,
|
||||
"node": node,
|
||||
"asked_by": askedByName(d.askedBy),
|
||||
"seal_to": public,
|
||||
"timeout_seconds": deskPromptWithin,
|
||||
})
|
||||
if err != nil {
|
||||
outcome = "refused"
|
||||
return "", fmt.Errorf("the desk on %s could not be asked: %w", desk, err)
|
||||
}
|
||||
var answer struct {
|
||||
@@ -121,8 +176,10 @@ func (d deskGive) give(node, module, name, desk string) (string, error) {
|
||||
}
|
||||
switch {
|
||||
case answer.TimedOut:
|
||||
outcome = "timed-out"
|
||||
return "", fmt.Errorf("%w: the prompt on %s was not answered within %d seconds", errNothingGiven, desk, deskPromptWithin)
|
||||
case answer.Cancelled:
|
||||
outcome = "dismissed"
|
||||
return "", fmt.Errorf("%w: the prompt on %s was dismissed", errNothingGiven, desk)
|
||||
case answer.Sealed == "":
|
||||
return "", fmt.Errorf("the desk on %s answered no sealed value", desk)
|
||||
@@ -137,6 +194,7 @@ func (d deskGive) give(node, module, name, desk string) (string, error) {
|
||||
opened[i] = 0
|
||||
}
|
||||
if strings.TrimSpace(value) == "" {
|
||||
outcome = "empty"
|
||||
return "", fmt.Errorf("%w: the prompt on %s was answered empty", errNothingGiven, desk)
|
||||
}
|
||||
untilStart, err := d.accept(value)
|
||||
@@ -144,8 +202,10 @@ func (d deskGive) give(node, module, name, desk string) (string, error) {
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
outcome = "given"
|
||||
act := link.HandAct{Verb: "secret accept", Args: []string{node, module, name, "--at-desk", desk},
|
||||
Why: fmt.Sprintf("the operator gave %s for %s on %s at the desk on %s", name, module, node, desk),
|
||||
Why: fmt.Sprintf("the operator gave %s for %s on %s at the desk on %s, asked by %s", name, module, node, desk,
|
||||
askedByName(d.askedBy)),
|
||||
Cause: "given-at-the-desk"}
|
||||
recorded := ""
|
||||
if err := d.record(act); err != nil {
|
||||
@@ -165,15 +225,23 @@ func (d deskGive) give(node, module, name, desk string) (string, error) {
|
||||
return words + recorded, nil
|
||||
}
|
||||
|
||||
// giveAtDesk is `secret accept <node> <module> <name> --at-desk <machine>`: the desk path, on this
|
||||
// controller's stores and bus.
|
||||
func giveAtDesk(ctx context.Context, node, module, name, desk string) error {
|
||||
// askAtDesk is `secret ask <node> <module> <name> [--at <machine>]`, and the terminal's `secret accept … --at-desk
|
||||
// <machine>`: the desk path, on this controller's stores and bus. The desk is the module's machine unless named.
|
||||
func askAtDesk(ctx context.Context, node, module, name, desk string) error {
|
||||
if desk == "" {
|
||||
desk = node
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
d := deskGive{
|
||||
askedBy: link.Caller(),
|
||||
open: func(node, module, name, desk string) (int64, error) {
|
||||
return open.inventory.OpenSecretAsk(ctx, node, module, name, askedByName(link.Caller()), desk)
|
||||
},
|
||||
end: func(id int64, outcome string) error { return open.inventory.EndSecretAsk(ctx, id, outcome) },
|
||||
declares: func(module, name string) error { return open.inventory.DeclaresOwnSecret(ctx, module, name) },
|
||||
known: func(machine string) error {
|
||||
_, err := open.inventory.NodeByName(ctx, machine)
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -124,12 +125,21 @@ func TestADismissedEmptyLateOrForeignAnswerTakesNothing(t *testing.T) {
|
||||
|
||||
func TestTheGiveVerbRunsTheDeskPathAndTheControllerMayAskTheDesk(t *testing.T) {
|
||||
argv, err := argvFor("give", map[string]any{"node": "anchor", "module": "telegram", "secret": "telegram-token", "at": "laptop"})
|
||||
if err != nil || strings.Join(argv, " ") != "secret accept anchor telegram telegram-token --at-desk laptop" {
|
||||
if err != nil || strings.Join(argv, " ") != "secret ask anchor telegram telegram-token --at laptop" {
|
||||
t.Fatalf("%v %v", argv, err)
|
||||
}
|
||||
if _, err := argvFor("give", map[string]any{"node": "anchor", "module": "telegram", "secret": "telegram-token"}); err == nil {
|
||||
t.Error("give without a desk was taken")
|
||||
}
|
||||
// secret-ask is the same line, with the desk the module's machine unless named (novox/hq ADR 0277).
|
||||
argv, err = argvFor("secret-ask", map[string]any{"node": "anchor", "module": "telegram", "secret": "telegram-token"})
|
||||
if err != nil || strings.Join(argv, " ") != "secret ask anchor telegram telegram-token" {
|
||||
t.Fatalf("%v %v", argv, err)
|
||||
}
|
||||
argv, err = argvFor("secret-ask", map[string]any{"node": "anchor", "module": "telegram", "secret": "telegram-token", "at": "laptop"})
|
||||
if err != nil || strings.Join(argv, " ") != "secret ask anchor telegram telegram-token --at laptop" {
|
||||
t.Fatalf("%v %v", argv, err)
|
||||
}
|
||||
perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -266,13 +276,22 @@ func TestASecretValueIsNeverAcceptedThroughAVerb(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The `give` verb's own line passes the terminal-only rule of ADR 0266, and no other `secret accept` does: a
|
||||
// value, a file, a provider or an extra word is still the terminal's alone.
|
||||
// The `give` and `secret-ask` verbs' own line passes the terminal-only rule of ADR 0266, and no `secret accept`
|
||||
// does: a value, a file, a provider or an extra word is still the terminal's alone (novox/hq ADR 0277).
|
||||
func TestOnlyTheGiveLinePassesTheTerminalRuleForSecrets(t *testing.T) {
|
||||
if err := terminalOnly([]string{"secret", "accept", "anchor", "telegram", "telegram-token", "--at-desk", "laptop"}); err != nil {
|
||||
t.Errorf("give's line refused: %v", err)
|
||||
for _, argv := range [][]string{
|
||||
{"secret", "ask", "anchor", "telegram", "telegram-token", "--at", "laptop"},
|
||||
{"secret", "ask", "anchor", "telegram", "telegram-token"},
|
||||
} {
|
||||
if err := terminalOnly(argv); err != nil {
|
||||
t.Errorf("%v refused: %v", argv, err)
|
||||
}
|
||||
}
|
||||
for _, argv := range [][]string{
|
||||
{"secret", "accept", "anchor", "telegram", "telegram-token", "--at-desk", "laptop"},
|
||||
{"secret", "ask", "anchor", "telegram", "telegram-token", "--from", "/tmp/x"},
|
||||
{"secret", "ask", "anchor", "telegram", "telegram-token", "--at", "laptop", "--local"},
|
||||
{"secret", "ask", "anchor", "telegram", "--at", "laptop"},
|
||||
{"secret", "accept", "anchor", "telegram", "telegram-token"},
|
||||
{"secret", "accept", "anchor", "telegram", "telegram-token", "--from", "/tmp/x"},
|
||||
{"secret", "accept", "anchor", "telegram", "telegram-token", "--at-desk", "laptop", "--local"},
|
||||
@@ -398,3 +417,96 @@ func TestAGiveNamingAMachineTheMeshDoesNotKnowAsksNobody(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0277: the prompt names who asked — the controller's word on the bus's caller, cut to a name's
|
||||
// characters — and never a word the caller chose: there is no argument for it.
|
||||
func TestThePromptNamesWhoAskedFromTheBussWordAlone(t *testing.T) {
|
||||
d, _, acts, asked := aDesk(t, sealedTo(t, typed))
|
||||
d.askedBy = "g14/claude-code, through the mesh-controller seat"
|
||||
if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := (*asked)[0]["asked_by"]; got != "g14/claude-code" {
|
||||
t.Errorf("asked_by %q", got)
|
||||
}
|
||||
if len(*acts) != 1 || !strings.Contains((*acts)[0].Why, "asked by g14/claude-code") {
|
||||
t.Errorf("the record: %+v", *acts)
|
||||
}
|
||||
for in, want := range map[string]string{
|
||||
"jochen at a shell on novox": "jochen at a shell on novox",
|
||||
"laptop/agent": "laptop/agent",
|
||||
"Your bank asks\nType your PIN, now": "Your bank asks-Type your PIN",
|
||||
"": "an unnamed caller",
|
||||
"<b>x</b>": "an unnamed caller",
|
||||
strings.Repeat("a", 100): strings.Repeat("a", 80),
|
||||
"--prompt, something else": "an unnamed caller",
|
||||
} {
|
||||
if got := askedByName(in); got != want {
|
||||
t.Errorf("askedByName(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
// The verb's schema has no argument that reaches the prompt's words.
|
||||
for _, verb := range []string{"give", "secret-ask"} {
|
||||
for _, free := range []string{"asked_by", "prompt", "message", "value", "from"} {
|
||||
if _, err := argvFor(verb, map[string]any{"node": "anchor", "module": "telegram", "secret": "telegram-token",
|
||||
"at": "laptop", free: "x"}); err == nil {
|
||||
t.Errorf("%s takes %s", verb, free)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An ask for a secret is bounded before anybody is asked to type (ADR 0277): the record refuses it, nothing is
|
||||
// asked; and how every ask ends is recorded.
|
||||
func TestASecretAskIsBoundedAndItsEndRecorded(t *testing.T) {
|
||||
d, accepted, _, asked := aDesk(t, sealedTo(t, typed))
|
||||
var ended []string
|
||||
d.open = func(node, module, name, desk string) (int64, error) { return 7, nil }
|
||||
d.end = func(id int64, outcome string) error {
|
||||
ended = append(ended, fmt.Sprintf("%d %s", id, outcome))
|
||||
return nil
|
||||
}
|
||||
if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.open = func(node, module, name, desk string) (int64, error) {
|
||||
return 0, errors.New("an ask for telegram-token of telegram on anchor is still open")
|
||||
}
|
||||
_, err := d.give("anchor", "telegram", "telegram-token", "laptop")
|
||||
if err == nil || !strings.Contains(err.Error(), "nobody was asked to type anything") || !strings.Contains(err.Error(), "still open") {
|
||||
t.Errorf("a second ask: %v", err)
|
||||
}
|
||||
if len(*asked) != 1 || len(*accepted) != 1 {
|
||||
t.Errorf("asked %d, accepted %d", len(*asked), len(*accepted))
|
||||
}
|
||||
d.open = func(node, module, name, desk string) (int64, error) { return 8, nil }
|
||||
d.ask = func(string, map[string]any) (json.RawMessage, error) {
|
||||
return json.RawMessage(`{"cancelled":true}`), nil
|
||||
}
|
||||
if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); !errors.Is(err, errNothingGiven) {
|
||||
t.Errorf("a dismissed prompt: %v", err)
|
||||
}
|
||||
if strings.Join(ended, "; ") != "7 given; 8 dismissed" {
|
||||
t.Errorf("ended: %v", ended)
|
||||
}
|
||||
}
|
||||
|
||||
// A secret ask cannot be turned into a secret read: the line carries no value, the answer carries none, and every
|
||||
// other `secret` line is the terminal's.
|
||||
func TestASecretAskIsNeverASecretRead(t *testing.T) {
|
||||
t.Setenv(verbVar, "mesh-controller.secret-ask")
|
||||
for _, args := range [][]string{
|
||||
{"ask", "anchor", "telegram", "telegram-token", "the-value"},
|
||||
{"ask", "anchor", "telegram"},
|
||||
{"ask", "anchor", "telegram", "telegram-token", "--from", "/dev/null"},
|
||||
} {
|
||||
if err := secretCommand(context.Background(), args); err == nil {
|
||||
t.Errorf("secret %v was taken", args)
|
||||
}
|
||||
}
|
||||
for _, args := range [][]string{{"recover", "anchor", "telegram", "telegram-token"}, {"export"}} {
|
||||
if err := terminalOnly(append([]string{"secret"}, args...)); err == nil {
|
||||
t.Errorf("secret %v passed the terminal rule", args)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -116,6 +116,11 @@ var probeRegistry = []probe{
|
||||
{ID: probeDeliveriesID, Asserts: "no delivery is held past its state's bound unsaid: mesh-delivery's " +
|
||||
"`stalled`, each with the transition its table lets healer H2 take", From: "ADR 0239",
|
||||
Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries},
|
||||
// The delivery budgets (novox/hq ADR 0282 decision 7): the newest delivery of each class within its budget,
|
||||
// read from mesh-delivery's `times`; a measurement said, never a delivery held.
|
||||
{ID: probeBudgetsID, Asserts: "the newest delivery of a leaf module ran on every machine within five minutes of " +
|
||||
"its merge, and of a core module within ten: mesh-delivery's `times`", From: "ADR 0282",
|
||||
Kind: kindOverBudget, Phase: 3, run: probeBudgets},
|
||||
// A client of the bus reconnecting in a loop (novox/hq issue 327), from the server's record of closed
|
||||
// connections, which the bus's own module reads.
|
||||
{ID: probeReconnectsID, Asserts: "no user of the bus had its connection dropped more than twelve times in the " +
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A provider whose wait fails the controller's check lists who waits on it (novox/hq issue 450).
|
||||
//
|
||||
// A wait that does not check out is judged unhealthy (ADR 0283 decision 3), so the provider raises its unhealthy
|
||||
// condition and its consumers are held under it (ADR 0240 rule 5). What is stored is what the machine said, the
|
||||
// wait unchecked: read as said, the provider seems to wait for the operator, and the held consumers were never
|
||||
// listed on the condition that is open.
|
||||
|
||||
// refusedWait is a wait for a secret the database's manifest does not declare: it fails the check.
|
||||
var refusedWait = inventory.Wait{Part: "postgres-database", Secret: "certificate", What: "the database's certificate"}
|
||||
|
||||
// judgedRefused is the provider's resource as the controller judges it: unhealthy, saying why.
|
||||
func judgedRefused() inventory.ResourceHealth {
|
||||
r := waitingDatabaseFor(refusedWait)
|
||||
r.State, r.Waits = link.StateUnhealthy, nil
|
||||
r.Reason = "says it waits for the secret certificate, which db does not declare"
|
||||
return r
|
||||
}
|
||||
|
||||
// The consumer's statement arrives after the provider's, so it is the consumer's judging (sayWaiters) that must list
|
||||
// it at the provider's unhealthy condition, made urgent by who waits on it.
|
||||
func TestAProviderWhoseWaitFailedItsCheckListsWhoWaitsOnIt(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
stored := waitingDatabaseFor(refusedWait)
|
||||
open := judgeBoth(t, k, []inventory.ResourceHealth{judgedRefused()},
|
||||
map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Resources: []inventory.ResourceHealth{stored}}},
|
||||
shopFailingBeside(stored))
|
||||
provider := conditionOf(open, moduleUnhealthyKey("db", "anchor"))
|
||||
if len(open) != 1 || provider == nil {
|
||||
t.Fatalf("a provider whose wait failed its check and a consumer of it raised %v; want the provider's "+
|
||||
"unhealthy alone", openKeysOf(open))
|
||||
}
|
||||
if said := provider.Evidence[0].Said; !strings.Contains(said, "shop on laptop") {
|
||||
t.Fatalf("the provider's unhealthy condition does not list shop on laptop as waiting on it: %s", said)
|
||||
}
|
||||
if provider.Severity != conditions.Urgent {
|
||||
t.Fatalf("the provider's unhealthy condition is %s with a consumer waiting on it; want urgent", provider.Severity)
|
||||
}
|
||||
}
|
||||
|
||||
// A provider that waits for a secret already given is unhealthy to its consumers too, before its own condition opens:
|
||||
// they are held under it as under any unhealthy provider, never as waiting for the operator, and its condition, when
|
||||
// open, is said as unhealthy with who waits on it.
|
||||
func TestAProviderWaitingForASecretAlreadyGivenIsUnhealthyToItsConsumers(t *testing.T) {
|
||||
given := holdingOf(nil, shopFailingBeside(waitingDatabase()), shopOnTheDatabase)
|
||||
given.waits.given["db@anchor"] = map[string]time.Time{"licence": time.Now().Add(-time.Hour)}
|
||||
by, held := given.heldWith("laptop", "shop", []inventory.ResourceHealth{failingConsumer("shop")})
|
||||
if !held || by.provider != theDatabase || len(by.waits) > 0 {
|
||||
t.Fatalf("shop under a database waiting for a licence already given: held %v under %v with waits %v; want "+
|
||||
"held under db on anchor as unhealthy, no waits", held, by.provider, by.waits)
|
||||
}
|
||||
if _, uncovered := given.waitingUncovered("laptop", "shop", []inventory.ResourceHealth{failingConsumer("shop")}); uncovered {
|
||||
t.Fatalf("a provider whose wait failed its check is said as waiting for another part")
|
||||
}
|
||||
}
|
||||
+79
-11
@@ -121,10 +121,13 @@ type gateFacts struct {
|
||||
health map[string]inventory.NodeHealth
|
||||
healthErr error
|
||||
// heldOn is, per "<module>@<machine>", the provider its findings are held under (ADR 0240 rule 5).
|
||||
heldOn map[string]string
|
||||
heldOn map[string]heldReading
|
||||
// groupsAdded is, per module, whether the move judged puts an account in a group its previous build did
|
||||
// not (issue 318 review): the only move whose wait for a new login is excused.
|
||||
groupsAdded map[string]bool
|
||||
// waits is what the controller holds to check a module's wait for the operator (novox/hq ADR 0283): the
|
||||
// manifest of each module's build judged, and the secrets given on each machine.
|
||||
waits operatorWaitFacts
|
||||
// sent is, per machine, the declaration the gate's own send carried there (novox/hq issue 352): a
|
||||
// report is held against it, never against the send made last. sentBuilds is what each machine was
|
||||
// last sent of every module, and judged the commit of each module this gate judges: a machine last
|
||||
@@ -134,6 +137,29 @@ type gateFacts struct {
|
||||
commits map[string]string
|
||||
}
|
||||
|
||||
// heldReading is the provider a module's findings are held under, as "<module> on <machine>", and, when that
|
||||
// provider only waits for the operator for the part the module needs, its wait in one sentence (novox/hq issue
|
||||
// 405): the module's gate then reads as a wait for a person (ADR 0254), a pass carrying the wait, as ADR 0283
|
||||
// decision 4 reads the waiting provider itself. A walk never waits on the operator's secret, there or here.
|
||||
type heldReading struct {
|
||||
on, waits string
|
||||
}
|
||||
|
||||
// heldReadings is, per "<module>@<machine>" in every machine's newest statement, what its findings are held under.
|
||||
func heldReadings(hold *holding) map[string]heldReading {
|
||||
out := map[string]heldReading{}
|
||||
for machine := range hold.healths {
|
||||
for module, by := range hold.heldModules(machine) {
|
||||
reading := heldReading{on: by.provider.Module + " on " + by.provider.Node}
|
||||
if len(by.waits) > 0 {
|
||||
reading.waits = operatorWaitSaid(by.provider.Module, by.provider.Node, by.waits)
|
||||
}
|
||||
out[module+"@"+machine] = reading
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// reportedOn says a machine's last report is on what the gate sent it (novox/hq issue 352): on that
|
||||
// declaration, or one it was sent after it — or, for a gate kept before sends were kept on it, on the
|
||||
// declaration last sent. On 2026-10-09 a release's gate read the control node's report against a newer
|
||||
@@ -199,12 +225,7 @@ var gatherGateFacts = func(ctx context.Context, open *stores, component string)
|
||||
// Whose findings wait on an unhealthy provider (ADR 0240 rule 5): their gates wait, not fail.
|
||||
if f.healthErr == nil && f.openErr == nil {
|
||||
if hold, err := readHolding(ctx, inv, f.open); err == nil {
|
||||
f.heldOn = map[string]string{}
|
||||
for machine := range f.health {
|
||||
for module, p := range hold.heldModules(machine) {
|
||||
f.heldOn[module+"@"+machine] = p.Module + " on " + p.Node
|
||||
}
|
||||
}
|
||||
f.heldOn = heldReadings(hold)
|
||||
}
|
||||
}
|
||||
if theLease != nil {
|
||||
@@ -257,10 +278,11 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string,
|
||||
firstLine(f.openErr.Error())
|
||||
}
|
||||
for _, c := range f.open {
|
||||
// A wait for a person's new login, or for a directory used as found to be handed over, is the module's
|
||||
// reading, not a fault raised since the send: the gate reads it from the statement below (ADR 0254,
|
||||
// novox/hq issue 339).
|
||||
if c.Source == gateProbe || c.OpenAt(since) || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound {
|
||||
// A wait for a person's new login, for a directory used as found to be handed over, or for the
|
||||
// operator's secret or setting, is the module's reading, not a fault raised since the send: the gate
|
||||
// reads it from the statement below (ADR 0254, novox/hq issue 339, ADR 0283).
|
||||
if c.Source == gateProbe || c.OpenAt(since) || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound ||
|
||||
c.Kind == kindNeedsOperator {
|
||||
continue
|
||||
}
|
||||
onIt := c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) ||
|
||||
@@ -477,6 +499,7 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
return "", err
|
||||
}
|
||||
facts.groupsAdded = movesAddingGroups(ctx, open.inventory, g, pairs, shelf)
|
||||
facts.waits = gateWaitFacts(ctx, open.inventory, g, pairs, shelf, facts.health)
|
||||
facts.sent = g.Sent
|
||||
facts.commits, facts.sentBuilds = judgedCommits(g, pairs), map[string]map[string]string{}
|
||||
// A module this gate put back at once (putBackBroken) was sent its earlier build by the gate itself:
|
||||
@@ -603,6 +626,7 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
pastBound := now.Sub(*g.Since) > gateBound
|
||||
switch {
|
||||
case worst == healthBroken:
|
||||
g.Read(now, false, g.BrokenWhy)
|
||||
var judging []string
|
||||
for _, m := range modules {
|
||||
if reading[m] != healthBroken && !passedAlone(m) {
|
||||
@@ -621,8 +645,10 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
// Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either —
|
||||
// the provider's own condition says what is wrong (ADR 0240 rule 5).
|
||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
||||
g.Read(now, false, why)
|
||||
case worst == healthNotYet:
|
||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
||||
g.Read(now, false, why)
|
||||
if pastBound {
|
||||
fail(fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why))
|
||||
}
|
||||
@@ -630,6 +656,7 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
// Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict.
|
||||
g.Passes++
|
||||
g.LastPass, g.Last, g.Failing = &now, "", nil
|
||||
g.Read(now, true, "")
|
||||
if g.Passes >= gatePasses && settled {
|
||||
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
|
||||
now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now)
|
||||
@@ -747,6 +774,47 @@ func movesAddingGroups(ctx context.Context, inv *inventory.Inventory, g *invento
|
||||
return out
|
||||
}
|
||||
|
||||
// gateWaitFacts reads what checks the waits for the operator of the modules a gate judges (novox/hq ADR 0283): the
|
||||
// manifest of the build judged — the one it moves to, else the catalogue's — and the secrets given on each machine
|
||||
// that says a module of them waits.
|
||||
func gateWaitFacts(ctx context.Context, inv *inventory.Inventory, g *inventory.PlanGate, pairs []judged,
|
||||
shelf map[string]catalogue.Manifest, health map[string]inventory.NodeHealth) operatorWaitFacts {
|
||||
var f operatorWaitFacts
|
||||
judgedManifests := map[string]catalogue.Manifest{}
|
||||
byMachine := map[string][]string{}
|
||||
for _, j := range pairs {
|
||||
waiting := false
|
||||
for _, r := range health[j.node].Resources {
|
||||
if r.Module == j.module && r.State == link.StateWaiting {
|
||||
waiting = true
|
||||
}
|
||||
}
|
||||
if !waiting {
|
||||
continue
|
||||
}
|
||||
byMachine[j.node] = append(byMachine[j.node], j.module)
|
||||
if _, done := judgedManifests[j.module]; done {
|
||||
continue
|
||||
}
|
||||
to := g.To
|
||||
for _, c := range g.Carried {
|
||||
if c.Module == j.module {
|
||||
to = c.To
|
||||
break
|
||||
}
|
||||
}
|
||||
if m, found, err := inv.ManifestAt(ctx, j.module, to); err == nil && found {
|
||||
judgedManifests[j.module] = m
|
||||
} else if m, known := shelf[j.module]; known {
|
||||
judgedManifests[j.module] = m
|
||||
}
|
||||
}
|
||||
for machine, modules := range byMachine {
|
||||
readWaitFacts(ctx, inv, machine, modules, judgedManifests, &f)
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// decide sets a gate's verdict.
|
||||
func decide(g *inventory.PlanGate, verdict, why string, now time.Time) {
|
||||
g.Verdict, g.Why, g.JudgedAt = verdict, why, &now
|
||||
|
||||
@@ -311,7 +311,7 @@ func usage() {
|
||||
the self-check: the last verdict, a run now, the probes, the signals' ages
|
||||
healers [--days N] [--json] the healers, what they did lately, and their brake (to-be 45 §7)
|
||||
durations [--kind K] [--days N] [--json]
|
||||
apply, heartbeat, plan-tier and build durations, per machine or module
|
||||
apply, heartbeat, plan-tier, build and walk-phase durations
|
||||
collection [--json] kept archives held/unheld by a manifest, and what the sweep may let go
|
||||
builder issue <name> a broker account for a build machine, scoped to build work,
|
||||
delivered as the builder module's broker secret (module add it first)
|
||||
|
||||
@@ -390,9 +390,10 @@ func TestAWithheldPathIsStoodInForByAPath(t *testing.T) {
|
||||
// controller's ask and by the facts — and finds the repository it checks from its origin.
|
||||
func TestACheckByHandClonesWhatTheSeatClones(t *testing.T) {
|
||||
for dir, refs := range map[string]map[string]string{
|
||||
"mesh-catalog": {"mesh-catalog": "main"},
|
||||
"mesh-host": {"mesh-host": "c0ffee"},
|
||||
"mesh-controller": {"mesh-controller": "c0ffee", "mesh-controller-main": "main", "mesh-lab": "main"},
|
||||
"mesh-catalog": {"mesh-catalog": "main"},
|
||||
"mesh-host": {"mesh-host": "c0ffee"},
|
||||
"mesh-controller": {"mesh-controller": "c0ffee", "mesh-controller-main": "main", "mesh-lab": "main",
|
||||
"mesh-sdk": sdkPinnedByGoMod(t)},
|
||||
} {
|
||||
got := besideRefs(dir, "c0ffee")
|
||||
for d, ref := range refs {
|
||||
@@ -412,3 +413,24 @@ func TestACheckByHandClonesWhatTheSeatClones(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sdkPinnedByGoMod is the SDK commit this tree's go.mod pins, read from the file rather than the build, so
|
||||
// sdkPinned is held to what the repository says (novox/hq issue 449).
|
||||
func sdkPinnedByGoMod(t *testing.T) string {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", "go.mod"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) >= 2 && fields[0] == sdkModule {
|
||||
if m := pseudoCommit.FindStringSubmatch(fields[1]); m != nil {
|
||||
return m[1]
|
||||
}
|
||||
return "go/" + fields[1]
|
||||
}
|
||||
}
|
||||
t.Fatalf("go.mod requires no %s", sdkModule)
|
||||
return ""
|
||||
}
|
||||
|
||||
@@ -74,9 +74,21 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke
|
||||
kept := inventory.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target,
|
||||
State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts,
|
||||
Check: r.Check, Needs: r.Needs, Account: r.Account, Root: r.Root}
|
||||
for _, w := range r.Waits {
|
||||
kept.Waits = append(kept.Waits, inventory.Wait{Part: w.Part, Secret: w.Secret, Setting: w.Setting, What: w.What})
|
||||
}
|
||||
resources = append(resources, kept)
|
||||
if r.State == link.StateUnhealthy && r.Module != "" {
|
||||
unhealthy[r.Module] = append(unhealthy[r.Module], kept)
|
||||
}
|
||||
// **A wait for the operator is checked before it is excused** (novox/hq ADR 0283): a waiting resource whose
|
||||
// wait does not check out is judged unhealthy, saying why; one that does is kept beside the unhealthy ones, so
|
||||
// judgeModuleHealth can say it as needs-operator. What is stored is what the machine said.
|
||||
var wf operatorWaitFacts
|
||||
if mods := waitingModules(resources); len(mods) > 0 {
|
||||
readWaitFacts(ctx, inv, node, mods, nil, &wf)
|
||||
}
|
||||
for _, r := range checkWaiting(node, resources, wf) {
|
||||
if (r.State == link.StateUnhealthy || r.State == link.StateWaiting) && r.Module != "" {
|
||||
unhealthy[r.Module] = append(unhealthy[r.Module], r)
|
||||
}
|
||||
}
|
||||
streaks := map[string]int{}
|
||||
@@ -135,16 +147,15 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
}
|
||||
standing := map[string]conditions.Condition{}
|
||||
for _, c := range open {
|
||||
if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound) &&
|
||||
if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound ||
|
||||
c.Kind == kindNeedsOperator) &&
|
||||
c.Subject.Machine == node {
|
||||
standing[c.Key] = c
|
||||
}
|
||||
}
|
||||
var hold *holding
|
||||
if inv != nil {
|
||||
if hold, err = readHolding(ctx, inv, open); err != nil {
|
||||
return err
|
||||
}
|
||||
hold, err := readHoldingFor(ctx, inv, open)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var problems []string
|
||||
modules := make([]string, 0, len(unhealthy))
|
||||
@@ -159,6 +170,39 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
heldOn := map[string]string{}
|
||||
providers := map[catalogue.Chosen]bool{}
|
||||
for _, m := range modules {
|
||||
// **A part that waits for the operator is said as that** (novox/hq ADR 0283): its waits already checked,
|
||||
// the operator's, never urgent, its words naming the act.
|
||||
if waits, waiting := operatorWait(m, unhealthy[m]); waiting {
|
||||
o := needsOperatorObservation(m, node, waits, unhealthy[m])
|
||||
// **A provider waiting for the operator says who waits on it** (novox/hq issue 405), as one waiting for a
|
||||
// login does: its consumers are held under it.
|
||||
if hold != nil {
|
||||
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
|
||||
}
|
||||
seen[o.Key()] = true
|
||||
became[m] = kindNeedsOperator
|
||||
if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen {
|
||||
continue
|
||||
}
|
||||
if _, err := k.Observe(ctx, o); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
continue
|
||||
}
|
||||
// **A wait for the operator beside anything else is said too** (novox/hq issue 405): a module with a checked
|
||||
// wait beside a new login owed, a directory used as found or a fault of its own is said as that, and the
|
||||
// operator's secret or setting it waits for was not mentioned until the other cleared. Its needs-operator
|
||||
// condition stands beside the other, on the same looks; what follows judges the rest without the wait.
|
||||
if waits, rest := besideAWait(m, unhealthy[m]); len(waits) > 0 {
|
||||
o := needsOperatorObservation(m, node, waits, waitingOf(m, unhealthy[m]))
|
||||
seen[o.Key()] = true
|
||||
if _, isOpen := standing[o.Key()]; streaks[m] >= moduleUnhealthyAfter || isOpen {
|
||||
if _, err := k.Observe(ctx, o); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
unhealthy[m] = rest
|
||||
}
|
||||
// **A directory used as found is said as that** (novox/hq issue 339): the operator's to hand over at the
|
||||
// machine, never urgent — nothing is broken by the wait that a person was not told of — and its own kind,
|
||||
// so the gate never reads it as a fault of the build that happened to be sent beside it.
|
||||
@@ -195,13 +239,24 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
}
|
||||
o := moduleUnhealthyObservation(m, node, unhealthy[m])
|
||||
if hold != nil {
|
||||
if p, held := hold.heldUnder(node, m, unhealthy[m]); held {
|
||||
// Held under the provider's condition: nothing of its own, and the provider's says it waits.
|
||||
heldOn[o.Key()] = p.Module + " on " + p.Node
|
||||
if by, held := hold.heldWith(node, m, unhealthy[m]); held {
|
||||
// Held under the provider's condition: nothing of its own, and the provider's says it waits. The
|
||||
// clearing line says which the provider is: unhealthy, or waiting for the operator (issue 405).
|
||||
p := by.provider
|
||||
heldOn[o.Key()] = p.Module + " on " + p.Node + ", which is unhealthy"
|
||||
if len(by.waits) > 0 {
|
||||
heldOn[o.Key()] = p.Module + " on " + p.Node + ", which waits for you"
|
||||
}
|
||||
providers[p] = true
|
||||
continue
|
||||
}
|
||||
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
|
||||
// A provider that waits for the operator for a part this finding cannot be matched to does not hold it
|
||||
// (novox/hq issue 405): raised as its own, and saying the provider waits, so neither is hidden.
|
||||
if p, waiting := hold.waitingUncovered(node, m, unhealthy[m]); waiting {
|
||||
o.Said += fmt.Sprintf("; %s on %s waits for you, but not for anything %s is known to need, so %s's "+
|
||||
"fault is said on its own", p.Module, p.Node, m, m)
|
||||
}
|
||||
}
|
||||
seen[o.Key()] = true
|
||||
became[m] = kindModuleUnhealthy
|
||||
@@ -228,8 +283,11 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
if c.Kind == kindUsedAsFound {
|
||||
why = fmt.Sprintf("%s says no directory of %s is used as found any more", node, module)
|
||||
}
|
||||
if c.Kind == kindNeedsOperator {
|
||||
why = fmt.Sprintf("%s says %s no longer waits for the operator", node, module)
|
||||
}
|
||||
if on, held := heldOn[key]; held {
|
||||
why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on)
|
||||
why = fmt.Sprintf("what %s finds on %s waits on %s: held under its condition", module, node, on)
|
||||
}
|
||||
// **A condition that became the other kind** is not "working again" (issue 318 review): its clearing
|
||||
// line says what it became.
|
||||
@@ -238,6 +296,12 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
case c.Kind == kindModuleUnhealthy && became[module] == kindReloginNeeded:
|
||||
why = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
|
||||
resolved = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
|
||||
case c.Kind == kindModuleUnhealthy && became[module] == kindNeedsOperator:
|
||||
why = fmt.Sprintf("%s on %s now only waits for the operator", module, node)
|
||||
resolved = fmt.Sprintf("%s on %s now only waits for you", module, node)
|
||||
case c.Kind == kindNeedsOperator && became[module] == kindModuleUnhealthy:
|
||||
why = fmt.Sprintf("%s on %s no longer only waits for the operator, and is not healthy", module, node)
|
||||
resolved = fmt.Sprintf("What %s on %s waited for is given, and it still does not work", module, node)
|
||||
case c.Kind == kindReloginNeeded && became[module] == kindModuleUnhealthy:
|
||||
why = fmt.Sprintf("%s on %s no longer waits for a new login, and is not healthy", module, node)
|
||||
resolved = fmt.Sprintf("The new login on %s is done, and %s still does not work", node, module)
|
||||
@@ -262,36 +326,74 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
// sayWaiters observes a provider's open condition again, with who waits on it, from its machine's newest
|
||||
// statement. Nothing when its condition is not open: it is raised by its own statements, on its own looks.
|
||||
func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p catalogue.Chosen, now time.Time) error {
|
||||
var raisedAt *conditions.Condition
|
||||
for i, c := range hold.open {
|
||||
if c.Key == moduleUnhealthyKey(p.Module, p.Node) || c.Key == reloginKey(p.Module, p.Node) {
|
||||
raisedAt = &hold.open[i]
|
||||
}
|
||||
}
|
||||
if raisedAt == nil {
|
||||
return nil
|
||||
}
|
||||
// Its statement as it was judged, its waits checked (novox/hq issue 450): a wait that failed the check is
|
||||
// unhealthy, so the condition built here is the one its own statement raised, and who waits on it is listed.
|
||||
var rs []inventory.ResourceHealth
|
||||
for _, r := range hold.healths[p.Node].Resources {
|
||||
if r.Module == p.Module && r.State == link.StateUnhealthy {
|
||||
for _, r := range hold.checked(p.Node) {
|
||||
if r.Module == p.Module && (r.State == link.StateUnhealthy || r.State == link.StateWaiting) {
|
||||
rs = append(rs, r)
|
||||
}
|
||||
}
|
||||
if len(rs) == 0 {
|
||||
return nil
|
||||
}
|
||||
o := moduleUnhealthyObservation(p.Module, p.Node, rs)
|
||||
if said, waits := personWait(p.Module, p.Node, rs); waits {
|
||||
o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rs)
|
||||
// The condition its own statement says it under: needs-operator while it only waits for the operator (novox/hq
|
||||
// issue 405), else that for the rest of it, a wait beside it said on its own.
|
||||
var o conditions.Observation
|
||||
if waits, waiting := operatorWait(p.Module, rs); waiting {
|
||||
o = needsOperatorObservation(p.Module, p.Node, waits, rs)
|
||||
} else {
|
||||
_, rest := besideAWait(p.Module, rs)
|
||||
o = moduleUnhealthyObservation(p.Module, p.Node, rest)
|
||||
if said, waits := personWait(p.Module, p.Node, rest); waits {
|
||||
o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rest)
|
||||
}
|
||||
}
|
||||
if o.Key() != raisedAt.Key {
|
||||
return nil // its own statement says it next
|
||||
raised := false
|
||||
for _, c := range hold.open {
|
||||
raised = raised || c.Key == o.Key()
|
||||
}
|
||||
if !raised {
|
||||
return nil // not open yet, or open as another kind: its own statement says it next
|
||||
}
|
||||
sayWaitingOn(&o, hold.waitersOn(p))
|
||||
_, err := k.Observe(ctx, o)
|
||||
return err
|
||||
}
|
||||
|
||||
// besideAWait is, for a module with something not healthy beside a part waiting for the operator, the waits (checked
|
||||
// already) and the rest without the waiting parts (novox/hq issue 405); no waits when nothing waits, or when the
|
||||
// module only waits (operatorWait says that whole). Pure.
|
||||
func besideAWait(module string, rs []inventory.ResourceHealth) ([]inventory.Wait, []inventory.ResourceHealth) {
|
||||
if _, only := operatorWait(module, rs); only {
|
||||
return nil, rs
|
||||
}
|
||||
var waits []inventory.Wait
|
||||
var rest []inventory.ResourceHealth
|
||||
for _, r := range rs {
|
||||
if r.Module == module && r.State == link.StateWaiting {
|
||||
waits = append(waits, r.Waits...)
|
||||
continue
|
||||
}
|
||||
rest = append(rest, r)
|
||||
}
|
||||
if len(waits) == 0 {
|
||||
return nil, rs
|
||||
}
|
||||
return waits, rest
|
||||
}
|
||||
|
||||
// waitingOf is a module's waiting resources: the evidence of its wait.
|
||||
func waitingOf(module string, rs []inventory.ResourceHealth) []inventory.ResourceHealth {
|
||||
var out []inventory.ResourceHealth
|
||||
for _, r := range rs {
|
||||
if r.Module == module && r.State == link.StateWaiting {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// reloginKey is a module's relogin-needed condition on a machine.
|
||||
func reloginKey(module, node string) string {
|
||||
return conditions.Key(conditions.ScopeModule, module+"."+node, kindReloginNeeded)
|
||||
@@ -363,12 +465,15 @@ func waitingAccounts(module string, rs []inventory.ResourceHealth) []string {
|
||||
}
|
||||
|
||||
// sayWaitingOn adds to a module's condition the consumers held under it (to-be 48 §6): urgent while anyone
|
||||
// waits on it, whether it is not working or waits for a new login.
|
||||
// waits on it, whether it is not working or waits for a new login. **A wait for the operator's secret or setting
|
||||
// stays a warning** (ADR 0283 decision 5, novox/hq issue 405): who waits on it is listed, and nothing escalates it.
|
||||
func sayWaitingOn(o *conditions.Observation, waiters []string) {
|
||||
if len(waiters) == 0 {
|
||||
return
|
||||
}
|
||||
o.Severity = conditions.Urgent
|
||||
if o.Kind != kindNeedsOperator {
|
||||
o.Severity = conditions.Urgent
|
||||
}
|
||||
o.Said += "; " + waitingWords(waiters)
|
||||
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
|
||||
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
|
||||
@@ -420,6 +525,11 @@ func reasonWords(r inventory.ResourceHealth) string {
|
||||
case "":
|
||||
return "is unhealthy"
|
||||
}
|
||||
// A wait for the operator that did not check out is said as the controller found it (ADR 0283): names of
|
||||
// secrets and settings only, never what the check itself said.
|
||||
if strings.HasPrefix(r.Reason, waitRefusedPrefix) {
|
||||
return r.Reason
|
||||
}
|
||||
// What a declared check found says an endpoint, a path or an address: evidence, never the summary the
|
||||
// operator's channel carries (ADR 0234 §6). The summary names the check.
|
||||
if r.Check != "" {
|
||||
@@ -511,7 +621,9 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
|
||||
if h.HeardAt.Before(since) {
|
||||
return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module)
|
||||
}
|
||||
wait, waits := personWait(module, machine, h.Resources)
|
||||
// **A wait for the operator is checked first** (novox/hq ADR 0283): one that does not check out is unhealthy.
|
||||
resources := checkWaiting(machine, h.Resources, f.waits)
|
||||
wait, waits := personWait(module, machine, resources)
|
||||
// **Only a build whose own send put the account in a new group is excused** (issue 318 review): read from
|
||||
// what the controller sent, never from when the machine says the wait began — that time is the engine's
|
||||
// memory, reset by its restart and moved by a change of words. A build that adds no account group cannot
|
||||
@@ -519,11 +631,18 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
|
||||
if waits && !f.groupsAdded[module] {
|
||||
waits = false
|
||||
}
|
||||
var found []string
|
||||
for _, r := range h.Resources {
|
||||
var found, onWaiting []string
|
||||
var forOperator []inventory.Wait
|
||||
for _, r := range resources {
|
||||
if r.Module != module {
|
||||
continue
|
||||
}
|
||||
// **Any build is excused while its wait for the operator checks out** (ADR 0283 decision 4): a secret not
|
||||
// given is owed by every build alike, so it is no fault of this one, and the verdict carries it.
|
||||
if r.State == link.StateWaiting {
|
||||
forOperator = append(forOperator, r.Waits...)
|
||||
continue
|
||||
}
|
||||
if waits && r.State == link.StateUnhealthy {
|
||||
continue
|
||||
}
|
||||
@@ -541,8 +660,16 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
|
||||
return healthNotYet, fmt.Sprintf("its %s %s on %s is still starting", r.Kind, r.Resource, machine)
|
||||
case link.StateUnhealthy:
|
||||
if on, held := f.heldOn[module+"@"+machine]; held {
|
||||
// **Held under a provider that only waits for the operator** (novox/hq issue 405): a wait for a
|
||||
// person, a pass carrying the wait (ADR 0254, ADR 0283 decision 4) — the walk never waits for the
|
||||
// operator's secret, whichever module owes it.
|
||||
if on.waits != "" {
|
||||
onWaiting = append(onWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which waits for you: %s",
|
||||
r.Kind, r.Resource, machine, on.on, on.waits))
|
||||
continue
|
||||
}
|
||||
return healthWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which is unhealthy", r.Kind,
|
||||
r.Resource, machine, on)
|
||||
r.Resource, machine, on.on)
|
||||
}
|
||||
return healthNotYet, fmt.Sprintf("its %s %s on %s %s", r.Kind, r.Resource, machine, reasonWords(r))
|
||||
default:
|
||||
@@ -550,11 +677,14 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
|
||||
reasonAfter(r.Reason))
|
||||
}
|
||||
}
|
||||
if waits || len(found) > 0 {
|
||||
var said []string
|
||||
if waits || len(found) > 0 || len(forOperator) > 0 || len(onWaiting) > 0 {
|
||||
said := onWaiting
|
||||
if waits {
|
||||
said = append(said, wait)
|
||||
}
|
||||
if len(forOperator) > 0 {
|
||||
said = append(said, operatorWaitSaid(module, machine, forOperator))
|
||||
}
|
||||
if len(found) > 0 {
|
||||
said = append(said, fmt.Sprintf("on %s, %s uses %s as found and waits for the operator to hand it over "+
|
||||
"(`nox node hand-over %s <directory>` on the control-node)", machine, module, strings.Join(found, ", "), machine))
|
||||
|
||||
@@ -3,10 +3,12 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/beside"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
@@ -239,13 +241,12 @@ func TestATakeoverIsNotComposedForAHubPlacedOffItsTunnel(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// theResolver is the catalogue's dnsmasq module as it is, or the test is skipped where the
|
||||
// catalogue is not beside this checkout.
|
||||
// theResolver is the catalogue's dnsmasq module as it is (internal/beside).
|
||||
func theResolver(t *testing.T) catalogue.Manifest {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile("../../../mesh-catalog/modules/dnsmasq/module.json")
|
||||
raw, err := os.ReadFile(filepath.Join(beside.Catalogue(t), "dnsmasq", "module.json"))
|
||||
if err != nil {
|
||||
t.Skipf("the catalogue is not beside this checkout: %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A part that waits for the operator's secret or setting (novox/hq ADR 0283, issue 386).
|
||||
//
|
||||
// **A module's tool check may say it waits**: nothing of it is wrong but a part that cannot work until the operator
|
||||
// gives one of its own secrets or one of its settings. The node-engine states such a resource `waiting`, with what
|
||||
// it waits for. A module saying so is an assertion, so **the controller checks each wait before it excuses it**:
|
||||
//
|
||||
// - a wait for a secret names an own secret the module's manifest declares — by its name, or as a member of a
|
||||
// secret family — said `"issued-by": "outside"`, and the store holds no value a person gave for it on that
|
||||
// machine;
|
||||
// - a wait for a setting names a setting the manifest declares (checked by name only: the controller cannot tell
|
||||
// whether a free-form value covers a part, and the needs-operator condition is where a false one shows).
|
||||
//
|
||||
// An excused wait is read by the first-node gate as *waits for a person* (ADR 0254), a pass carried in the verdict,
|
||||
// for any build of the module — a secret not given is owed by every build alike. It is said to the operator as
|
||||
// `module.<module>.<machine>.needs-operator`, naming the act. A wait that fails the check is judged unhealthy, saying
|
||||
// why, and raises the module's `unhealthy` condition.
|
||||
|
||||
// kindNeedsOperator is a module's condition while a part of it waits for the operator's secret or setting.
|
||||
const kindNeedsOperator = "needs-operator"
|
||||
|
||||
// needsOperatorKey is a module's needs-operator condition on a machine.
|
||||
func needsOperatorKey(module, node string) string {
|
||||
return conditions.Key(conditions.ScopeModule, module+"."+node, kindNeedsOperator)
|
||||
}
|
||||
|
||||
// operatorWaitFacts is what the controller holds to check a module's waits: the manifest judged per module, and per
|
||||
// "<module>@<machine>" the own secrets a person gave there, with when. A module or a machine absent is not known,
|
||||
// and no wait of it is excused.
|
||||
type operatorWaitFacts struct {
|
||||
manifests map[string]catalogue.Manifest
|
||||
given map[string]map[string]time.Time
|
||||
}
|
||||
|
||||
// checkWait is nil when a wait is excused, and otherwise why not, in words. Pure.
|
||||
func checkWait(module, machine string, w inventory.Wait, f operatorWaitFacts) error {
|
||||
m, known := f.manifests[module]
|
||||
if !known {
|
||||
return fmt.Errorf("says it waits for %s, and the mesh holds no manifest of %s to check it against", waitNames(w), module)
|
||||
}
|
||||
switch {
|
||||
case w.Secret != "" && w.Setting != "", w.Secret == "" && w.Setting == "":
|
||||
return fmt.Errorf("says it waits, naming %s, where a wait names one secret or one setting", waitNames(w))
|
||||
case w.Setting != "":
|
||||
if _, declared := m.Settings[w.Setting]; !declared {
|
||||
return fmt.Errorf("says it waits for the setting %s, which %s does not declare", w.Setting, module)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
own, _, declared := m.OwnSecrets.Lookup(w.Secret)
|
||||
if !declared {
|
||||
return fmt.Errorf("says it waits for the secret %s, which %s does not declare", w.Secret, module)
|
||||
}
|
||||
if own.IssuedBy != catalogue.IssuedOutside {
|
||||
return fmt.Errorf("says it waits for the secret %s, which the mesh makes itself: only a secret issued outside "+
|
||||
"the mesh waits for the operator", w.Secret)
|
||||
}
|
||||
given, readable := f.given[module+"@"+machine]
|
||||
if !readable {
|
||||
return fmt.Errorf("says it waits for the secret %s, and what was given on %s could not be read", w.Secret, machine)
|
||||
}
|
||||
if at, was := given[w.Secret]; was {
|
||||
return fmt.Errorf("says it waits for the secret %s, which was given at %s", w.Secret,
|
||||
at.UTC().Format("2006-01-02 15:04 MST"))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// waitRefusedPrefix opens every reason checkWait gives, so the words of a refused wait are told from a check's own.
|
||||
const waitRefusedPrefix = "says it waits"
|
||||
|
||||
// waitNames is what a wait names, as "the secret x" or "the setting y".
|
||||
func waitNames(w inventory.Wait) string {
|
||||
switch {
|
||||
case w.Secret != "" && w.Setting != "":
|
||||
return "the secret " + w.Secret + " and the setting " + w.Setting
|
||||
case w.Secret != "":
|
||||
return "the secret " + w.Secret
|
||||
case w.Setting != "":
|
||||
return "the setting " + w.Setting
|
||||
}
|
||||
return "nothing"
|
||||
}
|
||||
|
||||
// checkWaiting reads one machine's resources against the facts: every waiting resource whose waits all check out is
|
||||
// kept as said; one with a wait that does not, or with no wait at all, is answered as unhealthy with why. Pure; the
|
||||
// statement as kept is not changed.
|
||||
func checkWaiting(machine string, rs []inventory.ResourceHealth, f operatorWaitFacts) []inventory.ResourceHealth {
|
||||
out := make([]inventory.ResourceHealth, 0, len(rs))
|
||||
for _, r := range rs {
|
||||
if r.State == link.StateWaiting {
|
||||
var why error
|
||||
if len(r.Waits) == 0 {
|
||||
why = fmt.Errorf("says it waits, and names nothing it waits for")
|
||||
}
|
||||
for _, w := range r.Waits {
|
||||
if why == nil {
|
||||
why = checkWait(r.Module, machine, w, f)
|
||||
}
|
||||
}
|
||||
if why != nil {
|
||||
r.State, r.Reason = link.StateUnhealthy, why.Error()
|
||||
}
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// operatorWait is whether everything not healthy of a module on a machine is waiting with its waits checked
|
||||
// (checkWaiting already applied), and those waits. A module with anything unhealthy, starting or unknown beside it
|
||||
// does not wait: it is judged as before.
|
||||
func operatorWait(module string, rs []inventory.ResourceHealth) ([]inventory.Wait, bool) {
|
||||
var waits []inventory.Wait
|
||||
for _, r := range rs {
|
||||
if r.Module != module {
|
||||
continue
|
||||
}
|
||||
switch r.State {
|
||||
case link.StateHealthy:
|
||||
case link.StateWaiting:
|
||||
waits = append(waits, r.Waits...)
|
||||
default:
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
return waits, len(waits) > 0
|
||||
}
|
||||
|
||||
// operatorWaitSaid is a module's wait for the operator in one sentence, for the gate's verdict and the condition's
|
||||
// summary: what the operator gives and what it names, and for a secret the line that opens the desk prompt.
|
||||
func operatorWaitSaid(module, machine string, waits []inventory.Wait) string {
|
||||
var parts []string
|
||||
for _, w := range waits {
|
||||
part := fmt.Sprintf("%s (%s", w.What, waitNames(w))
|
||||
if w.Secret != "" {
|
||||
part += fmt.Sprintf(", given with `nox secret ask %s %s %s`", machine, module, w.Secret)
|
||||
}
|
||||
parts = append(parts, part+")")
|
||||
}
|
||||
return fmt.Sprintf("%s on %s waits for the operator: %s", module, machine, strings.Join(parts, "; "))
|
||||
}
|
||||
|
||||
// needsOperatorObservation is a module whose only parts not healthy wait for the operator (ADR 0283): the operator's,
|
||||
// a warning however long it stands, its plain words naming the act and never saying there is nothing to do.
|
||||
func needsOperatorObservation(module, node string, waits []inventory.Wait, rs []inventory.ResourceHealth) conditions.Observation {
|
||||
o := moduleUnhealthyObservation(module, node, rs)
|
||||
o.Token, o.Kind, o.Resolver, o.Severity = kindNeedsOperator, kindNeedsOperator, conditions.ResolverOperator, conditions.Warning
|
||||
o.Summary = operatorWaitSaid(module, node, waits)
|
||||
w := needsOperatorWords(module, node, waits)
|
||||
o.Headline, o.Explanation, o.Needs, o.Resolved, o.Actions = w.Headline, w.Explanation, w.Needs, w.Resolved, nil
|
||||
return o
|
||||
}
|
||||
|
||||
// needsOperatorWords is what the operator reads of a module waiting for them (ADR 0253, ADR 0283): the act, for a
|
||||
// secret typed at the machine's desk prompt and for a setting approved when an agent proposes it. The secret's and
|
||||
// the setting's names, and the line, are in the summary for whoever looks closer.
|
||||
func needsOperatorWords(module, node string, waits []inventory.Wait) words {
|
||||
var acts []string
|
||||
seen := map[string]bool{}
|
||||
secret := false
|
||||
for _, w := range waits {
|
||||
var act string
|
||||
switch {
|
||||
case w.Secret != "":
|
||||
act, secret = fmt.Sprintf("type %s at %s's desk prompt", w.What, node), true
|
||||
case w.Setting != "":
|
||||
act = fmt.Sprintf("approve %s of %s on %s when it is proposed to you", w.Setting, module, node)
|
||||
}
|
||||
if act != "" && !seen[act] {
|
||||
seen[act] = true
|
||||
acts = append(acts, act)
|
||||
}
|
||||
}
|
||||
needs := strings.Join(acts, "; and ") + "."
|
||||
// Plain words hold one sentence of at most conditions.NeedsMax characters: several acts are named in the
|
||||
// summary instead.
|
||||
if len(acts) == 0 || len(needs) > conditions.NeedsMax {
|
||||
needs = fmt.Sprintf("give what %s waits for on %s; the details name each secret and setting.", module, node)
|
||||
}
|
||||
explanation := fmt.Sprintf("Part of %s on %s cannot work until you give what it waits for.", module, node)
|
||||
if secret {
|
||||
explanation += " A hidden prompt opens at the desk when the secret is asked for, and what you type there " +
|
||||
"is sealed to the machine."
|
||||
}
|
||||
explanation += " Its update is in place and nothing was undone; it carries on by itself once it is given."
|
||||
return words{
|
||||
Headline: fmt.Sprintf("%s waits for you on %s", module, node),
|
||||
Needs: needs,
|
||||
Explanation: explanation,
|
||||
Resolved: fmt.Sprintf("%s on %s no longer waits for you", module, node),
|
||||
}
|
||||
}
|
||||
|
||||
// readWaitFacts reads what the controller holds to check the waits of the modules named on one machine: the
|
||||
// manifests (the catalogue's, or those given) and the secrets given there. A read that fails leaves that module
|
||||
// unknown, so none of its waits is excused.
|
||||
func readWaitFacts(ctx context.Context, inv *inventory.Inventory, machine string, modules []string,
|
||||
manifests map[string]catalogue.Manifest, f *operatorWaitFacts) {
|
||||
if f.manifests == nil {
|
||||
f.manifests = map[string]catalogue.Manifest{}
|
||||
}
|
||||
if f.given == nil {
|
||||
f.given = map[string]map[string]time.Time{}
|
||||
}
|
||||
if inv == nil {
|
||||
return
|
||||
}
|
||||
var shelf map[string]catalogue.Manifest
|
||||
sort.Strings(modules)
|
||||
for _, module := range modules {
|
||||
if _, has := f.manifests[module]; !has {
|
||||
if m, given := manifests[module]; given {
|
||||
f.manifests[module] = m
|
||||
} else {
|
||||
if shelf == nil {
|
||||
var err error
|
||||
if shelf, err = inv.Catalogue(ctx); err != nil {
|
||||
shelf = map[string]catalogue.Manifest{}
|
||||
}
|
||||
}
|
||||
if m, known := shelf[module]; known {
|
||||
f.manifests[module] = m
|
||||
}
|
||||
}
|
||||
}
|
||||
if _, read := f.given[module+"@"+machine]; read {
|
||||
continue
|
||||
}
|
||||
if given, err := inv.GivenOwnSecrets(ctx, machine, module); err == nil {
|
||||
f.given[module+"@"+machine] = given
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// waitingModules is every module with a waiting resource in a statement.
|
||||
func waitingModules(rs []inventory.ResourceHealth) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, r := range rs {
|
||||
if r.State == link.StateWaiting && r.Module != "" && !seen[r.Module] {
|
||||
seen[r.Module] = true
|
||||
out = append(out, r.Module)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A part that waits for the operator's secret or setting (novox/hq ADR 0283, issue 386).
|
||||
|
||||
var mountsManifest = catalogue.Manifest{Module: "mounts", Version: "1",
|
||||
Settings: map[string]catalogue.SettingDeclaration{"smb-users": {}, "sources": {}},
|
||||
OwnSecrets: catalogue.OwnSecrets{
|
||||
"smb-password-*": {Path: "/s/smb-password-*.secret", IssuedBy: catalogue.IssuedOutside},
|
||||
"broker": {Path: "/s/broker"},
|
||||
"made": {Path: "/s/made", Taken: catalogue.TakenAtStart},
|
||||
"licence": {Path: "/s/licence", IssuedBy: catalogue.IssuedOutside},
|
||||
}}
|
||||
|
||||
var passwordWait = inventory.Wait{Part: "the source games", Secret: "smb-password-games",
|
||||
What: "the password of the source games"}
|
||||
|
||||
var usernameWait = inventory.Wait{Part: "the source games", Setting: "smb-users", What: "the username of the source games"}
|
||||
|
||||
func waitingResource(waits ...inventory.Wait) inventory.ResourceHealth {
|
||||
return inventory.ResourceHealth{Module: "mounts", Resource: "mounts.watch", Kind: "process",
|
||||
Target: "mesh-mounts-watch.service", State: link.StateWaiting, Check: "tool",
|
||||
Reason: "the source games waits for its password", Waits: waits}
|
||||
}
|
||||
|
||||
func factsGiven(given map[string]time.Time) operatorWaitFacts {
|
||||
return operatorWaitFacts{manifests: map[string]catalogue.Manifest{"mounts": mountsManifest},
|
||||
given: map[string]map[string]time.Time{"mounts@workstation": given}}
|
||||
}
|
||||
|
||||
// Rule 3: a wait is excused only when what it names is the module's, issued outside the mesh, and not given there.
|
||||
func TestAWaitIsExcusedOnlyWhenItChecksOut(t *testing.T) {
|
||||
at := time.Date(2026, 10, 10, 15, 8, 0, 0, time.UTC)
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
w inventory.Wait
|
||||
f operatorWaitFacts
|
||||
says string // "" when excused
|
||||
}{
|
||||
{"a member of an outside family, not given", passwordWait, factsGiven(nil), ""},
|
||||
{"an outside secret by name, not given", inventory.Wait{Part: "p", Secret: "licence", What: "w"}, factsGiven(nil), ""},
|
||||
{"a declared setting", usernameWait, factsGiven(nil), ""},
|
||||
{"a member given", passwordWait, factsGiven(map[string]time.Time{"smb-password-games": at}), "was given at 2026-10-10 15:08"},
|
||||
{"a secret not declared", inventory.Wait{Part: "p", Secret: "smb-credentials", What: "w"}, factsGiven(nil), "does not declare"},
|
||||
{"a secret the mesh makes", inventory.Wait{Part: "p", Secret: "made", What: "w"}, factsGiven(nil), "mesh makes itself"},
|
||||
{"the bus account", inventory.Wait{Part: "p", Secret: "broker", What: "w"}, factsGiven(nil), "mesh makes itself"},
|
||||
{"a setting not declared", inventory.Wait{Part: "p", Setting: "logins", What: "w"}, factsGiven(nil), "does not declare"},
|
||||
{"both", inventory.Wait{Part: "p", Secret: "licence", Setting: "smb-users", What: "w"}, factsGiven(nil), "one secret or one setting"},
|
||||
{"neither", inventory.Wait{Part: "p", What: "w"}, factsGiven(nil), "one secret or one setting"},
|
||||
{"no manifest known", passwordWait, operatorWaitFacts{}, "no manifest"},
|
||||
{"what was given cannot be read", passwordWait,
|
||||
operatorWaitFacts{manifests: map[string]catalogue.Manifest{"mounts": mountsManifest}}, "could not be read"},
|
||||
} {
|
||||
err := checkWait("mounts", "workstation", c.w, c.f)
|
||||
switch {
|
||||
case c.says == "" && err != nil:
|
||||
t.Errorf("%s: refused: %v", c.name, err)
|
||||
case c.says != "" && (err == nil || !strings.Contains(err.Error(), c.says)):
|
||||
t.Errorf("%s: %v; want a refusal saying %q", c.name, err, c.says)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A waiting resource whose wait does not check out, or that names nothing, is judged unhealthy, saying why.
|
||||
func TestAWaitThatFailsItsCheckIsUnhealthy(t *testing.T) {
|
||||
given := factsGiven(map[string]time.Time{"smb-password-games": time.Now()})
|
||||
got := checkWaiting("workstation", []inventory.ResourceHealth{waitingResource(passwordWait)}, given)
|
||||
if got[0].State != link.StateUnhealthy || !strings.Contains(got[0].Reason, "was given") {
|
||||
t.Fatalf("a wait for a secret given: %+v", got[0])
|
||||
}
|
||||
got = checkWaiting("workstation", []inventory.ResourceHealth{waitingResource()}, factsGiven(nil))
|
||||
if got[0].State != link.StateUnhealthy || !strings.Contains(got[0].Reason, "names nothing") {
|
||||
t.Fatalf("a wait naming nothing: %+v", got[0])
|
||||
}
|
||||
got = checkWaiting("workstation", []inventory.ResourceHealth{waitingResource(passwordWait, usernameWait)}, factsGiven(nil))
|
||||
if got[0].State != link.StateWaiting {
|
||||
t.Fatalf("two waits that check out: %+v", got[0])
|
||||
}
|
||||
}
|
||||
|
||||
// Rule 4: the gate passes a module whose only parts not healthy wait for the operator, carrying the wait; anything
|
||||
// else beside it is judged as before; and any build is excused, not only one that added something.
|
||||
func TestTheGatePassesAWaitForTheOperatorCarriedAlong(t *testing.T) {
|
||||
now := time.Now()
|
||||
since := now.Add(-time.Minute)
|
||||
healthy := inventory.ResourceHealth{Module: "mounts", Resource: "mounts.apply", Kind: "process",
|
||||
Target: "mesh-mounts-apply.service", State: link.StateHealthy}
|
||||
f := gateFacts{now: now, waits: factsGiven(nil), groupsAdded: map[string]bool{"mounts": false},
|
||||
health: map[string]inventory.NodeHealth{"workstation": {Node: "workstation", HeardAt: now,
|
||||
Resources: []inventory.ResourceHealth{healthy, waitingResource(passwordWait)}}}}
|
||||
h, why := moduleHealthWord("mounts", "workstation", since, f)
|
||||
if h != healthPerson || !strings.Contains(why, "waits for the operator: the password of the source games") ||
|
||||
!strings.Contains(why, "nox secret ask workstation mounts smb-password-games") {
|
||||
t.Fatalf("an excused wait reads %v %q; want a wait for a person naming the act", h, why)
|
||||
}
|
||||
// A second resource unhealthy beside it: not yet, as before.
|
||||
down := healthy
|
||||
down.State, down.Reason = link.StateUnhealthy, "down"
|
||||
f.health["workstation"] = inventory.NodeHealth{Node: "workstation", HeardAt: now,
|
||||
Resources: []inventory.ResourceHealth{down, waitingResource(passwordWait)}}
|
||||
if h, why := moduleHealthWord("mounts", "workstation", since, f); h != healthNotYet {
|
||||
t.Fatalf("a resource down beside the wait reads %v %q", h, why)
|
||||
}
|
||||
// The password given and the module still saying it waits: not excused.
|
||||
f.waits = factsGiven(map[string]time.Time{"smb-password-games": now})
|
||||
f.health["workstation"] = inventory.NodeHealth{Node: "workstation", HeardAt: now,
|
||||
Resources: []inventory.ResourceHealth{healthy, waitingResource(passwordWait)}}
|
||||
if h, why := moduleHealthWord("mounts", "workstation", since, f); h != healthNotYet || !strings.Contains(why, "was given") {
|
||||
t.Fatalf("a wait for a secret given reads %v %q", h, why)
|
||||
}
|
||||
// Facts never read (no manifest): never excused.
|
||||
f.waits = operatorWaitFacts{}
|
||||
if h, _ := moduleHealthWord("mounts", "workstation", since, f); h != healthNotYet {
|
||||
t.Fatalf("a wait nothing could check reads %v", h)
|
||||
}
|
||||
}
|
||||
|
||||
func needsOperatorOpen(t *testing.T, k *conditions.Keeper) (*conditions.Condition, []conditions.Condition) {
|
||||
t.Helper()
|
||||
open, err := k.Open(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, c := range open {
|
||||
if c.Key == needsOperatorKey("mounts", "workstation") {
|
||||
return &open[i], open
|
||||
}
|
||||
}
|
||||
return nil, open
|
||||
}
|
||||
|
||||
// Rule 5: two statements of an excused wait raise needs-operator, the operator's, a warning however long, naming the
|
||||
// act; a statement without it clears it.
|
||||
func TestTheNeedsOperatorConditionNamesTheAct(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
ctx := t.Context()
|
||||
rs := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}}
|
||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 1}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, _ := needsOperatorOpen(t, k); got != nil {
|
||||
t.Fatal("raised on one statement")
|
||||
}
|
||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 2}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, open := needsOperatorOpen(t, k)
|
||||
if got == nil {
|
||||
t.Fatalf("not raised on two statements: %+v", open)
|
||||
}
|
||||
for _, c := range open {
|
||||
if c.Kind == kindModuleUnhealthy {
|
||||
t.Fatalf("raised as a fault too: %+v", c)
|
||||
}
|
||||
}
|
||||
if got.Kind != kindNeedsOperator || got.Resolver != conditions.ResolverOperator || got.Severity != conditions.Warning {
|
||||
t.Fatalf("the condition: %+v", got)
|
||||
}
|
||||
if !strings.Contains(got.Needs, "type the password of the source games at workstation's desk prompt") ||
|
||||
!strings.Contains(got.Explanation, "hidden prompt opens at the desk") {
|
||||
t.Fatalf("its needs do not name the act: %q", got.Needs)
|
||||
}
|
||||
if strings.Contains(strings.ToLower(got.Explanation), "nothing for you") || !strings.Contains(got.Explanation, "nothing was undone") {
|
||||
t.Fatalf("its explanation: %q", got.Explanation)
|
||||
}
|
||||
if !strings.Contains(got.Summary, "smb-password-games") || !strings.Contains(got.Summary, "nox secret ask workstation mounts smb-password-games") {
|
||||
t.Fatalf("its summary does not name the secret and the line: %q", got.Summary)
|
||||
}
|
||||
// Long open is still a warning: only the operator can end it.
|
||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 3}, time.Now().Add(48*time.Hour)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, _ := needsOperatorOpen(t, k); got == nil || got.Severity == conditions.Urgent {
|
||||
t.Fatalf("after two days: %+v", got)
|
||||
}
|
||||
// Given: the next statement does not say it, and it clears.
|
||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, _ := needsOperatorOpen(t, k); got != nil {
|
||||
t.Fatal("not cleared once given")
|
||||
}
|
||||
}
|
||||
|
||||
func TestASettingsWaitAsksForTheApproval(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
ctx := t.Context()
|
||||
rs := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(usernameWait)}}
|
||||
for i := 1; i <= 2; i++ {
|
||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": i}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, _ := needsOperatorOpen(t, k)
|
||||
if got == nil || !strings.Contains(got.Needs, "approve smb-users of mounts on workstation when it is proposed to you") {
|
||||
t.Fatalf("the condition: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A wait that fails its check is the module's own fault: unhealthy, with why, and no needs-operator.
|
||||
func TestAWaitThatFailsItsCheckRaisesUnhealthy(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
ctx := t.Context()
|
||||
checked := checkWaiting("workstation", []inventory.ResourceHealth{waitingResource(passwordWait)},
|
||||
factsGiven(map[string]time.Time{"smb-password-games": time.Now()}))
|
||||
rs := map[string][]inventory.ResourceHealth{"mounts": checked}
|
||||
for i := 1; i <= 2; i++ {
|
||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": i}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, open := needsOperatorOpen(t, k)
|
||||
if got != nil {
|
||||
t.Fatalf("a wait for a secret given raised needs-operator: %+v", got)
|
||||
}
|
||||
unhealthy := false
|
||||
for _, c := range open {
|
||||
unhealthy = unhealthy || c.Key == moduleUnhealthyKey("mounts", "workstation")
|
||||
}
|
||||
if !unhealthy {
|
||||
t.Fatalf("not raised as unhealthy: %+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
// A module that waited and is then broken says so when the wait clears, and the other way round.
|
||||
func TestANeedsOperatorThatBecameUnhealthySaysSo(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
ctx := t.Context()
|
||||
waiting := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}}
|
||||
for i := 1; i <= 2; i++ {
|
||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", waiting, map[string]int{"mounts": i}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
broken := waitingResource()
|
||||
broken.State, broken.Reason, broken.Waits = link.StateUnhealthy, "the source games refused its login", nil
|
||||
for i := 3; i <= 4; i++ {
|
||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": {broken}},
|
||||
map[string]int{"mounts": i}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, open := needsOperatorOpen(t, k)
|
||||
if got != nil {
|
||||
t.Fatal("needs-operator still open after it became a fault")
|
||||
}
|
||||
found := false
|
||||
for _, c := range open {
|
||||
found = found || c.Key == moduleUnhealthyKey("mounts", "workstation")
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("the fault is not raised: %+v", open)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A delivery over its budget is loud (novox/hq ADR 0282 decision 7, issue 382): the self-check reads
|
||||
// mesh-delivery's `times` and raises the warning `delivery.<class>.over-budget` when the newest delivery of a
|
||||
// class whose delivery time is known took longer than its class's budget — five minutes for a leaf module, ten
|
||||
// for a core module — naming the delivery and its longest phase. It clears when the next delivery of that class
|
||||
// lands within its budget. Measurement only: the condition says, it never holds or acts on a delivery.
|
||||
|
||||
// probeBudgetsID is the probe that reads the delivery times.
|
||||
const probeBudgetsID = "D16"
|
||||
|
||||
// kindOverBudget is what a delivery over its class's budget raises.
|
||||
const kindOverBudget = "over-budget"
|
||||
|
||||
// deliveryBudgets are the budgets by class (ADR 0282 decision 1); the probe raises nothing for another class.
|
||||
var deliveryBudgets = map[string]time.Duration{inventory.ClassLeaf: 5 * time.Minute, inventory.ClassCore: 10 * time.Minute}
|
||||
|
||||
// timesAnswer is what mesh-delivery's `times` answers, as far as the probe reads it.
|
||||
type timesAnswer struct {
|
||||
Classes []timesClass `json:"classes"`
|
||||
}
|
||||
|
||||
// timesClass is one class's line of `times`.
|
||||
type timesClass struct {
|
||||
Class string `json:"class"`
|
||||
Latest *timesLatest `json:"latest,omitempty"`
|
||||
}
|
||||
|
||||
// timesLatest is the newest delivery of a class whose delivery time is known.
|
||||
type timesLatest struct {
|
||||
ID string `json:"id"`
|
||||
TookMS int64 `json:"took_ms"`
|
||||
Longest string `json:"longest,omitempty"`
|
||||
Landed string `json:"landed,omitempty"`
|
||||
}
|
||||
|
||||
// deliveryTimes is what the delivery's owner says of its delivery times; nothing when no holder is on record or
|
||||
// none answers (D3 says that one).
|
||||
func deliveryTimes(ctx context.Context, conn *nats.Conn, held bool) (*timesAnswer, error) {
|
||||
if !held {
|
||||
return nil, nil
|
||||
}
|
||||
raw, err := askDeliveryOwner(ctx, conn, "times", map[string]any{})
|
||||
if errors.Is(err, link.ErrNothingServes) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var a timesAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("%s.times answered something unreadable: %w", catalogue.DeliverySeat, err)
|
||||
}
|
||||
return &a, nil
|
||||
}
|
||||
|
||||
// overBudgetObservations are the conditions of the classes whose newest delivery took longer than its budget:
|
||||
// strictly longer, so a delivery of exactly its budget is within it.
|
||||
func overBudgetObservations(a *timesAnswer) []conditions.Observation {
|
||||
if a == nil {
|
||||
return nil
|
||||
}
|
||||
var out []conditions.Observation
|
||||
for _, c := range a.Classes {
|
||||
budget, ok := deliveryBudgets[c.Class]
|
||||
if !ok || c.Latest == nil {
|
||||
continue
|
||||
}
|
||||
took := time.Duration(c.Latest.TookMS) * time.Millisecond
|
||||
if took <= budget {
|
||||
continue
|
||||
}
|
||||
longest := c.Latest.Longest
|
||||
if longest == "" {
|
||||
longest = "not known"
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeDelivery, ID: c.Class, Kind: kindOverBudget,
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the %s delivery %s took %s from its merge to running everywhere, over its budget of %s; "+
|
||||
"its longest phase: %s — `mesh-delivery.times`", c.Class, c.Latest.ID, humanDuration(took),
|
||||
humanDuration(budget), longest),
|
||||
Said: fmt.Sprintf("%s took %s (budget %s), longest phase %s", c.Latest.ID, took.Round(time.Second), budget, longest),
|
||||
Headline: fmt.Sprintf("A %s delivery took %s, over its %s budget", c.Class, humanDuration(took),
|
||||
humanDuration(budget)),
|
||||
Explanation: fmt.Sprintf("The delivery %s took %s from its merge until every machine ran it; a %s module is "+
|
||||
"held to %s (ADR 0282). Most of the time went to %s. Nothing was held or changed because of this: it is "+
|
||||
"a measurement.", c.Latest.ID, humanDuration(took), c.Class, humanDuration(budget), longest),
|
||||
Resolved: fmt.Sprintf("the next %s delivery lands within %s", c.Class, humanDuration(budget)),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// probeBudgets is D16: the newest delivery of each class lands within its class's budget.
|
||||
func probeBudgets(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
entries, err := d.open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var conn *nats.Conn
|
||||
if d.js != nil {
|
||||
conn = d.js.Conn()
|
||||
}
|
||||
a, err := deliveryTimes(ctx, conn, deliverySeatHeld(entries))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return overBudgetObservations(a), nil
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A leaf delivery of 5 minutes 10 seconds raises delivery.leaf.over-budget naming its longest phase; one of
|
||||
// exactly five minutes, a core one of nine and a class without a budget raise nothing (ADR 0282 decision 7).
|
||||
func TestADeliveryOverItsBudgetIsLoud(t *testing.T) {
|
||||
a := ×Answer{Classes: []timesClass{
|
||||
{Class: inventory.ClassLeaf, Latest: ×Latest{ID: "novox/mesh-catalog@abc", TookMS: (5*time.Minute + 10*time.Second).Milliseconds(),
|
||||
Longest: "judgement 2m40s"}},
|
||||
{Class: inventory.ClassCore, Latest: ×Latest{ID: "novox/mesh-controller@def", TookMS: (9 * time.Minute).Milliseconds(),
|
||||
Longest: "build 1m"}},
|
||||
{Class: "unclassed", Latest: ×Latest{ID: "x@y", TookMS: time.Hour.Milliseconds()}},
|
||||
}}
|
||||
obs := overBudgetObservations(a)
|
||||
if len(obs) != 1 {
|
||||
t.Fatalf("one class over its budget, got %d: %+v", len(obs), obs)
|
||||
}
|
||||
o := obs[0]
|
||||
if o.Key() != "delivery.leaf.over-budget" || !strings.Contains(o.Summary, "judgement 2m40s") ||
|
||||
!strings.Contains(o.Summary, "novox/mesh-catalog@abc") {
|
||||
t.Fatalf("the condition names its delivery and longest phase: %s — %s", o.Key(), o.Summary)
|
||||
}
|
||||
// The next leaf delivery within its budget clears it: the probe raises nothing for the class.
|
||||
a.Classes[0].Latest = ×Latest{ID: "novox/mesh-catalog@ghi", TookMS: (5 * time.Minute).Milliseconds()}
|
||||
if obs := overBudgetObservations(a); len(obs) != 0 {
|
||||
t.Fatalf("a delivery of exactly its budget is within it: %+v", obs)
|
||||
}
|
||||
a.Classes[1].Latest.TookMS = (10*time.Minute + time.Second).Milliseconds()
|
||||
if obs := overBudgetObservations(a); len(obs) != 1 || obs[0].Key() != "delivery.core.over-budget" {
|
||||
t.Fatalf("a core delivery over ten minutes: %+v", obs)
|
||||
}
|
||||
if obs := overBudgetObservations(nil); obs != nil {
|
||||
t.Fatal("no answer raises nothing")
|
||||
}
|
||||
// No delivery of a class with a known time yet: nothing said of it.
|
||||
if obs := overBudgetObservations(×Answer{Classes: []timesClass{{Class: inventory.ClassLeaf}}}); len(obs) != 0 {
|
||||
t.Fatalf("a class with no delivery: %+v", obs)
|
||||
}
|
||||
}
|
||||
|
||||
// The probe's question is one the controller's grant names: a question the bus refuses checks nothing.
|
||||
func TestTheControllerMayAskForTheDeliveryTimes(t *testing.T) {
|
||||
found := false
|
||||
for _, v := range broker.VerbsTheControllerAsksTheDeliveryOwner {
|
||||
found = found || (v.Seat == catalogue.DeliverySeat && v.Verb == "times")
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("the grant does not name mesh-delivery.times")
|
||||
}
|
||||
}
|
||||
|
||||
// A walk's class is core when it walks a module of the controller's own path or one holding the mesh's resolver,
|
||||
// leaf otherwise; its window closed at its batch's window, or its maximum, never after its cut.
|
||||
func TestAWalksClassAndWindowAreReadAtItsCut(t *testing.T) {
|
||||
entries := []inventory.Entry{
|
||||
{Manifest: catalogue.Manifest{Module: "dnsmasq", Claims: []catalogue.Claim{{Name: resolverSeat}}}},
|
||||
{Manifest: catalogue.Manifest{Module: "gitea"}},
|
||||
}
|
||||
walk := func(modules ...string) inventory.Plan {
|
||||
p := inventory.Plan{Modules: map[string]*inventory.PlanModule{}}
|
||||
for _, m := range modules {
|
||||
p.Modules[m] = &inventory.PlanModule{}
|
||||
}
|
||||
return p
|
||||
}
|
||||
for want, w := range map[string]inventory.Plan{
|
||||
inventory.ClassLeaf: walk("gitea"), inventory.ClassCore: walk("gitea", "mesh-controller"),
|
||||
} {
|
||||
if got := classOf(w, entries); got != want {
|
||||
t.Errorf("%v: %s, want %s", w.Modules, got, want)
|
||||
}
|
||||
}
|
||||
if got := classOf(walk("dnsmasq"), entries); got != inventory.ClassCore {
|
||||
t.Errorf("the resolver's holder is core: %s", got)
|
||||
}
|
||||
now := time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC)
|
||||
batch := inventory.Plan{Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{
|
||||
ClosesAt: now.Add(-20 * time.Second), AtMost: now.Add(5 * time.Minute)}}}
|
||||
times := walkTimesAtCut(batch, walk("gitea"), entries, now)
|
||||
if times.Cut == nil || !times.Cut.Equal(now) || times.WindowClosed == nil || !times.WindowClosed.Equal(now.Add(-20*time.Second)) ||
|
||||
times.Class != inventory.ClassLeaf {
|
||||
t.Fatalf("times at the cut: %+v", times)
|
||||
}
|
||||
batch.Delivery.Batch.AtMost = now.Add(-time.Minute)
|
||||
if times := walkTimesAtCut(batch, walk("gitea"), entries, now); !times.WindowClosed.Equal(now.Add(-time.Minute)) {
|
||||
t.Fatalf("a window closed at its maximum: %v", times.WindowClosed)
|
||||
}
|
||||
if times := walkTimesAtCut(inventory.Plan{Delivery: &inventory.PlanDelivery{Alone: true}}, walk("gitea"), entries, now); times.WindowClosed != nil {
|
||||
t.Fatalf("a merge walked alone had no window: %v", times.WindowClosed)
|
||||
}
|
||||
}
|
||||
|
||||
// What each machine of the rest was sent is kept only for the machines the send reached.
|
||||
func TestTheRestIsKeptForTheMachinesTheSendReached(t *testing.T) {
|
||||
got := restOf([]string{"ace", "g14"}, []string{"ace", "shanks"}, map[string]inventory.SentDeclaration{"ace": {Digest: "d1"}})
|
||||
if len(got) != 1 || got["ace"].Digest != "d1" {
|
||||
t.Fatalf("rest: %+v", got)
|
||||
}
|
||||
if restOf([]string{"g14"}, []string{"ace"}, nil) != nil {
|
||||
t.Fatal("no machine reached: nothing kept")
|
||||
}
|
||||
}
|
||||
@@ -221,6 +221,16 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
||||
w := usedAsFoundObservation(orModule(module), machineOr(o, "a machine"), o.Summary, nil)
|
||||
return words{Headline: w.Headline, Explanation: w.Explanation, Needs: w.Needs, Resolved: w.Resolved}
|
||||
}),
|
||||
kindNeedsOperator: worded(func(o conditions.Observation) words {
|
||||
// The observation carries the act itself (ADR 0283); these are its words when only the kind is known.
|
||||
module := ""
|
||||
if o.Scope == conditions.ScopeModule && o.Machine != "" {
|
||||
module = strings.TrimSuffix(o.ID, "."+o.Machine)
|
||||
}
|
||||
node := machineOr(o, "a machine")
|
||||
w := needsOperatorWords(orModule(module), node, nil)
|
||||
return w
|
||||
}),
|
||||
kindProviderFailing: worded(func(o conditions.Observation) words {
|
||||
thing, consumer := conditions.ThingWords(o), idPart(o, 2)
|
||||
if consumer == "" {
|
||||
@@ -818,6 +828,9 @@ func stalledWords(l stalledLine, o conditions.Observation) (headline, explanatio
|
||||
"to check it: the forge never announced it. It cannot merge until it is checked.", pull, long),
|
||||
fmt.Sprintf("%s has a merge check now, or is closed", pull), needs, nil
|
||||
}
|
||||
if l.State == "unanswered" {
|
||||
return unansweredWords(l, o)
|
||||
}
|
||||
held := l.State
|
||||
if held == "" {
|
||||
held = "held"
|
||||
@@ -847,6 +860,74 @@ func stalledWords(l stalledLine, o conditions.Observation) (headline, explanatio
|
||||
fmt.Sprintf("Delivery of %s is no longer %s", name, held), needs, actions
|
||||
}
|
||||
|
||||
// unansweredWords are the plain words of a pull request's head whose merge check was asked and has not answered
|
||||
// within its bound (novox/hq issue 438): mesh-delivery holds no delivery of it, so there is nothing to stop, release
|
||||
// or close, and no action is offered. The operator's acts are a new commit, which asks the check again, or a look
|
||||
// at the build queue, where a check that never ran may still wait.
|
||||
func unansweredWords(l stalledLine, o conditions.Observation) (headline, explanation, resolved, needs string,
|
||||
actions []conditions.Action) {
|
||||
repository, _, _ := strings.Cut(l.ID, "@")
|
||||
pull := "A pull request of " + repoName(repository)
|
||||
if l.Number > 0 {
|
||||
pull = fmt.Sprintf("Pull request %s #%d", repoName(repository), l.Number)
|
||||
}
|
||||
long, limit := "for too long", ""
|
||||
if d, err := time.ParseDuration(l.For); err == nil {
|
||||
long = "for " + humanDuration(d)
|
||||
}
|
||||
if d, err := time.ParseDuration(l.Bound); err == nil {
|
||||
limit = ", past its limit of " + humanDuration(d)
|
||||
}
|
||||
which := "Its merge check"
|
||||
if said := uncheckedWaitWords(l); len(said) > 0 {
|
||||
which = "Its merge check (" + strings.Join(said, ", ") + ")"
|
||||
}
|
||||
if o.Resolver == conditions.ResolverOperator {
|
||||
needs = "push a new commit to its branch, which asks the check again, or see whether its check still waits " +
|
||||
"in the build queue: mesh-controller.queue."
|
||||
}
|
||||
return fmt.Sprintf("%s's merge check has not answered %s", pull, long),
|
||||
fmt.Sprintf("%s is open on a branch that requires the merge check. %s was asked and has not answered %s%s. "+
|
||||
"It cannot merge until its check answers.", pull, which, long, limit),
|
||||
fmt.Sprintf("%s has an answer from its merge check, or is closed", pull), needs, nil
|
||||
}
|
||||
|
||||
// uncheckedWaitWords are the merge checks an unanswered line waits on, as the operator reads them: from the checks
|
||||
// given as data, each time in the controller's local zone, as every other time in a message — "mesh/merge-gate
|
||||
// pending since 02:11" — never the UTC time inside mesh-delivery's finished words, which cannot be said again in
|
||||
// another zone (novox/hq issue 443). A line from a mesh-delivery that gives no such data is said by its words.
|
||||
func uncheckedWaitWords(l stalledLine) []string {
|
||||
if len(l.Checks) == 0 {
|
||||
return l.Waiting
|
||||
}
|
||||
out := make([]string, 0, len(l.Checks))
|
||||
for _, c := range l.Checks {
|
||||
switch c.State {
|
||||
case "never-set":
|
||||
out = append(out, c.Context+" never set")
|
||||
case "pending":
|
||||
out = append(out, c.Context+" pending"+sinceWords(c.Since))
|
||||
default:
|
||||
out = append(out, c.Context+" not answered")
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sinceWords is " since 02:11" in the controller's local zone, with its day when that is not today ("since 23:05 on
|
||||
// 9 Oct"), so the time stays absolute; a time not given, or not readable, is said so and never made up.
|
||||
func sinceWords(since string) string {
|
||||
at, err := time.Parse(time.RFC3339, since)
|
||||
if err != nil {
|
||||
return ", since a time the forge did not say"
|
||||
}
|
||||
local, today := at.In(wordsZone()), wordsNow().In(wordsZone())
|
||||
if local.YearDay() == today.YearDay() && local.Year() == today.Year() {
|
||||
return " since " + local.Format("15:04")
|
||||
}
|
||||
return " since " + local.Format("15:04 on 2 Jan")
|
||||
}
|
||||
|
||||
// causeWords is a hand-act's cause as a person says it.
|
||||
func causeWords(cause string) string {
|
||||
return strings.NewReplacer(".", " ", "_", " ").Replace(cause)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -347,3 +348,107 @@ func TestAnUnannouncedPullRequestSaysToPushANewCommit(t *testing.T) {
|
||||
t.Fatalf("it reads %q / %q", o.Headline, o.Explanation)
|
||||
}
|
||||
}
|
||||
|
||||
// **A pull request whose merge check never answered says which check, since when, and what to do** (novox/hq issue
|
||||
// 438): mesh-delivery says one as a stalled line in state `unanswered`, the line below exactly as its test makes it.
|
||||
// No delivery of the head is held, so there is nothing to stop, release or close: no action is offered, and the
|
||||
// operator's acts are a new commit, which asks the check again, or a look at the build queue.
|
||||
func TestAnUnansweredMergeCheckSaysWhichCheckAndWhatToDo(t *testing.T) {
|
||||
var l stalledLine
|
||||
if err := json.Unmarshal([]byte(`{"id":"novox/mesh-controller@c11222026a3b","number":212,"state":"unanswered",`+
|
||||
`"waiting":["mesh/merge-gate pending since 2026-10-11T00:11:39Z","mesh/repo-check never set"],"for":"1h1m0s",`+
|
||||
`"bound":"1h0m0s","h2":"none: no delivery of it is held here, so there is none to close — the operator's",`+
|
||||
`"says":"novox/mesh-controller#212 is open on main, which requires the merge check, and its head's check `+
|
||||
`started and never answered: mesh/merge-gate pending since 2026-10-11T00:11:39Z, mesh/repo-check never set. `+
|
||||
`No delivery of it is held here, so nothing asks it again. A new commit on its branch announces it and asks `+
|
||||
`its check"}`), &l); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
obs := stalledObservations([]stalledLine{l})
|
||||
if len(obs) != 1 || obs[0].Resolver != conditions.ResolverOperator {
|
||||
t.Fatalf("an unanswered merge check is not the operator's: %+v", obs)
|
||||
}
|
||||
o := obs[0]
|
||||
t.Logf("headline: %s\nexplanation: %s\nneeds: %s\nresolved: %s", o.Headline, o.Explanation, o.Needs, o.Resolved)
|
||||
if len(o.Actions) != 0 {
|
||||
t.Fatalf("it offers an action a head with no delivery cannot take: %+v", o.Actions)
|
||||
}
|
||||
if strings.Contains(o.Needs, "stop") || strings.Contains(o.Needs, "release") || !strings.Contains(o.Needs, "commit") ||
|
||||
!strings.Contains(o.Needs, "in the build queue: mesh-controller.queue") || strings.Contains(o.Needs, "build seat") {
|
||||
t.Fatalf("it says to %q", o.Needs)
|
||||
}
|
||||
if o.Headline != "Pull request mesh-controller #212's merge check has not answered for 61 minutes" {
|
||||
t.Fatalf("its headline reads %q", o.Headline)
|
||||
}
|
||||
for _, want := range []string{"mesh/merge-gate pending since 2026-10-11T00:11:39Z", "mesh/repo-check never set",
|
||||
"past its limit of 60 minutes"} {
|
||||
if !strings.Contains(o.Explanation, want) {
|
||||
t.Fatalf("its explanation does not say %q: %q", want, o.Explanation)
|
||||
}
|
||||
}
|
||||
if strings.Contains(o.Explanation, "never asked") || strings.Contains(o.Explanation, "never announced") {
|
||||
t.Fatalf("it says the mesh was never asked, of a head whose check started: %q", o.Explanation)
|
||||
}
|
||||
|
||||
// A line from a mesh-delivery that names no checks still says the check did not answer, and offers nothing.
|
||||
var bare stalledLine
|
||||
if err := json.Unmarshal([]byte(`{"id":"novox/mesh-controller@c11222026a3b","number":212,"state":"unanswered",`+
|
||||
`"for":"1h1m0s","bound":"1h0m0s","h2":"none: no delivery of it is held here, so there is none to close — the operator's"}`),
|
||||
&bare); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
o = stalledObservations([]stalledLine{bare})[0]
|
||||
if len(o.Actions) != 0 || !strings.Contains(o.Explanation, "has not answered") || strings.Contains(o.Headline, "Delivery of") {
|
||||
t.Fatalf("a line naming no checks reads %q / %q, actions %+v", o.Headline, o.Explanation, o.Actions)
|
||||
}
|
||||
}
|
||||
|
||||
// **An unanswered merge check's time reaches the operator in their own time, never as raw UTC** (novox/hq issue
|
||||
// 443): mesh-delivery says each check waited on as data — its context, its state and since when, in RFC 3339 — beside
|
||||
// the finished words it gave before, and the controller words it in its local zone, as every other time in a message.
|
||||
// The line is the one mesh-delivery says, as in the report of issue 443, with the checks it now carries.
|
||||
func TestAnUnansweredMergeChecksTimeIsSaidInLocalTime(t *testing.T) {
|
||||
// The operator's zone given through the seam, never by writing time.Local: other tests' goroutines read it, and
|
||||
// the build seat runs the suite under the race detector.
|
||||
wasZone, wasNow := wordsZone, wordsNow
|
||||
wordsZone = func() *time.Location { return time.FixedZone("CEST", 2*60*60) }
|
||||
wordsNow = func() time.Time { return time.Date(2026, 10, 11, 1, 12, 39, 0, time.UTC) }
|
||||
t.Cleanup(func() { wordsZone, wordsNow = wasZone, wasNow })
|
||||
|
||||
var l stalledLine
|
||||
if err := json.Unmarshal([]byte(`{"id":"novox/mesh-controller@c11222026a3b","number":212,"state":"unanswered",`+
|
||||
`"waiting":["mesh/merge-gate pending since 2026-10-11T00:11:39Z","mesh/repo-check never set"],`+
|
||||
`"checks":[{"context":"mesh/merge-gate","state":"pending","since":"2026-10-11T00:11:39Z"},`+
|
||||
`{"context":"mesh/repo-check","state":"never-set"}],"for":"1h1m0s",`+
|
||||
`"bound":"1h0m0s","h2":"none: no delivery of it is held here, so there is none to close — the operator's",`+
|
||||
`"says":"novox/mesh-controller#212 is open on main, which requires the merge check, and its head's check `+
|
||||
`started and never answered: mesh/merge-gate pending since 2026-10-11T00:11:39Z, mesh/repo-check never set. `+
|
||||
`No delivery of it is held here, so nothing asks it again. A new commit on its branch announces it and asks `+
|
||||
`its check"}`), &l); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
o := stalledObservations([]stalledLine{l})[0]
|
||||
t.Logf("explanation: %s", o.Explanation)
|
||||
for _, raw := range []string{"2026-10-11T00:11:39Z", "00:11", "UTC"} {
|
||||
if strings.Contains(o.Explanation, raw) || strings.Contains(o.Headline, raw) {
|
||||
t.Fatalf("the operator reads %q: %q / %q", raw, o.Headline, o.Explanation)
|
||||
}
|
||||
}
|
||||
for _, want := range []string{"mesh/merge-gate pending since 02:11", "mesh/repo-check never set"} {
|
||||
if !strings.Contains(o.Explanation, want) {
|
||||
t.Fatalf("its explanation does not say %q: %q", want, o.Explanation)
|
||||
}
|
||||
}
|
||||
|
||||
// A check pending since another day than today says which day, so the time stays absolute.
|
||||
l.Checks[0].Since = "2026-10-09T21:05:00Z"
|
||||
if o = stalledObservations([]stalledLine{l})[0]; !strings.Contains(o.Explanation, "mesh/merge-gate pending since 23:05 on 9 Oct") {
|
||||
t.Fatalf("a check pending since an earlier day reads %q", o.Explanation)
|
||||
}
|
||||
|
||||
// A pending check whose time the forge did not say is said so, with no time made up.
|
||||
l.Checks[0].Since = ""
|
||||
if o = stalledObservations([]stalledLine{l})[0]; !strings.Contains(o.Explanation, "mesh/merge-gate pending, since a time the forge did not say") {
|
||||
t.Fatalf("a pending check without its time reads %q", o.Explanation)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -664,7 +664,33 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
needed := map[string]map[string]string{}
|
||||
foreseen := map[string]map[string]bool{}
|
||||
for _, m := range plan.Modules {
|
||||
for name := range m.OwnSecrets {
|
||||
// **A secret family is never made** (novox/hq ADR 0283): each member a person gave on this machine is
|
||||
// placed, and one not given is nothing — the module says it waits for it.
|
||||
for _, family := range m.OwnSecrets.Families() {
|
||||
members, err := inv.GivenMembers(ctx, node, m.Module, family)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
for _, g := range members {
|
||||
if _, fam, ok := m.OwnSecrets.Lookup(g.Name); !ok || fam != family {
|
||||
continue // a longer family's member, or a name no longer of this family
|
||||
}
|
||||
if !g.Current {
|
||||
if choosing == Allocating {
|
||||
return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf(
|
||||
"%s on %s holds %q, which was given to the mesh rather than made by it, and %s has "+
|
||||
"since generated a new sealing key. The mesh cannot make another; give it again",
|
||||
m.Module, node, g.Name, node)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if needed[m.Module] == nil {
|
||||
needed[m.Module] = map[string]string{}
|
||||
}
|
||||
needed[m.Module][g.Name] = g.Sealed
|
||||
}
|
||||
}
|
||||
for name := range m.OwnSecrets.Plain() {
|
||||
// Minted on the send path and only read on every other. Making one is an insert, and
|
||||
// a question that writes is a question that can block against the machine it is about.
|
||||
var sealed string
|
||||
|
||||
@@ -288,13 +288,24 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
||||
"those walks answer them; a newer merge, or `rebuild <module>`, builds again", p.ID)
|
||||
}
|
||||
}
|
||||
// Settled from the build records before anything is judged (novox/hq issue 457): a build of the tier
|
||||
// that failed after the plan did is as failed as the one that failed it. What toRetry says is the
|
||||
// failed set every step below works from.
|
||||
var failed []string
|
||||
if p.Tier < len(p.Tiers) {
|
||||
recorded, byID, err := recordsOfAsked(ctx, inv, &p, p.Tiers[p.Tier])
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
failed = toRetry(&p, recorded, byID)
|
||||
}
|
||||
if err := retryRefusal(p, plans); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if again := unjudgedAtGate(p); len(failedIn(p)) == 0 && len(again) > 0 {
|
||||
if again := unjudgedAtGate(p); len(failed) == 0 && len(again) > 0 {
|
||||
return retryTierWhole(ctx, open, &p, again)
|
||||
}
|
||||
if len(failedIn(p)) == 0 {
|
||||
if len(failed) == 0 {
|
||||
return retryRollouts(ctx, open, &p)
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
@@ -305,7 +316,6 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
||||
for _, e := range entries {
|
||||
byName[e.Manifest.Module] = e
|
||||
}
|
||||
failed := failedIn(p)
|
||||
var asked []string
|
||||
for _, m := range failed {
|
||||
askModule(ctx, &p, m, byName)
|
||||
@@ -525,3 +535,16 @@ func retryTierWhole(ctx context.Context, open *stores, p *inventory.Plan, again
|
||||
func sendAgain(s *inventory.PlanModule) {
|
||||
s.First, s.FirstAt, s.Gate, s.GatedBy, s.Previous, s.Why = nil, nil, nil, "", "", ""
|
||||
}
|
||||
|
||||
// toRetry is the modules a retry asks again: every one of the tier that failed, the plan's state
|
||||
// settled from the build records first (novox/hq issue 457). A plan fails on the first failure in its
|
||||
// tier, and an outcome arriving after that finds no open plan to answer — it is kept only in the build
|
||||
// records. Read from the plan alone, a retry asked only the build that failed first, and the records
|
||||
// then failed the plan again on the next: each failed build of a tier took a retry of its own. What
|
||||
// still runs is left asked, and its outcome is the plan's once the retry sets it building.
|
||||
func toRetry(p *inventory.Plan, recorded map[string][]inventory.Build, byID map[string]inventory.Build) []string {
|
||||
if p.Tier < len(p.Tiers) {
|
||||
settleFromRecords(p, p.Tiers[p.Tier], recorded, byID)
|
||||
}
|
||||
return failedIn(*p)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A retry asks every failed build of the tier, not one (novox/hq issue 457). Seen 2026-10-11: gitea
|
||||
// and plex both failed in tier 0 while the registry was held still; gitea's failure failed the plan,
|
||||
// and plex's, arriving after, found no open plan and was kept only in the build records. The first
|
||||
// retry asked gitea alone, the records then failed the plan again on plex, and a second retry asked
|
||||
// plex.
|
||||
func TestARetryAsksEveryFailedBuildOfTheTier(t *testing.T) {
|
||||
asked := time.Date(2026, 10, 11, 1, 29, 0, 0, time.UTC)
|
||||
failedAt := asked.Add(2 * time.Minute)
|
||||
p := inventory.Plan{ID: "plan-457", State: inventory.PlanFailed, Tier: 0,
|
||||
Tiers: [][]string{{"gitea", "plex"}}, Note: "gitea failed to build in tier 0",
|
||||
Modules: map[string]*inventory.PlanModule{
|
||||
"gitea": {State: "failed", AskedAt: &asked, Build: "build-gitea", Why: "cannot reach the registry"},
|
||||
"plex": {State: "asked", AskedAt: &asked, Build: "build-plex"},
|
||||
}}
|
||||
byID := map[string]inventory.Build{
|
||||
"build-plex": {ID: "build-plex", Module: "plex", At: failedAt, Failed: "cannot reach the registry"},
|
||||
}
|
||||
|
||||
if got := toRetry(&p, nil, byID); !reflect.DeepEqual(got, []string{"gitea", "plex"}) {
|
||||
t.Fatalf("a retry of a tier where gitea and plex failed asks %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A build of the tier still running when the plan failed is not asked again: its outcome is the plan's
|
||||
// once the retry sets it building, and one that is recorded built is taken as built.
|
||||
func TestARetryLeavesABuildThatRunsOrWorked(t *testing.T) {
|
||||
asked := time.Date(2026, 10, 11, 1, 29, 0, 0, time.UTC)
|
||||
builtAt := asked.Add(3 * time.Minute)
|
||||
p := inventory.Plan{ID: "plan-457", State: inventory.PlanFailed, Tier: 0,
|
||||
Tiers: [][]string{{"a", "b", "c"}},
|
||||
Modules: map[string]*inventory.PlanModule{
|
||||
"a": {State: "failed", AskedAt: &asked, Build: "build-a", Why: "broken"},
|
||||
"b": {State: "asked", AskedAt: &asked, Build: "build-b"},
|
||||
"c": {State: "asked", AskedAt: &asked, Build: "build-c"},
|
||||
}}
|
||||
byID := map[string]inventory.Build{"build-c": {ID: "build-c", Module: "c", At: builtAt, Commit: "c0ffee"}}
|
||||
|
||||
if got := toRetry(&p, nil, byID); !reflect.DeepEqual(got, []string{"a"}) {
|
||||
t.Fatalf("asks %v, want a alone", got)
|
||||
}
|
||||
if s := p.Modules["c"]; s.State != "built" || s.Commit != "c0ffee" {
|
||||
t.Errorf("c, recorded built, is %+v", s)
|
||||
}
|
||||
if s := p.Modules["b"]; s.State != "asked" {
|
||||
t.Errorf("b, still building, is %+v", s)
|
||||
}
|
||||
}
|
||||
@@ -27,13 +27,6 @@ package main
|
||||
// ask says so, and the whole is read with `settings proposals <id>` — whose fingerprint must be the one on the
|
||||
// phone. Fail closed: a proposal nothing can carry to the operator is refused at once, in words, and never left
|
||||
// waiting for an answer that cannot come.
|
||||
//
|
||||
// **On a channel that proves who answers, the values are shown WHOLE** (novox/hq issue 383, the operator's
|
||||
// decision of 2026-10-10): a mount point, a share name or a private address is the thing being approved and must
|
||||
// be readable, so the ask carries them whole beside the explanation (asks.Ask.Whole), the router shows that only
|
||||
// on a kind that verifies its sender, and only a value shaped like a secret is withheld there. The message is the
|
||||
// headline, one line per key, who proposed it and when; the fingerprint and how to read the proposal whole are
|
||||
// the Details answer's (asks.Ask.Details). The masking stays for conditions and for channels that prove nothing.
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -161,39 +154,39 @@ func (p settingsProposal) ask(id string, machines []string) (asks.Ask, map[strin
|
||||
if len([]rune(headline)) > asks.HeadlineLength {
|
||||
headline = verb + " settings?"
|
||||
}
|
||||
// The message (issue 383): the change, one line per key, then who proposed it and when — the headline says
|
||||
// what is set where, and the router adds what every ask says. The fingerprint and the how-to are Details'.
|
||||
compose := func(change string) string {
|
||||
var b strings.Builder
|
||||
if p.Clear && !p.HadLayer {
|
||||
b.WriteString("There is no layer to remove; approving changes nothing.\n")
|
||||
} else {
|
||||
b.WriteString(change + "\n")
|
||||
}
|
||||
fmt.Fprintf(&b, "Proposed by %s at %s.", sayable(p.From, machines), p.At.Local().Format("15:04 on 2 Jan"))
|
||||
return b.String()
|
||||
}
|
||||
shown, shownWhole := p.change(machines, false)
|
||||
whole, _ := p.change(machines, true)
|
||||
var d strings.Builder
|
||||
fmt.Fprintf(&d, "Fingerprint %s.\n", fingerprint(p.Digest))
|
||||
if !shownWhole {
|
||||
d.WriteString("On a channel that does not prove who answers, these values are shown as ‹address›, ‹path› or ‹withheld›.\n")
|
||||
}
|
||||
fmt.Fprintf(&d, "Read it whole, with this fingerprint: settings proposals %s at the controller's terminal, or "+
|
||||
"mesh-controller.settings with proposal %s through the mesh MCP server.\n", id, id)
|
||||
shown, whole := p.change(machines)
|
||||
var b strings.Builder
|
||||
if p.Clear {
|
||||
d.WriteString("Approved, the layer is removed at once and the machine takes it at its next push.")
|
||||
fmt.Fprintf(&b, "Clear the settings of %s on %s, back to what the module says?\n\n", p.Module, p.where())
|
||||
} else {
|
||||
d.WriteString("Approved, the layer is set at once and the machine takes it at its next push.")
|
||||
fmt.Fprintf(&b, "Set the settings of %s on %s to these values?\n\n", p.Module, p.where())
|
||||
}
|
||||
q := asks.Ask{ID: id, Headline: headline, Explanation: compose(shown), Who: asks.Operator,
|
||||
fmt.Fprintf(&b, "Proposed by %s, at %s. ", sayable(p.From, machines), p.At.Local().Format("15:04 on 2 Jan"))
|
||||
switch {
|
||||
case p.Clear && p.HadLayer:
|
||||
b.WriteString("The layer it removes:\n\n")
|
||||
case p.Clear:
|
||||
b.WriteString("There is no layer to remove; approving changes nothing.")
|
||||
case !p.HadLayer:
|
||||
b.WriteString("There is no layer yet; this is the whole of it:\n\n")
|
||||
default:
|
||||
b.WriteString("The layer is replaced whole; what changes against it:\n\n")
|
||||
}
|
||||
b.WriteString(shown)
|
||||
fmt.Fprintf(&b, "\n\nFingerprint %s.", fingerprint(p.Digest))
|
||||
if !whole {
|
||||
b.WriteString(" Parts shown as ‹address›, ‹path› or ‹withheld› may not leave the mesh: read it whole, with " +
|
||||
"this fingerprint, through the mesh MCP server (mesh-controller.settings, proposal " + id + ") or with " +
|
||||
"mesh-cli settings proposals " + id + ".")
|
||||
}
|
||||
if p.Clear {
|
||||
b.WriteString(" Approved, the layer is removed at once and the machine takes it at its next push.")
|
||||
} else {
|
||||
b.WriteString(" Approved, the layer is set at once and the machine takes it at its next push.")
|
||||
}
|
||||
q := asks.Ask{ID: id, Headline: headline, Explanation: b.String(), Who: asks.Operator,
|
||||
Expires: p.At.Add(askApproveFor), OnExpiry: "the proposal is discarded; nothing changes",
|
||||
About: p.about(), Details: d.String()}
|
||||
if whole != shown {
|
||||
// Only where a value was masked: an ask whose values all pass the content rule shows the same everywhere.
|
||||
q.Whole = compose(whole)
|
||||
}
|
||||
About: p.about()}
|
||||
options := map[string]int{}
|
||||
for i, act := range p.actions(id) {
|
||||
binds, _ := asks.ActDigest(boundAct(act))
|
||||
@@ -215,17 +208,13 @@ func (p settingsProposal) ask(id string, machines []string) (asks.Ask, map[strin
|
||||
// shownMost is the most of a change the phone is shown, in bytes; the rest is read whole with `settings proposals`.
|
||||
const shownMost = 1400
|
||||
|
||||
// change is what changes, line by line, and whether every value was shown whole: "+ key: value" added,
|
||||
// "~ key: value (was: old)" changed, "- key (was: old)" removed, and the count of keys unchanged. Each value is
|
||||
// shown under the content rule (sayableValue), or — exact, for a channel that proves who answers — as it is,
|
||||
// withheld only when shaped like a secret (wholeValue).
|
||||
func (p settingsProposal) change(machines []string, exact bool) (string, bool) {
|
||||
// change is what changes, line by line, each value shown under the content rule, and whether every value was
|
||||
// shown whole: "+ key: value" added, "~ key: value (was: old)" changed, "- key (was: old)" removed, and the
|
||||
// count of keys unchanged.
|
||||
func (p settingsProposal) change(machines []string) (string, bool) {
|
||||
whole := true
|
||||
say := func(v any) string {
|
||||
s, w := sayableValue(v, machines)
|
||||
if exact {
|
||||
s, w = wholeValue(v, machines)
|
||||
}
|
||||
whole = whole && w
|
||||
return s
|
||||
}
|
||||
@@ -253,8 +242,6 @@ func (p settingsProposal) change(machines []string, exact bool) (string, bool) {
|
||||
lines = append(lines, fmt.Sprintf("= nothing changes: the %d key(s) are as they stand", unchanged))
|
||||
case unchanged > 0:
|
||||
lines = append(lines, fmt.Sprintf("= %d key(s) unchanged", unchanged))
|
||||
case len(lines) == 0:
|
||||
lines = append(lines, "= the layer has no keys")
|
||||
}
|
||||
}
|
||||
out := strings.Join(lines, "\n")
|
||||
@@ -313,27 +300,6 @@ func sayableValue(v any, machines []string) (string, bool) {
|
||||
return out, out == text
|
||||
}
|
||||
|
||||
// wholeValue is a value as a channel that proves who answers is shown it (asks.Ask.Whole), and whether it was
|
||||
// shown whole: as it is, unless it is shaped like a secret (outward.Secret), which is withheld whole — never in
|
||||
// part, so no half of a key reaches the phone.
|
||||
func wholeValue(v any, machines []string) (string, bool) {
|
||||
text, ok := v.(string)
|
||||
if !ok {
|
||||
raw, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return markWithheld, false
|
||||
}
|
||||
text = string(raw)
|
||||
}
|
||||
if text == "" {
|
||||
return `""`, true
|
||||
}
|
||||
if _, ok := outward.Secret(text, machines...); !ok {
|
||||
return markWithheld, false
|
||||
}
|
||||
return text, true
|
||||
}
|
||||
|
||||
// sayable is a text with what may not leave the mesh replaced in place by a marker; what the markers cannot make
|
||||
// pass is withheld whole.
|
||||
func sayable(text string, machines []string) string {
|
||||
@@ -466,12 +432,6 @@ func (pr proposer) propose(ctx context.Context, in proposeInput) (string, error)
|
||||
return refuse("the ask's words would carry %s, which may not leave the mesh, and the change could not be "+
|
||||
"shown without it; %s", refusal, atTheTerminalInstead(in))
|
||||
}
|
||||
// The whole words are shown only where the sender is proven, and never a secret's shape: wholeValue withholds
|
||||
// one, and the router would refuse the ask if one were left, so it is refused here first, in words.
|
||||
if refusal, ok := outward.Secret(q.Whole, machines...); !ok {
|
||||
return refuse("the change shown whole would carry %s, which may not leave the mesh on any channel; %s",
|
||||
refusal, atTheTerminalInstead(in))
|
||||
}
|
||||
// Fail closed, before anything is kept: no router, no grant, no channel means no ask.
|
||||
if pr.routerHere != nil {
|
||||
here, err := pr.routerHere(ctx)
|
||||
|
||||
@@ -116,36 +116,24 @@ func TestAProposalAsksAtTheLevelApproveWithTheExactChange(t *testing.T) {
|
||||
if q.Options[0].Binds == q.Options[1].Binds {
|
||||
t.Error("Approve and Decline bind the same act")
|
||||
}
|
||||
// The message's shape (issue 383): one line per key, then who proposed it and when — and nothing else: the
|
||||
// fingerprint and the how-to are the Details answer's.
|
||||
proposedBy := "Proposed by g14/claude-code, through the mesh-controller seat at " + r.now.Local().Format("15:04 on 2 Jan") + "."
|
||||
if q.Explanation != "+ sources: recalbox=‹address›@‹path›:ro\n~ shares: library=‹path› (was: none)\n- old (was: x)\n"+proposedBy {
|
||||
t.Errorf("the explanation:\n%s", q.Explanation)
|
||||
for _, line := range []string{
|
||||
"Set the settings of mounts on shanks to these values?",
|
||||
"Proposed by g14/claude-code, through the mesh-controller seat, at " + r.now.Local().Format("15:04 on 2 Jan") + ".",
|
||||
"+ sources: recalbox=‹address›@‹path›:ro",
|
||||
"~ shares: library=‹path› (was: none)",
|
||||
"- old (was: x)",
|
||||
"Fingerprint " + fingerprint(layerDigest(mountsSources)) + ".",
|
||||
"read it whole, with this fingerprint",
|
||||
} {
|
||||
if !strings.Contains(q.Explanation, line) {
|
||||
t.Errorf("the explanation lacks %q:\n%s", line, q.Explanation)
|
||||
}
|
||||
}
|
||||
for _, leak := range []string{"nas.lan", "/mnt/recalbox", "/mnt/library", "smb://"} {
|
||||
if strings.Contains(q.Explanation, leak) {
|
||||
t.Errorf("the explanation carries %q, which may not leave the mesh", leak)
|
||||
}
|
||||
}
|
||||
// Where the sender is proven, the values whole: the mount point and the share are what is approved.
|
||||
if q.Whole != "+ sources: recalbox=smb://nas.lan/recalbox@/mnt/recalbox:ro\n~ shares: library=/mnt/library (was: none)\n- old (was: x)\n"+proposedBy {
|
||||
t.Errorf("the whole words:\n%s", q.Whole)
|
||||
}
|
||||
for _, line := range []string{
|
||||
"Fingerprint " + fingerprint(layerDigest(mountsSources)) + ".",
|
||||
"On a channel that does not prove who answers, these values are shown as ‹address›, ‹path› or ‹withheld›.",
|
||||
"Read it whole, with this fingerprint: settings proposals " + kept(r).ID + " at the controller's terminal",
|
||||
"Approved, the layer is set at once and the machine takes it at its next push.",
|
||||
} {
|
||||
if !strings.Contains(q.Details, line) {
|
||||
t.Errorf("Details lack %q:\n%s", line, q.Details)
|
||||
}
|
||||
}
|
||||
for _, howTo := range []string{"ingerprint", "settings proposals", "mesh-controller.settings", "does not prove", "Approved,"} {
|
||||
if strings.Contains(q.Explanation, howTo) || strings.Contains(q.Whole, howTo) {
|
||||
t.Errorf("the message carries the how-to %q", howTo)
|
||||
}
|
||||
}
|
||||
all := []string{q.Headline, q.Explanation, q.OnExpiry}
|
||||
for _, o := range q.Options {
|
||||
all = append(all, o.Label, o.Does)
|
||||
@@ -153,9 +141,6 @@ func TestAProposalAsksAtTheLevelApproveWithTheExactChange(t *testing.T) {
|
||||
if refusal, ok := outward.Check(strings.Join(all, "\n"), "anchor", "laptop", "shanks"); !ok {
|
||||
t.Errorf("the router would refuse the ask: %s", refusal)
|
||||
}
|
||||
if refusal, ok := outward.Secret(q.Whole, "anchor", "laptop", "shanks"); !ok {
|
||||
t.Errorf("the router would refuse the whole words: %s", refusal)
|
||||
}
|
||||
// Kept as the controller's own ask, about no condition, with the proposal whole and its digests.
|
||||
kept := r.theProposal(t)
|
||||
p := kept.Proposal
|
||||
@@ -188,84 +173,11 @@ func TestAMeshWideLayerIsProposed(t *testing.T) {
|
||||
}
|
||||
q := r.askSent(t)
|
||||
if q.Headline != "Set notes on the whole mesh?" || q.About != "settings.notes.mesh" ||
|
||||
!strings.HasPrefix(q.Explanation, "+ x: 1\nProposed by ") || q.Whole != "" {
|
||||
!strings.Contains(q.Explanation, "Set the settings of notes on the whole mesh to these values?") {
|
||||
t.Errorf("%+v", q)
|
||||
}
|
||||
}
|
||||
|
||||
// kept is the one proposal the rig keeps.
|
||||
func kept(r *proposerRig) asked {
|
||||
for _, a := range r.store {
|
||||
if a.Proposal != nil {
|
||||
return a
|
||||
}
|
||||
}
|
||||
return asked{}
|
||||
}
|
||||
|
||||
// The switch between the masked and the whole change (issue 383): the same change, under the content rule and
|
||||
// exact, differs in the masked values alone; a value shaped like a secret is withheld in both, whole, with its
|
||||
// key still named; and a change no value of which is masked carries no whole words of its own.
|
||||
func TestAProposalShowsItsValuesWholeOnlyWhereTheSenderIsProvenAndNeverASecret(t *testing.T) {
|
||||
r := newProposerRig(t)
|
||||
values := map[string]any{"shares": "media=/storage/media", "sources": "recalbox=smb://nas.lan/recalbox@/mnt/recalbox",
|
||||
"github": "ghp_abcdefghijklmnopqrstuvwxyz0123456789", "cert": "-----BEGIN CERTIFICATE-----", "font": "Inter 13"}
|
||||
if _, err := r.pr.propose(context.Background(), proposeInput{module: "mounts", node: "shanks", values: values}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q := r.askSent(t)
|
||||
masked, _ := kept(r).Proposal.change([]string{"anchor", "laptop", "shanks"}, false)
|
||||
whole, _ := kept(r).Proposal.change([]string{"anchor", "laptop", "shanks"}, true)
|
||||
if !strings.Contains(q.Explanation, masked) || !strings.Contains(q.Whole, whole) {
|
||||
t.Fatalf("the ask does not carry the change masked and whole:\n%s\n--\n%s", q.Explanation, q.Whole)
|
||||
}
|
||||
for _, line := range []string{"+ cert: ‹withheld›", "+ font: Inter 13", "+ github: ‹withheld›", "+ shares: media=‹path›", "+ sources: recalbox=‹address›@‹path›"} {
|
||||
if !strings.Contains(masked, line) {
|
||||
t.Errorf("masked, lacks %q:\n%s", line, masked)
|
||||
}
|
||||
}
|
||||
for _, line := range []string{"+ cert: ‹withheld›", "+ font: Inter 13", "+ github: ‹withheld›", "+ shares: media=/storage/media",
|
||||
"+ sources: recalbox=smb://nas.lan/recalbox@/mnt/recalbox"} {
|
||||
if !strings.Contains(whole, line) {
|
||||
t.Errorf("whole, lacks %q:\n%s", line, whole)
|
||||
}
|
||||
}
|
||||
for _, secret := range []string{"ghp_", "BEGIN CERTIFICATE"} {
|
||||
if strings.Contains(q.Explanation, secret) || strings.Contains(q.Whole, secret) || strings.Contains(q.Details, secret) {
|
||||
t.Errorf("the secret %q reaches the phone", secret)
|
||||
}
|
||||
}
|
||||
if _, ok := outward.Secret(q.Whole, "shanks"); !ok {
|
||||
t.Error("the router would refuse the whole words")
|
||||
}
|
||||
// Mutation: the masked and the whole change differ in exactly the lines whose value was masked.
|
||||
m, w := strings.Split(masked, "\n"), strings.Split(whole, "\n")
|
||||
if len(m) != len(w) {
|
||||
t.Fatalf("masked %d lines, whole %d", len(m), len(w))
|
||||
}
|
||||
differ := 0
|
||||
for i := range m {
|
||||
if m[i] != w[i] {
|
||||
differ++
|
||||
if !strings.Contains(m[i], "‹") || strings.Contains(w[i], "‹") {
|
||||
t.Errorf("the lines differ otherwise than by the mask:\n%s\n%s", m[i], w[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
if differ != 2 {
|
||||
t.Errorf("%d lines differ, and the mask covered 2", differ)
|
||||
}
|
||||
// No value masked: the message is the same everywhere, and the ask says no whole words.
|
||||
r2 := newProposerRig(t)
|
||||
if _, err := r2.pr.propose(context.Background(), proposeInput{module: "dunst", node: "laptop", values: map[string]any{"font-size": 13, "width": 500}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if q2 := r2.askSent(t); q2.Whole != "" || !strings.HasPrefix(q2.Explanation, "+ font-size: 13\n+ width: 500\nProposed by ") ||
|
||||
strings.Contains(q2.Details, "does not prove who answers") {
|
||||
t.Errorf("%+v", q2)
|
||||
}
|
||||
}
|
||||
|
||||
// A proposal expired unanswered is kept so by the reconciling, and a warrant for it afterwards sets nothing.
|
||||
func TestAnExpiredProposalIsKeptExpired(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
@@ -293,8 +205,7 @@ func TestAClearIsProposedAndShowsWhatItRemoves(t *testing.T) {
|
||||
}
|
||||
q := r.askSent(t)
|
||||
if q.Headline != "Clear notes on laptop?" || !strings.Contains(q.Explanation, "- places.data.owner: 1001:1001") ||
|
||||
!strings.Contains(q.Explanation, "- places.data.path: ‹path›") || !strings.Contains(q.Whole, "- places.data.path: /srv/notes") ||
|
||||
!strings.Contains(q.Details, "Approved, the layer is removed at once") {
|
||||
!strings.Contains(q.Explanation, "- places.data.path: ‹path›") {
|
||||
t.Errorf("%+v", q)
|
||||
}
|
||||
if p := r.theProposal(t).Proposal; !p.Clear || p.Digest != layerDigest(r.before) || len(r.judged) != 0 {
|
||||
|
||||
@@ -37,6 +37,32 @@ type holding struct {
|
||||
shelf map[string]catalogue.Manifest
|
||||
// providers memoises providerFor by machine, consumer and provision.
|
||||
providers map[string]providerLookup
|
||||
// waits is what the waits of a statement are checked against, read as they are asked for (novox/hq issue 450).
|
||||
waits operatorWaitFacts
|
||||
}
|
||||
|
||||
// checked is a machine's newest statement as the controller judges it: each wait checked (ADR 0283 decision 3), so
|
||||
// a wait that does not check out reads as unhealthy, as it did when the statement was judged (novox/hq issue 450).
|
||||
// What is stored is what the machine said, the waits unchecked; read as stored, a provider whose wait failed its
|
||||
// check seems to wait for the operator while its unhealthy condition is open.
|
||||
func (h *holding) checked(machine string) []inventory.ResourceHealth {
|
||||
rs := h.healths[machine].Resources
|
||||
mods := waitingModules(rs)
|
||||
if len(mods) == 0 {
|
||||
return rs
|
||||
}
|
||||
if h.waits.manifests == nil {
|
||||
h.waits.manifests = map[string]catalogue.Manifest{}
|
||||
}
|
||||
for _, module := range mods {
|
||||
if _, has := h.waits.manifests[module]; !has {
|
||||
if m, ok := h.manifestOf(module); ok {
|
||||
h.waits.manifests[module] = m
|
||||
}
|
||||
}
|
||||
}
|
||||
readWaitFacts(h.ctx, h.inv, machine, mods, nil, &h.waits)
|
||||
return checkWaiting(machine, rs, h.waits)
|
||||
}
|
||||
|
||||
type providerLookup struct {
|
||||
@@ -53,6 +79,15 @@ func readHolding(ctx context.Context, inv *inventory.Inventory, open []condition
|
||||
return &holding{ctx: ctx, inv: inv, healths: healths, open: open, providers: map[string]providerLookup{}}, nil
|
||||
}
|
||||
|
||||
// readHoldingFor is how a judging reads its holding: nothing without a store. A variable so a test can hand a
|
||||
// judging the record it holds under (novox/hq issue 405).
|
||||
var readHoldingFor = func(ctx context.Context, inv *inventory.Inventory, open []conditions.Condition) (*holding, error) {
|
||||
if inv == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return readHolding(ctx, inv, open)
|
||||
}
|
||||
|
||||
// heldFinding says a resource's state is a finding of its declared check that names a provision: what
|
||||
// may be held. Down and restarting are liveness, the resource's own.
|
||||
func heldFinding(r inventory.ResourceHealth) bool {
|
||||
@@ -106,42 +141,137 @@ func (h *holding) lookUpProvider(machine, consumer, provision string) (catalogue
|
||||
return catalogue.Chosen{}, false
|
||||
}
|
||||
|
||||
// unhealthy says a provider is unhealthy on the record: its condition is open, or its machine's newest
|
||||
// statement says a resource of it is unhealthy.
|
||||
func (h *holding) unhealthy(p catalogue.Chosen) bool {
|
||||
key := moduleUnhealthyKey(p.Module, p.Node)
|
||||
// providerState is how a provider stands on the record: unhealthy when its unhealthy condition is open or its
|
||||
// machine's newest statement says a resource of it is unhealthy; else waiting for the operator when that statement
|
||||
// says a resource of it waits, with the waits it names, or its needs-operator condition is open (waits then
|
||||
// unknown); else healthy.
|
||||
func (h *holding) providerState(p catalogue.Chosen) (unhealthy, waiting bool, waits []inventory.Wait) {
|
||||
for _, c := range h.open {
|
||||
if c.Key == key {
|
||||
return true
|
||||
if c.Key == moduleUnhealthyKey(p.Module, p.Node) {
|
||||
return true, false, nil
|
||||
}
|
||||
}
|
||||
for _, r := range h.healths[p.Node].Resources {
|
||||
if r.Module == p.Module && r.State == link.StateUnhealthy {
|
||||
return true
|
||||
for _, r := range h.checked(p.Node) {
|
||||
if r.Module != p.Module {
|
||||
continue
|
||||
}
|
||||
switch r.State {
|
||||
case link.StateUnhealthy:
|
||||
return true, false, nil
|
||||
case link.StateWaiting:
|
||||
waiting = true
|
||||
waits = append(waits, r.Waits...)
|
||||
}
|
||||
}
|
||||
return false
|
||||
if !waiting {
|
||||
for _, c := range h.open {
|
||||
waiting = waiting || c.Key == needsOperatorKey(p.Module, p.Node)
|
||||
}
|
||||
}
|
||||
return false, waiting, waits
|
||||
}
|
||||
|
||||
// manifestOf is a module's manifest from the catalogue, read once; false when it cannot be read.
|
||||
func (h *holding) manifestOf(module string) (catalogue.Manifest, bool) {
|
||||
if h.shelf == nil && h.inv != nil {
|
||||
shelf, err := h.inv.Catalogue(h.ctx)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, false
|
||||
}
|
||||
h.shelf = shelf
|
||||
}
|
||||
m, ok := h.shelf[module]
|
||||
return m, ok
|
||||
}
|
||||
|
||||
// covering is the waits of a provider that cover a provision it gives (novox/hq issue 405): a module that waits
|
||||
// says nothing else of it is wrong and names each part that waits (ADR 0283 decision 1), so only a consumer of
|
||||
// the waiting part waits on it. A wait covers a provision when its part is that provision, or the secret it waits
|
||||
// for is the provision's shared credential (ADR 0158). Nothing when no wait can be matched: a consumer failing
|
||||
// then is not held, since holding it would hide a fault that may be its own.
|
||||
func (h *holding) covering(p catalogue.Chosen, provision string, waits []inventory.Wait) []inventory.Wait {
|
||||
credential := ""
|
||||
if m, ok := h.manifestOf(p.Module); ok {
|
||||
credential, _ = m.SharedCredentialOf(provision)
|
||||
}
|
||||
var out []inventory.Wait
|
||||
for _, w := range waits {
|
||||
if w.Part == provision || (w.Secret != "" && w.Secret == credential) {
|
||||
out = append(out, w)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// heldUnderProvider is the provider a consumer's findings are held under, and — when that provider only waits for the
|
||||
// operator, for the part the consumer needs — the waits that hold it.
|
||||
type heldUnderProvider struct {
|
||||
provider catalogue.Chosen
|
||||
// waits is set when every finding held waits on a provider that only waits for the operator: the consumer's
|
||||
// gate then reads as ADR 0254's waits for a person, a pass with the wait carried (ADR 0283 decision 4).
|
||||
waits []inventory.Wait
|
||||
}
|
||||
|
||||
// heldUnder is the provider a consumer's unhealthy resources wait on: when every one of them is a finding
|
||||
// of a check naming a provision whose provider for this consumer is unhealthy on the record. False when any
|
||||
// is the consumer's own.
|
||||
// of a check naming a provision whose provider for this consumer is unhealthy on the record, or waits for the
|
||||
// operator for the part that gives it (novox/hq issue 405). False when any is the consumer's own.
|
||||
func (h *holding) heldUnder(machine, module string, rs []inventory.ResourceHealth) (catalogue.Chosen, bool) {
|
||||
var on catalogue.Chosen
|
||||
by, held := h.heldWith(machine, module, rs)
|
||||
return by.provider, held
|
||||
}
|
||||
|
||||
func (h *holding) heldWith(machine, module string, rs []inventory.ResourceHealth) (heldUnderProvider, bool) {
|
||||
var on heldUnderProvider
|
||||
onlyWaits := true
|
||||
for _, r := range rs {
|
||||
if r.State != link.StateUnhealthy {
|
||||
continue
|
||||
}
|
||||
if !heldFinding(r) {
|
||||
return catalogue.Chosen{}, false
|
||||
return heldUnderProvider{}, false
|
||||
}
|
||||
p, ok := h.providerFor(machine, module, r.Needs)
|
||||
if !ok || (p.Node == machine && p.Module == module) || !h.unhealthy(p) {
|
||||
return catalogue.Chosen{}, false
|
||||
if !ok || (p.Node == machine && p.Module == module) {
|
||||
return heldUnderProvider{}, false
|
||||
}
|
||||
on = p
|
||||
unhealthy, waiting, waits := h.providerState(p)
|
||||
switch {
|
||||
case unhealthy:
|
||||
onlyWaits = false
|
||||
case waiting:
|
||||
covered := h.covering(p, r.Needs, waits)
|
||||
if len(covered) == 0 {
|
||||
return heldUnderProvider{}, false
|
||||
}
|
||||
on.waits = append(on.waits, covered...)
|
||||
default:
|
||||
return heldUnderProvider{}, false
|
||||
}
|
||||
on.provider = p
|
||||
}
|
||||
return on, on.Module != ""
|
||||
if !onlyWaits {
|
||||
on.waits = nil
|
||||
}
|
||||
return on, on.provider.Module != ""
|
||||
}
|
||||
|
||||
// waitingUncovered is a provider of a consumer's failing finding that waits for the operator for a part the
|
||||
// finding cannot be matched to (novox/hq issue 405): the consumer is raised on its own, and its condition says
|
||||
// the provider waits, so neither is hidden.
|
||||
func (h *holding) waitingUncovered(machine, module string, rs []inventory.ResourceHealth) (catalogue.Chosen, bool) {
|
||||
for _, r := range rs {
|
||||
if r.State != link.StateUnhealthy || !heldFinding(r) {
|
||||
continue
|
||||
}
|
||||
p, ok := h.providerFor(machine, module, r.Needs)
|
||||
if !ok || (p.Node == machine && p.Module == module) {
|
||||
continue
|
||||
}
|
||||
if unhealthy, waiting, waits := h.providerState(p); !unhealthy && waiting && len(h.covering(p, r.Needs, waits)) == 0 {
|
||||
return p, true
|
||||
}
|
||||
}
|
||||
return catalogue.Chosen{}, false
|
||||
}
|
||||
|
||||
// waitersOn is every consumer held under a provider, as "<module> on <machine>", sorted.
|
||||
@@ -165,8 +295,8 @@ func (h *holding) waitersOn(p catalogue.Chosen) []string {
|
||||
}
|
||||
|
||||
// heldModules is, for one machine's statement, each module whose finding is held, with the provider.
|
||||
func (h *holding) heldModules(machine string) map[string]catalogue.Chosen {
|
||||
out := map[string]catalogue.Chosen{}
|
||||
func (h *holding) heldModules(machine string) map[string]heldUnderProvider {
|
||||
out := map[string]heldUnderProvider{}
|
||||
byModule := map[string][]inventory.ResourceHealth{}
|
||||
for _, r := range h.healths[machine].Resources {
|
||||
if r.Module != "" && r.State == link.StateUnhealthy {
|
||||
@@ -174,7 +304,7 @@ func (h *holding) heldModules(machine string) map[string]catalogue.Chosen {
|
||||
}
|
||||
}
|
||||
for module, rs := range byModule {
|
||||
if on, held := h.heldUnder(machine, module, rs); held {
|
||||
if on, held := h.heldWith(machine, module, rs); held {
|
||||
out[module] = on
|
||||
}
|
||||
}
|
||||
|
||||
@@ -446,6 +446,47 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string,
|
||||
advanceHeld(ctx, open)
|
||||
}
|
||||
|
||||
// startedBeside is the id of a started walk open beside this one — a merge's walk past its wait, not the
|
||||
// backlog's — or empty: one walk at a time (novox/hq ADR 0276).
|
||||
func startedBeside(ctx context.Context, inv *inventory.Inventory, id string, created time.Time) (string, error) {
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, q := range plans {
|
||||
if q.ID == id || q.Release != nil || q.Waiting() {
|
||||
continue
|
||||
}
|
||||
// Started: a tier asked, or on its way (let go, or waiting for nobody) before this one.
|
||||
if q.Tier > 0 || askedAny(q) || q.Created.Before(created) {
|
||||
return q.ID, nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// deferredNote begins the note of a walk deferred behind another.
|
||||
const deferredNote = "let go; starts once "
|
||||
|
||||
// deferred says a walk has its word and has not started: let go while another walk was started, it waits for
|
||||
// that one to end (startedBeside), and its note says so. Read as a wait, not as a tier running late: `plans`
|
||||
// and `status` say its note, and S3 leaves it out. A let-go walk whose first ask failed carries that error as
|
||||
// its note instead, and is watched as before.
|
||||
func deferred(p inventory.Plan) bool {
|
||||
return p.Open() && p.Release == nil && p.Tier == 0 && !askedAny(p) && p.Delivery != nil &&
|
||||
p.Delivery.Awaits != "" && p.Delivery.Go != nil && strings.HasPrefix(p.Note, deferredNote)
|
||||
}
|
||||
|
||||
// askedAny says a plan asked any module.
|
||||
func askedAny(p inventory.Plan) bool {
|
||||
for _, s := range p.Modules {
|
||||
if s != nil && s.State != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// advancePlans moves every open plan as far as the facts allow: a tier whose modules are all built
|
||||
// and whose gates are applied gives way to the next; the last tier done is the plan done. Called
|
||||
// after every outcome and on a timer, so a plan waiting on a machine's report moves when it comes.
|
||||
@@ -469,6 +510,14 @@ func advancePlans(ctx context.Context, open *stores) {
|
||||
advanceHeld(ctx, open)
|
||||
}
|
||||
|
||||
// keepWalkPhases keeps where the walks that ended lately spent their time (novox/hq ADR 0282), outside the hold
|
||||
// on the plans so it never lengthens it: measured, never acted on, and an error only said.
|
||||
func keepWalkPhases(ctx context.Context, open *stores) {
|
||||
if err := recordWalkPhases(ctx, open.inventory, time.Now()); err != nil {
|
||||
fmt.Printf("plans: the phases of the walks ended lately could not be kept: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
// advanceHeld is advancePlans for a caller already holding the plans.
|
||||
func advanceHeld(ctx context.Context, open *stores) {
|
||||
inv := open.inventory
|
||||
@@ -578,6 +627,18 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
}
|
||||
}
|
||||
if unasked == len(tier) {
|
||||
// **One walk at a time** (novox/hq ADR 0276): a walk about to ask its first tier while another started
|
||||
// walk is open waits for that one to end, let go or not, and says so.
|
||||
if p.Tier == 0 {
|
||||
if behind, err := startedBeside(ctx, inv, p.ID, p.Created); err != nil {
|
||||
return false, err
|
||||
} else if behind != "" {
|
||||
note := fmt.Sprintf("%s%s ended — one walk at a time", deferredNote, behind)
|
||||
changed := p.Note != note
|
||||
p.Note = note
|
||||
return changed, nil
|
||||
}
|
||||
}
|
||||
if err := askTier(ctx, inv, p); err != nil {
|
||||
return false, err
|
||||
}
|
||||
@@ -588,26 +649,9 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
// the controller in its first tier: the build that produced the new one is recorded, and the
|
||||
// plan never hears it. The record is the fact; a build recorded after the ask is that tier's
|
||||
// outcome, whoever was listening.
|
||||
recorded := map[string][]inventory.Build{}
|
||||
byID := map[string]inventory.Build{}
|
||||
for _, m := range tier {
|
||||
if s := p.Modules[m]; s != nil && s.State == "asked" {
|
||||
builds, err := inv.Builds(ctx, m, 5)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
recorded[m] = builds
|
||||
// Its own ask's record, by id — found even when the outcome named no module (ADR 0219).
|
||||
if s.Build != "" {
|
||||
b, found, err := inv.BuildByID(ctx, s.Build)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if found {
|
||||
byID[s.Build] = b
|
||||
}
|
||||
}
|
||||
}
|
||||
recorded, byID, err := recordsOfAsked(ctx, inv, p, tier)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if settleFromRecords(p, tier, recorded, byID) {
|
||||
return true, nil
|
||||
@@ -811,8 +855,12 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
strings.Join(machines, ", "), p.Tier, err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
// What each machine of the rest was sent, kept for when it reports it applied (novox/hq ADR 0282 decision
|
||||
// 6): measured, never acted on.
|
||||
sentWhat := sentNow(ctx, open.inventory, sent)
|
||||
for _, m := range rest {
|
||||
p.Modules[m].SentAt = &now
|
||||
p.Modules[m].Rest = restOf(restTo[m], sent, sentWhat)
|
||||
}
|
||||
fmt.Printf("%s: tier %d built; sent %s to %s, one send each\n", p.ID, p.Tier, strings.Join(rest, ", "),
|
||||
strings.Join(sent, ", "))
|
||||
@@ -893,6 +941,20 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// restOf is, for one module, every machine of its rest that the send reached and what it carried there.
|
||||
func restOf(to, sent []string, what map[string]inventory.SentDeclaration) map[string]inventory.SentDeclaration {
|
||||
out := map[string]inventory.SentDeclaration{}
|
||||
for _, n := range to {
|
||||
if slices.Contains(sent, n) {
|
||||
out[n] = what[n]
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// firstSend sends one machine, in one send, every module of the plan's tier whose first machine it is
|
||||
// (novox/hq issue 281), and records the send on each: what that machine ran of it before — read once,
|
||||
// before the send, so a module the same send carries is never read as already moved — the machines it
|
||||
@@ -1140,6 +1202,7 @@ func sayUnsent(p *inventory.Plan, rollsOut func(string) bool) {
|
||||
// planTicker advances open plans on a timer, for the steps outcomes alone cannot take.
|
||||
func planTicker(ctx context.Context, open *stores) {
|
||||
advancePlans(ctx, open)
|
||||
keepWalkPhases(ctx, open)
|
||||
tick := time.NewTicker(30 * time.Second)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
@@ -1148,6 +1211,7 @@ func planTicker(ctx context.Context, open *stores) {
|
||||
return
|
||||
case <-tick.C:
|
||||
advancePlans(ctx, open)
|
||||
keepWalkPhases(ctx, open)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1181,27 +1245,38 @@ func inTierSince(p inventory.Plan) time.Time {
|
||||
// planLineWith is planLine knowing whether the build seat is paused (novox/hq ADR 0219): a plan
|
||||
// waiting on builds nobody will take until a person resumes the seat says so, and is not late. bound is
|
||||
// the plan's tier bound, the one its stalled condition is raised at (tierBounds): LATE is that condition
|
||||
// said on the line (novox/hq issue 296).
|
||||
// said on the line (novox/hq issue 296). The machines running what it moves are not named: planLineOn.
|
||||
func planLineWith(p inventory.Plan, now time.Time, pause pauseView, bound time.Duration) string {
|
||||
return planLineOn(p, now, pause, bound, nil)
|
||||
}
|
||||
|
||||
// planLineOn is planLineWith naming the plan by what it moves and where (planHeadline), given what runs each
|
||||
// module; nil names no machine.
|
||||
func planLineOn(p inventory.Plan, now time.Time, pause pauseView, bound time.Duration, running func(string) []string) string {
|
||||
name := planHeadline(p, running)
|
||||
where := fmt.Sprintf("tier %d of %d", min(p.Tier+1, len(p.Tiers)), len(p.Tiers))
|
||||
switch p.State {
|
||||
case inventory.PlanAssembling, inventory.PlanQueued:
|
||||
// A batch not yet a walk (novox/hq ADR 0276): what it holds and how long is left.
|
||||
return batchWords(p, now)
|
||||
case inventory.PlanDone:
|
||||
return fmt.Sprintf("%s done, %d tier(s)", p.Named(), len(p.Tiers))
|
||||
return fmt.Sprintf("%s · done, %d tier(s)", name, len(p.Tiers))
|
||||
case inventory.PlanFailed:
|
||||
return fmt.Sprintf("%s FAILED at %s: %s", p.Named(), where, p.Note)
|
||||
return fmt.Sprintf("%s · FAILED at %s: %s", name, where, p.Note)
|
||||
case inventory.PlanSuperseded:
|
||||
return fmt.Sprintf("%s %s", p.Named(), p.Note)
|
||||
return fmt.Sprintf("%s · %s", name, p.Note)
|
||||
}
|
||||
since := now.Sub(inTierSince(p)).Round(time.Second)
|
||||
if p.Waiting() {
|
||||
// Waiting for its delivery's word is no lateness of the walk's (novox/hq ADR 0239).
|
||||
return fmt.Sprintf("%s %s, %s, for %s", p.Named(), where, waitingNote(p), since)
|
||||
return fmt.Sprintf("%s · %s, %s, for %s", name, where, waitingNote(p), since)
|
||||
}
|
||||
if deferred(p) {
|
||||
// Nor is waiting for the walk before it to end (one walk at a time, novox/hq ADR 0276).
|
||||
return fmt.Sprintf("%s · %s, %s, for %s", name, where, p.Note, since)
|
||||
}
|
||||
if waiting, paused := pausedWaiting(p, pause, now); paused {
|
||||
return fmt.Sprintf("%s %s, %s", p.Named(), where, waiting)
|
||||
return fmt.Sprintf("%s · %s, %s", name, where, waiting)
|
||||
}
|
||||
late := ""
|
||||
if since > bound {
|
||||
@@ -1211,7 +1286,53 @@ func planLineWith(p inventory.Plan, now time.Time, pause pauseView, bound time.D
|
||||
if p.State == inventory.PlanRolling {
|
||||
what = p.Note
|
||||
}
|
||||
return fmt.Sprintf("%s %s, %s for %s%s", p.Named(), where, what, since, late)
|
||||
return fmt.Sprintf("%s · %s, %s for %s%s", name, where, what, since, late)
|
||||
}
|
||||
|
||||
// planHeadline names a plan as a person knows it (asked by the operator, 2026-10-10: a plan called by its
|
||||
// repository alone said nothing of what it delivers): each repository as its pull request and title, what the
|
||||
// plan moves, and the machines running that — "mesh-catalog #175 a tap shows its outcome · messenger,
|
||||
// telegram → novox". A record naming no pull request is named by its commit, as before; one naming no moves
|
||||
// says none; running nil names no machine.
|
||||
func planHeadline(p inventory.Plan, running func(string) []string) string {
|
||||
var repos []string
|
||||
moves := map[string]bool{}
|
||||
for _, c := range p.Carried() {
|
||||
seg := repoName(c.Repository) + " " + short(c.Commit)
|
||||
if p.Delivery != nil {
|
||||
for _, m := range p.Delivery.Merges {
|
||||
if !strings.EqualFold(m.Repository, c.Repository) {
|
||||
continue
|
||||
}
|
||||
for _, n := range m.Moves {
|
||||
moves[n] = true
|
||||
}
|
||||
if m.Commit == c.Commit && m.Number > 0 {
|
||||
seg = repoName(c.Repository) + " " + pullWords(m)
|
||||
}
|
||||
}
|
||||
}
|
||||
repos = append(repos, seg)
|
||||
}
|
||||
out := strings.Join(repos, " + ")
|
||||
if len(moves) == 0 {
|
||||
return out
|
||||
}
|
||||
names := sortedKeysOf(boolsToStrings(moves))
|
||||
out += " · " + strings.Join(names, ", ")
|
||||
if running == nil {
|
||||
return out
|
||||
}
|
||||
nodes := map[string]bool{}
|
||||
for _, n := range names {
|
||||
for _, node := range running(n) {
|
||||
nodes[node] = true
|
||||
}
|
||||
}
|
||||
if len(nodes) > 0 {
|
||||
out += " → " + strings.Join(sortedKeysOf(boolsToStrings(nodes)), ", ")
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// planFailedBuild marks the module a failed build was for when the result names no module: by the
|
||||
@@ -1353,8 +1474,11 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
bounds := readTierBounds(ctx, inv, now)
|
||||
fmt.Printf("%s — %s\n", p.ID, planLineWith(p, now, buildSeatPause(ctx, inv, []inventory.Plan{p}),
|
||||
bounds.of(p.Repository)))
|
||||
fmt.Printf("%s — %s\n", p.ID, planLineOn(p, now, buildSeatPause(ctx, inv, []inventory.Plan{p}),
|
||||
bounds.of(p.Repository), func(module string) []string {
|
||||
on, _ := inv.Running(ctx, module)
|
||||
return on
|
||||
}))
|
||||
if r := p.Release; r != nil {
|
||||
// A release plan's walk (ADR 0236): machines done, the one judged, those to come.
|
||||
fmt.Printf(" machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "),
|
||||
@@ -1490,14 +1614,22 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
for _, b := range batches {
|
||||
fmt.Printf("%-28s %s\n", b.ID, batchWords(b, now))
|
||||
// One line per repository: its pull request, what it answers, what it moves.
|
||||
for _, line := range batchLines(b) {
|
||||
fmt.Printf("%-28s %s\n", "", line)
|
||||
}
|
||||
}
|
||||
pause := buildSeatPause(ctx, inv, plans)
|
||||
bounds := readTierBounds(ctx, inv, now)
|
||||
running := func(module string) []string {
|
||||
on, _ := inv.Running(ctx, module)
|
||||
return on
|
||||
}
|
||||
for _, p := range plans {
|
||||
if p.Batch() {
|
||||
continue
|
||||
}
|
||||
fmt.Printf("%-28s %s\n", p.ID, planLineWith(p, now, pause, bounds.of(p.Repository)))
|
||||
fmt.Printf("%-28s %s\n", p.ID, planLineOn(p, now, pause, bounds.of(p.Repository), running))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1627,6 +1759,34 @@ func splitList(s string) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// recordsOfAsked is what settleFromRecords reads: for every module of the tier still `asked`, its last
|
||||
// builds and the record of its own ask, by id.
|
||||
func recordsOfAsked(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan, tier []string) (
|
||||
map[string][]inventory.Build, map[string]inventory.Build, error) {
|
||||
recorded := map[string][]inventory.Build{}
|
||||
byID := map[string]inventory.Build{}
|
||||
for _, m := range tier {
|
||||
if s := p.Modules[m]; s != nil && s.State == "asked" {
|
||||
builds, err := inv.Builds(ctx, m, 5)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
recorded[m] = builds
|
||||
// Its own ask's record, by id — found even when the outcome named no module (ADR 0219).
|
||||
if s.Build != "" {
|
||||
b, found, err := inv.BuildByID(ctx, s.Build)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if found {
|
||||
byID[s.Build] = b
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return recorded, byID, nil
|
||||
}
|
||||
|
||||
// settleFromRecords marks every module of the tier still `asked` built — or failed — from a build
|
||||
// recorded after it was asked, and says whether it changed anything (novox/hq 04-ISSUES/214).
|
||||
// Newest first, as Builds answers: the first record after the ask is the outcome of that ask.
|
||||
|
||||
@@ -768,10 +768,22 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
}
|
||||
return append(argv, "--json"), nil
|
||||
case "give":
|
||||
// The same line as secret-ask with the desk named (novox/hq ADR 0277): one path, one set of bounds.
|
||||
if err := need("node", "module", "secret", "at"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []string{"secret", "accept", str("node"), str("module"), str("secret"), "--at-desk", str("at")}, nil
|
||||
return []string{"secret", "ask", str("node"), str("module"), str("secret"), "--at", str("at")}, nil
|
||||
case "secret-ask":
|
||||
// A module's own secret asked for, typed by the operator at the desk (novox/hq ADR 0277): never a value
|
||||
// in the arguments. The desk is the module's machine unless at names another.
|
||||
if err := need("node", "module", "secret"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argv := []string{"secret", "ask", str("node"), str("module"), str("secret")}
|
||||
if at := str("at"); at != "" {
|
||||
argv = append(argv, "--at", at)
|
||||
}
|
||||
return argv, nil
|
||||
case "rotate":
|
||||
if p := str("provision"); p != "" {
|
||||
argv := []string{"rotate", p}
|
||||
@@ -1354,7 +1366,7 @@ func splitCommandLine(line string) ([]string, error) {
|
||||
var words []string
|
||||
var cur strings.Builder
|
||||
inWord := false
|
||||
quote := rune(0)
|
||||
quote, opened := rune(0), 0
|
||||
runes := []rune(line)
|
||||
for i := 0; i < len(runes); i++ {
|
||||
r := runes[i]
|
||||
@@ -1375,7 +1387,7 @@ func splitCommandLine(line string) ([]string, error) {
|
||||
cur.WriteRune(r)
|
||||
}
|
||||
case r == '\'' || r == '"':
|
||||
quote = r
|
||||
quote, opened = r, i
|
||||
inWord = true
|
||||
case r == '\\' && i+1 < len(runes):
|
||||
i++
|
||||
@@ -1393,7 +1405,7 @@ func splitCommandLine(line string) ([]string, error) {
|
||||
}
|
||||
}
|
||||
if quote != 0 {
|
||||
return nil, fmt.Errorf("command has an unclosed %c quote", quote)
|
||||
return nil, unclosedQuote(quote, opened)
|
||||
}
|
||||
if inWord {
|
||||
words = append(words, cur.String())
|
||||
@@ -1401,6 +1413,17 @@ func splitCommandLine(line string) ([]string, error) {
|
||||
return words, nil
|
||||
}
|
||||
|
||||
// unclosedQuote is the refusal of a line whose quote is never closed. Most often an apostrophe inside
|
||||
// a single-quoted value ended that quote early and a later quote was left open, so the refusal says
|
||||
// where the open quote is and how a quote is written inside a quoted value — the shell's own two
|
||||
// ways, which this splitter already reads (novox/hq issue 294). It quotes none of the line: a refusal
|
||||
// is kept on the bus as the call's answer, and the line may carry a setting's value.
|
||||
func unclosedQuote(quote rune, opened int) error {
|
||||
return fmt.Errorf("command has an unclosed %c quote, opened at character %d — an apostrophe "+
|
||||
"inside a single-quoted value ends it; write a ' inside single quotes as '\\'' (it'\\''s), or use "+
|
||||
"double quotes and write \\\" for a \" and \\\\ for a \\ inside them", quote, opened+1)
|
||||
}
|
||||
|
||||
// seatAnnouncement is what the controller says it serves on the bus (novox/hq ADR 0197): the
|
||||
// mesh-controller seat, one endpoint per verb it answers, each with the seat's own description and
|
||||
// argument schema — the same facts `tools` answers from the records, as NATS's services format.
|
||||
@@ -1538,10 +1561,11 @@ var terminalOnlyCommands = map[string]string{
|
||||
"licence": "the licences' secrets",
|
||||
}
|
||||
|
||||
// givenAtTheDesk is exactly the line the `give` verb composes, and nothing beside it: `secret accept <node>
|
||||
// <module> <secret> --at-desk <machine>`, with no other word — no value, no file, no provider.
|
||||
// givenAtTheDesk is exactly the line the `give` and `secret-ask` verbs compose, and nothing beside it: `secret ask
|
||||
// <node> <module> <secret>`, with `--at <machine>` or no other word — no value, no file, no provider (novox/hq
|
||||
// ADR 0277). The terminal's own `secret accept … --at-desk` is the terminal's.
|
||||
func givenAtTheDesk(argv []string) bool {
|
||||
if len(argv) != 7 || argv[0] != "secret" || argv[1] != "accept" || argv[5] != "--at-desk" {
|
||||
if (len(argv) != 5 && len(argv) != 7) || argv[0] != "secret" || argv[1] != "ask" {
|
||||
return false
|
||||
}
|
||||
for _, w := range argv[2:5] {
|
||||
@@ -1549,7 +1573,10 @@ func givenAtTheDesk(argv []string) bool {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return argv[6] != "" && !strings.HasPrefix(argv[6], "-")
|
||||
if len(argv) == 5 {
|
||||
return true
|
||||
}
|
||||
return argv[5] == "--at" && argv[6] != "" && !strings.HasPrefix(argv[6], "-")
|
||||
}
|
||||
|
||||
// terminalOnly refuses, through any verb, a command that is the operator's at the controller's terminal
|
||||
|
||||
@@ -276,12 +276,6 @@ var accountedFlags = map[string]map[string]string{
|
||||
"all": "withheld: every measurement of a fortnight is more than a call should carry; `command` reaches it",
|
||||
},
|
||||
// The desk path of `secret accept` (novox/hq ADR 0259 §10): a value is never an argument of a call.
|
||||
"secret accept": {
|
||||
"at-desk": "=at",
|
||||
"from": "withheld: a file of the control node's is read at a shell, never named by a call",
|
||||
"provider": "withheld: a pair credential's value is given at a shell; give takes a module's own secret",
|
||||
"local": "withheld: it goes with --provider",
|
||||
},
|
||||
"hand-acts": {"json": "set by the verb: the answer is data"},
|
||||
"conditions": {"json": "set by the verb: the answer is data"},
|
||||
"retire": {"json": "set by the verb: the answer is data"},
|
||||
|
||||
@@ -415,3 +415,48 @@ func TestNoVerbSetsTheOperatorsKeyOrRevealsASecret(t *testing.T) {
|
||||
t.Fatalf("behind %v: %v", behind, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A value with a quote in it can be said on a `command` line, as in a shell, and a line whose quote
|
||||
// is left open is refused naming where it opened and how a quote is written, quoting none of it (novox/hq issue 294: an
|
||||
// apostrophe inside a single-quoted JSON value cut the line, and the refusal named no cause).
|
||||
func TestAQuotedValueCanHoldAQuote(t *testing.T) {
|
||||
for _, c := range []struct{ line, want string }{
|
||||
{`settings set claude-code '{"role":"the operator'\''s laptop"}'`, `{"role":"the operator's laptop"}`},
|
||||
{`settings set claude-code "{\"role\":\"the operator's laptop\"}"`, `{"role":"the operator's laptop"}`},
|
||||
{"x \"a \\\\ b\nc\"", "a \\ b\nc"},
|
||||
{`x 'a\b'`, `a\b`},
|
||||
} {
|
||||
argv, err := splitCommandLine(c.line)
|
||||
if err != nil || argv[len(argv)-1] != c.want {
|
||||
t.Errorf("%s: read back %q %v, want %q", c.line, argv, err, c.want)
|
||||
}
|
||||
}
|
||||
|
||||
_, err := splitCommandLine(`settings set claude-code '{"role":"the operator's laptop"}' --node g14`)
|
||||
if err == nil {
|
||||
t.Fatal("an apostrophe that leaves a quote open was accepted")
|
||||
}
|
||||
for _, want := range []string{"character 57", `'\''`, `\"`} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not say %q: %v", want, err)
|
||||
}
|
||||
}
|
||||
if strings.Contains(err.Error(), "laptop") || strings.Contains(err.Error(), "g14") {
|
||||
t.Errorf("the refusal quotes the line, which may carry a setting's value: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Every value reads back as it was when written the way the refusal says: single-quoted with '\”
|
||||
// for each quote, or double-quoted with \ before each " and \ — newlines and backslashes included.
|
||||
func TestAQuotedValueRoundTrips(t *testing.T) {
|
||||
for _, v := range []string{`plain`, `it's`, `say "hi"`, `back\slash\`, "two\nlines", `'"\'\"`, `''`, ``} {
|
||||
single := "x '" + strings.ReplaceAll(v, "'", `'\''`) + "'"
|
||||
double := `x "` + strings.NewReplacer(`\`, `\\`, `"`, `\"`).Replace(v) + `"`
|
||||
for _, line := range []string{single, double} {
|
||||
argv, err := splitCommandLine(line)
|
||||
if err != nil || len(argv) != 2 || argv[1] != v {
|
||||
t.Errorf("%s: read back %q %v, want %q", line, argv, err, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,6 +39,8 @@ func secretCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
switch args[0] {
|
||||
case "accept":
|
||||
case "ask":
|
||||
return secretAsk(ctx, args[1:])
|
||||
case "rotate":
|
||||
return secretRotate(ctx, args[1:])
|
||||
case "recover":
|
||||
@@ -78,7 +80,7 @@ func secretCommand(ctx context.Context, args []string) error {
|
||||
if *from != "" || *provider != "" {
|
||||
return errors.New("--at-desk gives a module's own secret, and takes neither --from nor --provider")
|
||||
}
|
||||
return giveAtDesk(ctx, node, module, name, *desk)
|
||||
return askAtDesk(ctx, node, module, name, *desk)
|
||||
}
|
||||
|
||||
value, err := valueFor(node, module, name, *from)
|
||||
@@ -148,7 +150,24 @@ func secretCommand(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// secretAsk is `secret ask <node> <module> <name> [--at <machine>]` (novox/hq ADR 0277): the operator is asked
|
||||
// for a module's own secret in a prompt at the desk, which only they answer. The one `secret` line a verb may
|
||||
// run beside rotate (givenAtTheDesk): it carries no value and answers none.
|
||||
func secretAsk(ctx context.Context, args []string) error {
|
||||
rest, flags := split(args)
|
||||
set := flag.NewFlagSet("secret ask", flag.ContinueOnError)
|
||||
at := set.String("at", "", "the machine the operator sits at, where the prompt opens; the module's machine when absent")
|
||||
if err := set.Parse(flags); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(rest) != 3 {
|
||||
return errors.New("secret ask <node> <module> <name> [--at <machine>]")
|
||||
}
|
||||
return askAtDesk(ctx, rest[0], rest[1], rest[2], *at)
|
||||
}
|
||||
|
||||
const secretUsage = "secret rotate <node> <module> <name> [--why <text> [--cause <word>]]\n" +
|
||||
"secret ask <node> <module> <name> [--at <machine>]\n" +
|
||||
"secret accept <node> <module> <name> [--from <file> | --at-desk <machine>] [--provider <node> [--local <name>]]\n" +
|
||||
"secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>] [--provider <node>]\n" +
|
||||
"secret export [--out <file>]"
|
||||
|
||||
@@ -523,6 +523,14 @@ func watchAdvisories(f *signalFacts) []conditions.Observation {
|
||||
if a.Kind == link.AdvisoryConsumerLost && !f.lostConsumers[a.Stream+"."+a.Consumer] {
|
||||
continue // it exists again, or the mesh no longer expects it: a removal, not a loss
|
||||
}
|
||||
if a.Kind == link.AdvisoryMaxDeliveries && a.Token == "" {
|
||||
// A consumer's max-deliveries key is the dead-letter row's (novox/hq issue 330): said while
|
||||
// DEAD_LETTERS holds what it gave up on, cleared when that is delivered again or dropped. The
|
||||
// listener records no such advisory today; one read here as well added a look to the count and
|
||||
// overwrote the row's words on every look (novox/hq issue 440). Only one that could not be kept
|
||||
// (AdvisoryNotKept), which DEAD_LETTERS cannot say, is said from here.
|
||||
continue
|
||||
}
|
||||
severity := conditions.Warning
|
||||
times := ""
|
||||
if a.Count > 1 {
|
||||
@@ -533,7 +541,10 @@ func watchAdvisories(f *signalFacts) []conditions.Observation {
|
||||
machine = f.host
|
||||
}
|
||||
o := conditions.Observation{Scope: conditions.ScopeBus, ID: a.ID, Kind: a.Kind, Token: a.Token,
|
||||
Machine: machine, Severity: severity, Summary: a.Said + times, Said: a.Said}
|
||||
Machine: machine, Severity: severity, Summary: a.Said + times, Said: a.Said,
|
||||
// The advisory is remembered and read again on every look for an hour: the condition counts
|
||||
// what the bus said and when, not the looks (novox/hq issue 402).
|
||||
Happened: a.Last, Times: a.Count}
|
||||
if a.Kind == link.AdvisoryMaxDeliveries && a.Token == link.AdvisoryNotKept {
|
||||
o.Machine = consumerMachine(a.Stream, a.Consumer)
|
||||
o.Headline = clip(conditions.Capital(fmt.Sprintf("%s gave up on a message, not kept",
|
||||
@@ -565,7 +576,17 @@ func watchDeadLetters(f *signalFacts) []conditions.Observation {
|
||||
messages, them = fmt.Sprintf("%d messages", n), "them"
|
||||
}
|
||||
who := consumerWho(stream, consumer)
|
||||
// DEAD_LETTERS is a record of what was given up on: the condition says when the newest held message
|
||||
// was kept, and its count is a running total of the messages given up on while it is open, never
|
||||
// how often the controller looked (novox/hq issues 402 and 440). It is at least the number held now,
|
||||
// and does not go down when one is delivered again or dropped. Without that time it counts its
|
||||
// looks, as a source of the present does.
|
||||
happened, times := f.deadLettersNewest[key], 0
|
||||
if !happened.IsZero() {
|
||||
times = n
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: key, Kind: link.AdvisoryMaxDeliveries,
|
||||
Happened: happened, Times: times,
|
||||
Machine: consumerMachine(stream, consumer), Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s gave up on %s; %s kept in %s until delivered again or dropped, with why, "+
|
||||
"through the controller's dead-letters verb", link.ConsumerInWords(stream, consumer), messages,
|
||||
|
||||
@@ -100,14 +100,15 @@ var suppressions = map[string]suppression{
|
||||
inside: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-29 * time.Minute))} },
|
||||
past: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-31 * time.Minute))} },
|
||||
},
|
||||
// A message given up on and not kept: the one max-deliveries advisory S9 says itself (issue 440).
|
||||
"S9": {
|
||||
inside: func(f *signalFacts) {
|
||||
f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop", Said: "gave up",
|
||||
First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-61 * time.Minute), Count: 1}}
|
||||
f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop",
|
||||
Token: link.AdvisoryNotKept, Said: "gave up, not kept", First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-61 * time.Minute), Count: 1}}
|
||||
},
|
||||
past: func(f *signalFacts) {
|
||||
f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop", Said: "gave up",
|
||||
First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-59 * time.Minute), Count: 1}}
|
||||
f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop",
|
||||
Token: link.AdvisoryNotKept, Said: "gave up, not kept", First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-59 * time.Minute), Count: 1}}
|
||||
},
|
||||
},
|
||||
"S10": {
|
||||
|
||||
@@ -0,0 +1,384 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A provider waiting for the operator holds its consumers, and a wait beside another wait is said (novox/hq
|
||||
// issue 405, found in the review of ADR 0283's controller change).
|
||||
//
|
||||
// The shapes are those of issue 386 (mounts waiting for smb-password-games: waitingResource, passwordWait) and of
|
||||
// the openrazer wait on the workstation of 2026-10-11 (relogin, userUnit): an account in its group whose session
|
||||
// began before it was, and its unit failed in that account's own service manager.
|
||||
|
||||
// waitingDatabase is a provider whose only part not healthy waits for the operator's secret: a database's
|
||||
// process stated waiting, as the node-engine states a tool check answering waits (ADR 0283 decision 2). Its wait
|
||||
// names the part that waits by the provision it gives.
|
||||
func waitingDatabase() inventory.ResourceHealth {
|
||||
return waitingDatabaseFor(inventory.Wait{Part: "postgres-database", Secret: "licence",
|
||||
What: "the licence key of the database"})
|
||||
}
|
||||
|
||||
func waitingDatabaseFor(waits ...inventory.Wait) inventory.ResourceHealth {
|
||||
return inventory.ResourceHealth{Module: "db", Resource: "db.server", Kind: "process", Target: "db.service",
|
||||
State: link.StateWaiting, Check: "tool", Reason: "the database waits for its licence", Waits: waits}
|
||||
}
|
||||
|
||||
// backupsWait is a wait of the same provider for another part than the one its consumers need.
|
||||
var backupsWait = inventory.Wait{Part: "the nightly backups", Secret: "backup-key", What: "the key of the backups"}
|
||||
|
||||
// failingConsumer is a consumer whose check that needs the database fails.
|
||||
func failingConsumer(module string) inventory.ResourceHealth {
|
||||
return inventory.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module,
|
||||
State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"}
|
||||
}
|
||||
|
||||
// dbSecrets is what the database's waits name: own secrets issued outside the mesh, so a wait for one checks out
|
||||
// while nobody gave it (ADR 0283 decision 3, novox/hq issue 450).
|
||||
var dbSecrets = catalogue.OwnSecrets{
|
||||
"licence": {Path: "/s/licence", IssuedBy: catalogue.IssuedOutside},
|
||||
"backup-key": {Path: "/s/backup-key", IssuedBy: catalogue.IssuedOutside},
|
||||
}
|
||||
|
||||
// holdingOf is one reading of the record without a store: the newest statements, the open conditions, the
|
||||
// catalogue, what was given on the provider's machine (nothing), and each consumer's provider already looked up,
|
||||
// as providerFor memoises it.
|
||||
func holdingOf(open []conditions.Condition, healths map[string]inventory.NodeHealth, bound map[[3]string]catalogue.Chosen) *holding {
|
||||
h := &holding{ctx: context.Background(), healths: healths, open: open, providers: map[string]providerLookup{},
|
||||
shelf: map[string]catalogue.Manifest{"db": {Module: "db", Version: "1", OwnSecrets: dbSecrets,
|
||||
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}}},
|
||||
waits: operatorWaitFacts{given: map[string]map[string]time.Time{"db@anchor": {}}}}
|
||||
for k, p := range bound {
|
||||
h.providers[k[0]+"\x00"+k[1]+"\x00"+k[2]] = providerLookup{p, true}
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
var theDatabase = catalogue.Chosen{Node: "anchor", Module: "db"}
|
||||
|
||||
var shopOnTheDatabase = map[[3]string]catalogue.Chosen{{"laptop", "shop", "postgres-database"}: theDatabase}
|
||||
|
||||
func shopFailingBeside(provider ...inventory.ResourceHealth) map[string]inventory.NodeHealth {
|
||||
return map[string]inventory.NodeHealth{
|
||||
"anchor": {Node: "anchor", Resources: provider},
|
||||
"laptop": {Node: "laptop", Resources: []inventory.ResourceHealth{failingConsumer("shop")}},
|
||||
}
|
||||
}
|
||||
|
||||
// (1) A provider whose only part not healthy waits for the operator holds the findings of the consumers of the
|
||||
// waiting part under it (ADR 0240 rule 5); a consumer of another part, or one whose part cannot be matched, is
|
||||
// its own (ADR 0283 decision 1: a module that waits says nothing else of it is wrong).
|
||||
func TestAProviderWaitingForTheOperatorHoldsOnlyTheConsumersOfTheWaitingPart(t *testing.T) {
|
||||
shop := []inventory.ResourceHealth{failingConsumer("shop")}
|
||||
unhealthyDB := waitingDatabase()
|
||||
unhealthyDB.State, unhealthyDB.Waits, unhealthyDB.Reason = link.StateUnhealthy, nil, "its tool check: refused"
|
||||
healthyDB := waitingDatabase()
|
||||
healthyDB.State, healthyDB.Waits = link.StateHealthy, nil
|
||||
byCredential := holdingOf(nil, shopFailingBeside(waitingDatabaseFor(inventory.Wait{Part: "the server",
|
||||
Secret: "licence", What: "the licence key"})), shopOnTheDatabase)
|
||||
byCredential.shelf["db"] = catalogue.Manifest{Module: "db", Version: "1", OwnSecrets: dbSecrets, Provides: []catalogue.Offer{{
|
||||
Name: "postgres-database", Credential: &catalogue.OfferCredential{Own: "licence"}}}}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
hold *holding
|
||||
held bool
|
||||
onlyWaits bool
|
||||
uncovered bool
|
||||
}{
|
||||
{"the waiting part is the provision", holdingOf(nil, shopFailingBeside(waitingDatabase()), shopOnTheDatabase), true, true, false},
|
||||
{"the wait is for the provision's shared credential", byCredential, true, true, false},
|
||||
{"the wait is for another part", holdingOf(nil, shopFailingBeside(waitingDatabaseFor(backupsWait)), shopOnTheDatabase), false, false, true},
|
||||
{"only the needs-operator condition, its parts not said", holdingOf(
|
||||
[]conditions.Condition{{Key: needsOperatorKey("db", "anchor"), Kind: kindNeedsOperator}},
|
||||
shopFailingBeside(), shopOnTheDatabase), false, false, true},
|
||||
{"an unhealthy provider holds as before", holdingOf(nil, shopFailingBeside(unhealthyDB), shopOnTheDatabase), true, false, false},
|
||||
{"a healthy provider holds nothing", holdingOf(nil, shopFailingBeside(healthyDB), shopOnTheDatabase), false, false, false},
|
||||
} {
|
||||
by, held := c.hold.heldWith("laptop", "shop", shop)
|
||||
if held != c.held || (held && by.provider != theDatabase) || (len(by.waits) > 0) != c.onlyWaits {
|
||||
t.Errorf("%s: held %v under %v with waits %v; want held %v, only waits %v", c.name, held, by.provider,
|
||||
by.waits, c.held, c.onlyWaits)
|
||||
}
|
||||
if _, uncovered := c.hold.waitingUncovered("laptop", "shop", shop); uncovered != c.uncovered {
|
||||
t.Errorf("%s: said as a provider waiting for another part %v; want %v", c.name, uncovered, c.uncovered)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// (1) The consumer's first-node gate under a provider that only waits for the operator passes as a wait for a
|
||||
// person (ADR 0254), carrying the wait (ADR 0283 decision 4); under an unhealthy provider it waits, as before.
|
||||
func TestAConsumersGateUnderAWaitingProviderPassesCarryingTheWait(t *testing.T) {
|
||||
now := time.Now()
|
||||
since := now.Add(-time.Minute)
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
provider inventory.ResourceHealth
|
||||
want health
|
||||
says []string
|
||||
}{
|
||||
{"a provider that only waits", waitingDatabase(), healthPerson,
|
||||
[]string{"waits on db on anchor, which waits for you", "the licence key of the database", "nox secret ask anchor db licence"}},
|
||||
{"an unhealthy provider", func() inventory.ResourceHealth {
|
||||
r := waitingDatabase()
|
||||
r.State, r.Waits, r.Reason = link.StateUnhealthy, nil, "its tool check: refused"
|
||||
return r
|
||||
}(), healthWaiting, []string{"waits on db on anchor, which is unhealthy"}},
|
||||
} {
|
||||
healths := shopFailingBeside(c.provider)
|
||||
for m, h := range healths {
|
||||
h.HeardAt = now
|
||||
healths[m] = h
|
||||
}
|
||||
f := gateFacts{now: now, health: healths, heldOn: heldReadings(holdingOf(nil, healths, shopOnTheDatabase))}
|
||||
h, why := moduleHealthWord("shop", "laptop", since, f)
|
||||
if h != c.want {
|
||||
t.Errorf("%s: the consumer's gate reads %v %q; want %v", c.name, h, why, c.want)
|
||||
continue
|
||||
}
|
||||
for _, s := range c.says {
|
||||
if !strings.Contains(why, s) {
|
||||
t.Errorf("%s: the gate's reading %q does not say %q", c.name, why, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// judgeBoth judges the provider's statement and then the consumer's, two looks each, over a record that changes
|
||||
// as the statements do.
|
||||
func judgeBoth(t *testing.T, k *conditions.Keeper, provider []inventory.ResourceHealth,
|
||||
byProvider, byConsumer map[string]inventory.NodeHealth) []conditions.Condition {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
was := readHoldingFor
|
||||
t.Cleanup(func() { readHoldingFor = was })
|
||||
healths := byProvider
|
||||
readHoldingFor = func(_ context.Context, _ *inventory.Inventory, open []conditions.Condition) (*holding, error) {
|
||||
return holdingOf(open, healths, shopOnTheDatabase), nil
|
||||
}
|
||||
for look := 1; look <= 2; look++ {
|
||||
healths = byProvider
|
||||
if err := judgeModuleHealth(ctx, nil, k, "anchor", map[string][]inventory.ResourceHealth{"db": provider},
|
||||
map[string]int{"db": look}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for look := 1; look <= 2; look++ {
|
||||
healths = byConsumer
|
||||
if err := judgeModuleHealth(ctx, nil, k, "laptop", map[string][]inventory.ResourceHealth{"shop": {failingConsumer("shop")}},
|
||||
map[string]int{"shop": look}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
open, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return open
|
||||
}
|
||||
|
||||
func conditionOf(open []conditions.Condition, key string) *conditions.Condition {
|
||||
for i, c := range open {
|
||||
if c.Key == key {
|
||||
return &open[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// (1) The consumer raises nothing of its own; the provider's needs-operator condition lists who waits on it and
|
||||
// stays a warning (ADR 0283 decision 5: never escalated). The consumer's statement arrives after the provider's,
|
||||
// so it is the consumer's judging (sayWaiters) that lists it at the provider — no store involved.
|
||||
func TestAWaitingProvidersConditionListsWhoWaitsOnItAndStaysAWarning(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
open := judgeBoth(t, k, []inventory.ResourceHealth{waitingDatabase()},
|
||||
map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Resources: []inventory.ResourceHealth{waitingDatabase()}}}, shopFailingBeside(waitingDatabase()))
|
||||
provider := conditionOf(open, needsOperatorKey("db", "anchor"))
|
||||
if len(open) != 1 || provider == nil {
|
||||
t.Fatalf("a provider waiting for the operator and a consumer of its waiting part raised %v; want the "+
|
||||
"provider's needs-operator alone", openKeysOf(open))
|
||||
}
|
||||
if said := provider.Evidence[0].Said; !strings.Contains(said, "shop on laptop") {
|
||||
t.Fatalf("the provider's needs-operator does not list shop on laptop as waiting on it: %s", said)
|
||||
}
|
||||
if provider.Severity != conditions.Warning {
|
||||
t.Fatalf("the provider's needs-operator became %s with a consumer waiting; it stays a warning", provider.Severity)
|
||||
}
|
||||
if provider.Resolver != conditions.ResolverOperator || !strings.Contains(provider.Needs, "desk prompt") {
|
||||
t.Fatalf("the provider's condition: %+v", provider)
|
||||
}
|
||||
}
|
||||
|
||||
// (1) A consumer of another part than the one waiting is raised on its own, and says its provider waits.
|
||||
func TestAConsumerOfAnotherPartIsRaisedOnItsOwn(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
backups := waitingDatabaseFor(backupsWait)
|
||||
open := judgeBoth(t, k, []inventory.ResourceHealth{backups}, shopFailingBeside(backups), shopFailingBeside(backups))
|
||||
consumer := conditionOf(open, moduleUnhealthyKey("shop", "laptop"))
|
||||
if consumer == nil || conditionOf(open, needsOperatorKey("db", "anchor")) == nil {
|
||||
t.Fatalf("raised %v; want shop's own unhealthy beside db's needs-operator", openKeysOf(open))
|
||||
}
|
||||
if said := consumer.Evidence[0].Said; !strings.Contains(said, "db on anchor waits for you, but not for anything shop is known to need, so shop's fault is said on its own") {
|
||||
t.Fatalf("the consumer's condition does not say its provider waits: %s", said)
|
||||
}
|
||||
}
|
||||
|
||||
// relogin and userUnit are person_wait_test.go's; mounts is said here with the same account and unit beside its
|
||||
// wait for the password.
|
||||
func reloginBesideAWait() []inventory.ResourceHealth {
|
||||
return []inventory.ResourceHealth{relogin("mounts", "operator"), userUnit("mounts", "operator"),
|
||||
waitingResource(passwordWait)}
|
||||
}
|
||||
|
||||
// (2) A module with a checked wait beside a relogin-needed account says both: the new login, and the act the
|
||||
// operator owes it.
|
||||
func TestAWaitBesideAReloginIsSaid(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
ctx := t.Context()
|
||||
for look := 1; look <= 2; look++ {
|
||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": reloginBesideAWait()},
|
||||
map[string]int{"mounts": look}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, open := needsOperatorOpen(t, k)
|
||||
if got == nil {
|
||||
t.Fatalf("a wait for the operator beside a relogin is not said: %v", openKeysOf(open))
|
||||
}
|
||||
if !strings.Contains(got.Summary, "smb-password-games") || got.Severity != conditions.Warning {
|
||||
t.Fatalf("the needs-operator beside the relogin: %+v", got)
|
||||
}
|
||||
relogged := false
|
||||
for _, c := range open {
|
||||
relogged = relogged || c.Key == reloginKey("mounts", "workstation")
|
||||
}
|
||||
if !relogged {
|
||||
t.Fatalf("the relogin is no longer said beside the wait: %v", openKeysOf(open))
|
||||
}
|
||||
// The login done, the wait stays said and the relogin clears.
|
||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}},
|
||||
map[string]int{"mounts": 3}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, open = needsOperatorOpen(t, k)
|
||||
if got == nil || len(open) != 1 {
|
||||
t.Fatalf("after the new login: %v", openKeysOf(open))
|
||||
}
|
||||
}
|
||||
|
||||
// (2) The same beside a directory used as found.
|
||||
func TestAWaitBesideADirectoryUsedAsFoundIsSaid(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
ctx := t.Context()
|
||||
found := foundDirectory("mounts", time.Now().Add(-time.Hour))
|
||||
for look := 1; look <= 2; look++ {
|
||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{
|
||||
"mounts": {found, waitingResource(passwordWait)}}, map[string]int{"mounts": look}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, open := needsOperatorOpen(t, k)
|
||||
if got == nil {
|
||||
t.Fatalf("a wait for the operator beside a directory used as found is not said: %v", openKeysOf(open))
|
||||
}
|
||||
asFound := false
|
||||
for _, c := range open {
|
||||
asFound = asFound || c.Key == usedAsFoundKey("mounts", "workstation")
|
||||
}
|
||||
if !asFound {
|
||||
t.Fatalf("the directory used as found is no longer said beside the wait: %v", openKeysOf(open))
|
||||
}
|
||||
}
|
||||
|
||||
// (2) Beside a fault of its own, the wait is said too, and the fault's words do not count the waiting part among
|
||||
// what fails.
|
||||
func TestAWaitBesideAFaultIsSaidAndTheFaultIsTheFaultAlone(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
ctx := t.Context()
|
||||
down := inventory.ResourceHealth{Module: "mounts", Resource: "mounts.apply", Kind: "process",
|
||||
Target: "mesh-mounts-apply.service", State: link.StateUnhealthy, Reason: "down"}
|
||||
for look := 1; look <= 2; look++ {
|
||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{
|
||||
"mounts": {down, waitingResource(passwordWait)}}, map[string]int{"mounts": look}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, open := needsOperatorOpen(t, k)
|
||||
if got == nil {
|
||||
t.Fatalf("a wait for the operator beside a fault is not said: %v", openKeysOf(open))
|
||||
}
|
||||
var fault *conditions.Condition
|
||||
for i, c := range open {
|
||||
if c.Key == moduleUnhealthyKey("mounts", "workstation") {
|
||||
fault = &open[i]
|
||||
}
|
||||
}
|
||||
if fault == nil {
|
||||
t.Fatalf("the fault is not said: %v", openKeysOf(open))
|
||||
}
|
||||
if strings.Contains(fault.Summary, "mounts.watch") {
|
||||
t.Fatalf("the fault's summary counts the waiting part as failing: %q", fault.Summary)
|
||||
}
|
||||
}
|
||||
|
||||
func openKeysOf(cs []conditions.Condition) []string {
|
||||
var out []string
|
||||
for _, c := range cs {
|
||||
out = append(out, c.Key)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// A consumer raised on its own, whose provider then waits for the operator for the part it needs, is cleared saying
|
||||
// which the provider is: it waits for you, not that it is unhealthy (novox/hq issue 405 review).
|
||||
func TestAConsumerHeldOnceItsProviderWaitsSaysWhichItIs(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
ctx := t.Context()
|
||||
start := time.Now().Add(-time.Second)
|
||||
healthy := waitingDatabase()
|
||||
healthy.State, healthy.Waits = link.StateHealthy, nil
|
||||
healths := shopFailingBeside(healthy)
|
||||
was := readHoldingFor
|
||||
t.Cleanup(func() { readHoldingFor = was })
|
||||
readHoldingFor = func(_ context.Context, _ *inventory.Inventory, open []conditions.Condition) (*holding, error) {
|
||||
return holdingOf(open, healths, shopOnTheDatabase), nil
|
||||
}
|
||||
shop := map[string][]inventory.ResourceHealth{"shop": {failingConsumer("shop")}}
|
||||
for look := 1; look <= 2; look++ {
|
||||
if err := judgeModuleHealth(ctx, nil, k, "laptop", shop, map[string]int{"shop": look}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if open, _ := k.Open(ctx); conditionOf(open, moduleUnhealthyKey("shop", "laptop")) == nil {
|
||||
t.Fatalf("shop beside a healthy provider is not raised: %v", openKeysOf(open))
|
||||
}
|
||||
healths = shopFailingBeside(waitingDatabase())
|
||||
if err := judgeModuleHealth(ctx, nil, k, "laptop", shop, map[string]int{"shop": 3}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var why string
|
||||
for deadline := time.Now().Add(2 * time.Second); why == "" && time.Now().Before(deadline); {
|
||||
events, err := k.HistorySince(ctx, start)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, e := range events {
|
||||
if e.Key == moduleUnhealthyKey("shop", "laptop") && e.Change == conditions.ChangeCleared {
|
||||
why = e.Why
|
||||
}
|
||||
}
|
||||
if why == "" {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(why, "waits on db on anchor, which waits for you") {
|
||||
t.Fatalf("shop's clearing says %q; want it to say db on anchor waits for you", why)
|
||||
}
|
||||
}
|
||||
@@ -79,8 +79,11 @@ type signalFacts struct {
|
||||
lostConsumers map[string]bool
|
||||
// deadLetters are how many messages DEAD_LETTERS holds per consumer, by `<stream>.<consumer>`
|
||||
// (novox/hq issue 330): each consumer's max-deliveries condition is open while it holds any.
|
||||
deadLetters map[string]int
|
||||
advisoriesErr error
|
||||
deadLetters map[string]int
|
||||
// deadLettersNewest is when DEAD_LETTERS kept the newest message it holds for each of those
|
||||
// consumers: the condition counts the messages given up on, not the looks (novox/hq issue 440).
|
||||
deadLettersNewest map[string]time.Time
|
||||
advisoriesErr error
|
||||
|
||||
selfCheck selfCheckFacts
|
||||
|
||||
@@ -350,6 +353,16 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
|
||||
if f.advisoriesErr == nil && w.js != nil {
|
||||
f.deadLetters, f.advisoriesErr = link.HeldDeadLetters(w.js.Context())
|
||||
}
|
||||
if f.advisoriesErr == nil && len(f.deadLetters) > 0 {
|
||||
f.deadLettersNewest, f.advisoriesErr = link.NewestDeadLetters(w.js.Context(), f.deadLetters)
|
||||
for key := range f.deadLetters {
|
||||
if _, ok := f.deadLettersNewest[key]; !ok && f.advisoriesErr == nil {
|
||||
// Delivered again or dropped between the two reads: left out of this look, rather than
|
||||
// observed without a time and counted as a look (novox/hq issue 440).
|
||||
delete(f.deadLetters, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
f.handActs, f.handActsErr = w.gatherHandActs(ctx, now)
|
||||
f.facts.taken, _, f.facts.began, f.facts.err = exportedFacts.last()
|
||||
return f
|
||||
@@ -490,6 +503,11 @@ func gatherPlans(ctx context.Context, inv *inventory.Inventory, now time.Time, b
|
||||
awaits: p.Delivery.Awaits, since: p.Created, modules: planModules(p), merges: p.Delivery.Merges})
|
||||
continue
|
||||
}
|
||||
// A walk let go and waiting for the walk before it to end (ADR 0276) is no tier late either: the walk
|
||||
// before it is the one S3 watches.
|
||||
if deferred(p) {
|
||||
continue
|
||||
}
|
||||
_, paused := pausedWaiting(p, pause, now)
|
||||
bound := bounds.of(p.Repository)
|
||||
out = append(out, planFacts{id: p.ID, repository: p.Repository, commit: p.Commit, tier: p.Tier,
|
||||
|
||||
@@ -3,7 +3,7 @@ module github.com/novox/mesh-controller
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.13
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970
|
||||
github.com/jackc/pgx/v5 v5.10.0
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261009231844-b8c854611812 h1:pzVzwF5VMWaTECxu8+Pd1dNoOHNEm7upC5wPadQTkBw=
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261009231844-b8c854611812/go.mod h1:K3/xEzVgmrNKLMV2vv4M80MwmPnQNXqvQ4C5Jj0fJT4=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.13 h1:Su4JYpZ+zhNovkGA2DgxiZdcuHpjw2tPJzc/7PljhXg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.13/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970 h1:9tFDQsgmI+4X7/BpZGXIr+HemPKE7YddYGqWV0lINAI=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
// Package beside is where a test finds another repository of the mesh it reads: the catalogue's manifests,
|
||||
// the node-engine's genesis template (novox/hq issue 432), the SDK's conformance fixtures (issue 449).
|
||||
//
|
||||
// A test used to read the checkout beside this one, `../../../mesh-catalog`, so its verdict depended on
|
||||
// whatever sat on the machine running it: a stale or dirty checkout failed it on a desktop, and where none
|
||||
// was beside it skipped and said nothing. Now there are two inputs, both named, and no third:
|
||||
//
|
||||
// - In a merge check, the build seat clones each core repository beside the one checked (the catalogue
|
||||
// at its main, the node-engine at the commit the mesh runs) and says where in MESH_CHECK_BESIDE. A
|
||||
// test judges against those clones, so agreement with the other repository is checked where
|
||||
// `mesh/repo-check` runs; a repository missing there fails the test, never skips it. Which clones a
|
||||
// check gets is chosen from the inventory: mesh-catalog by the source of the `nats` module, mesh-host
|
||||
// by the source of `mesh-host`, and mesh-sdk as the controller's sibling at the commit the controller's
|
||||
// go.mod pins. In a mesh where either module has no source repository nothing is
|
||||
// cloned, and these tests fail loudly with "not beside this check": a cause in the setup, not in the
|
||||
// change. And in a delivery group that holds a mesh-catalog pull request, these tests read the
|
||||
// catalogue's main, not the group's head.
|
||||
// - Anywhere else, the test judges against the copy captured in this repository's testdata/beside, at the
|
||||
// commit testdata/beside/CAPTURED names. Never against a developer's own checkout: to judge one, set
|
||||
// MESH_CHECK_BESIDE to the directory holding it, as the check does.
|
||||
//
|
||||
// The captured manifests are named module.json.captured, and put back as module.json in a directory of
|
||||
// the test's own: a module.json anywhere in this repository is a module of it to the forge's announcer and
|
||||
// the planner, and a merge would build and register a hundred copies of the catalogue's.
|
||||
package beside
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Env is where a merge check says the repositories cloned beside the one checked are (internal/builder's
|
||||
// EnvBeside, repeated here so a test does not import the builder).
|
||||
const Env = "MESH_CHECK_BESIDE"
|
||||
|
||||
// CapturedSuffix is what a captured file's name carries that the repository's own does not.
|
||||
const CapturedSuffix = ".captured"
|
||||
|
||||
// Dir is the named repository a test reads — the clone beside a merge check, or the captured copy — and
|
||||
// says which in the test's log.
|
||||
func Dir(t testing.TB, repository string) string {
|
||||
t.Helper()
|
||||
if root := os.Getenv(Env); root != "" {
|
||||
dir := filepath.Join(root, repository)
|
||||
if _, err := os.Stat(dir); err != nil {
|
||||
t.Fatalf("%s is not beside this check in %s=%s, so its agreement with this repository cannot be "+
|
||||
"judged here: %v", repository, Env, root, err)
|
||||
}
|
||||
t.Logf("judged against %s as cloned beside this check", dir)
|
||||
return dir
|
||||
}
|
||||
from := filepath.Join(Captured(), repository)
|
||||
if _, err := os.Stat(from); err != nil {
|
||||
t.Fatalf("no captured copy of %s at %s: %v", repository, from, err)
|
||||
}
|
||||
dir := filepath.Join(t.TempDir(), repository)
|
||||
err := filepath.WalkDir(from, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel, _ := filepath.Rel(from, path)
|
||||
into := filepath.Join(dir, strings.TrimSuffix(rel, CapturedSuffix))
|
||||
if d.IsDir() {
|
||||
return os.MkdirAll(into, 0o755)
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(into, raw, 0o644)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("the captured copy of %s could not be laid out: %v", repository, err)
|
||||
}
|
||||
t.Logf("judged against the copy of %s captured in testdata/beside (see CAPTURED); a merge check "+
|
||||
"judges it against the repository's own clone", repository)
|
||||
return dir
|
||||
}
|
||||
|
||||
// Catalogue is the catalogue's modules directory, as Dir finds the catalogue.
|
||||
func Catalogue(t testing.TB) string {
|
||||
t.Helper()
|
||||
return filepath.Join(Dir(t, "mesh-catalog"), "modules")
|
||||
}
|
||||
|
||||
// Captured is this repository's testdata/beside, found from this file rather than from the working
|
||||
// directory, so a test in any package reaches it.
|
||||
func Captured() string {
|
||||
_, file, _, _ := runtime.Caller(0)
|
||||
return filepath.Join(filepath.Dir(file), "..", "..", "testdata", "beside")
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/beside"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
@@ -74,10 +75,10 @@ func TestTheAgreementCheckCatchesASubscriptionThatMatchesNothing(t *testing.T) {
|
||||
|
||||
func theCataloguesEvents(t *testing.T) ([]AnEmitter, []AConsumer, []DeclaredSeat) {
|
||||
t.Helper()
|
||||
root := filepath.Join("..", "..", "..", "mesh-catalog", "modules")
|
||||
root := beside.Catalogue(t)
|
||||
entries, err := os.ReadDir(root)
|
||||
if err != nil {
|
||||
t.Skipf("catalogue sibling not present: %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
var emitters []AnEmitter
|
||||
var consumers []AConsumer
|
||||
@@ -119,7 +120,7 @@ func theCataloguesEvents(t *testing.T) ([]AnEmitter, []AConsumer, []DeclaredSeat
|
||||
}
|
||||
}
|
||||
if len(emitters) == 0 {
|
||||
t.Skip("no manifests found beside this checkout")
|
||||
t.Fatalf("no module under %s emits anything, so this proved nothing", root)
|
||||
}
|
||||
return emitters, consumers, seats
|
||||
}
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// A tool call names its caller (novox/hq issue 365, by ADR 0259 §3's precedent for asks).
|
||||
//
|
||||
// mesh.mod.<module>.call.<tool>[.<node>].<caller>
|
||||
// mesh.seat.<seat>.call.<verb>[.<node>].<caller>
|
||||
//
|
||||
// The last token is the bus user that published the call, and each user is granted these subjects with its own
|
||||
// name there and no other — the caller is a fact the server enforces, and the tool runtime hands it to the
|
||||
// module from the subject a call arrived on (mesh-tools node-tools internal/bus caller.go holds the same shape).
|
||||
//
|
||||
// **A kind of its own, `call`, not the `tool` subject with a token appended.** Every grant to call a tool is a
|
||||
// wildcard over the `tool` kind — `.tool.<t>.*` for the instance on any machine, `mesh.mod.*.tool.>` for every
|
||||
// tool — and either would match a `tool` subject with any caller appended, so a caller could name another.
|
||||
// Under `call` nothing is granted but the caller-named subjects. No stream's filter covers it: a tool call is
|
||||
// never persisted (design 25 §3).
|
||||
//
|
||||
// **Derived from the `tool` grants, in one place** (callerNamed): wherever a principal may call or answer a
|
||||
// tool, it may call it in its own name, or answer it naming any caller — so no kind of principal is left
|
||||
// unable to call in its own name, and a new grant to call is one in both shapes by construction.
|
||||
//
|
||||
// The `tool` grants stay beside these for one release, so every caller and every runtime moves without a gap:
|
||||
// their retirement is hq issue 464.
|
||||
const CallKind = "call"
|
||||
|
||||
var userName = regexp.MustCompile(`^[A-Za-z0-9_-]+(\.[A-Za-z0-9_-]+)*$`)
|
||||
|
||||
// CallerToken is a bus user's name as the last token of a call it publishes: each dot written `~`, which no part
|
||||
// of a user's name may hold (safeSubject), so the token names that user and no other. "" for a name that is not
|
||||
// a user's.
|
||||
func CallerToken(user string) string {
|
||||
if !userName.MatchString(user) {
|
||||
return ""
|
||||
}
|
||||
return strings.ReplaceAll(user, ".", "~")
|
||||
}
|
||||
|
||||
// toolGrant splits a grant on the `tool` kind — `mesh.mod.<m>.tool.<rest>` or `mesh.seat.<s>.tool.<rest>`, any
|
||||
// part possibly a wildcard — at the kind; ok is false for any other subject.
|
||||
func toolGrant(subject string) (head, rest string, ok bool) {
|
||||
parts := strings.SplitN(subject, ".", 5)
|
||||
if len(parts) != 5 || parts[0] != "mesh" || (parts[1] != "mod" && parts[1] != "seat") || parts[3] != "tool" ||
|
||||
parts[2] == "" || parts[4] == "" {
|
||||
return "", "", false
|
||||
}
|
||||
return parts[0] + "." + parts[1] + "." + parts[2], parts[4], true
|
||||
}
|
||||
|
||||
// CalledSubject is where a call to a tool subject goes naming its caller; "" when the subject is not a tool's
|
||||
// or the user is not a bus user.
|
||||
func CalledSubject(subject, user string) string {
|
||||
head, rest, ok := toolGrant(subject)
|
||||
token := CallerToken(user)
|
||||
if !ok || token == "" || strings.ContainsAny(rest, "*>") {
|
||||
return ""
|
||||
}
|
||||
return head + "." + CallKind + "." + rest + "." + token
|
||||
}
|
||||
|
||||
// CalledPattern is what a holder answering a tool subject also subscribes, to hear the calls that name their
|
||||
// caller: the same address under the `call` kind, any caller last. "" for a subject that is not a tool's.
|
||||
func CalledPattern(subject string) string {
|
||||
head, rest, ok := toolGrant(subject)
|
||||
if !ok || strings.ContainsAny(rest, "*>") {
|
||||
return ""
|
||||
}
|
||||
return head + "." + CallKind + "." + rest + ".*"
|
||||
}
|
||||
|
||||
// calledPublish is a grant to call on the `tool` kind, as the same grant in the caller's own name: its last
|
||||
// token the caller's, and nothing that reaches past it. A `>` is every tail a call carries — the tool alone or
|
||||
// the tool and the machine — so it becomes both, each ending in the caller.
|
||||
func calledPublish(grant, token string) []string {
|
||||
head, rest, ok := toolGrant(grant)
|
||||
if !ok || token == "" {
|
||||
return nil
|
||||
}
|
||||
base := head + "." + CallKind + "."
|
||||
switch {
|
||||
case rest == ">":
|
||||
return []string{base + "*." + token, base + "*.*." + token}
|
||||
case strings.HasSuffix(rest, ".>"):
|
||||
return []string{base + strings.TrimSuffix(rest, ">") + "*." + token}
|
||||
}
|
||||
return []string{base + rest + "." + token}
|
||||
}
|
||||
|
||||
// calledSubscribe is a grant to answer on the `tool` kind, as the same grant for the calls naming any caller.
|
||||
func calledSubscribe(grant string) []string {
|
||||
head, rest, ok := toolGrant(grant)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
base := head + "." + CallKind + "."
|
||||
if strings.HasSuffix(rest, ">") {
|
||||
return []string{base + rest}
|
||||
}
|
||||
return []string{base + rest + ".*"}
|
||||
}
|
||||
|
||||
// callerNamed adds, beside a principal's grants on the `tool` kind, the same grants under `call`: to publish in
|
||||
// its own name, to subscribe naming anybody.
|
||||
func callerNamed(user string, pub, sub []string) ([]string, []string) {
|
||||
token := CallerToken(user)
|
||||
for _, g := range pub {
|
||||
pub = append(pub, calledPublish(g, token)...)
|
||||
}
|
||||
for _, g := range sub {
|
||||
sub = append(sub, calledSubscribe(g)...)
|
||||
}
|
||||
return unique(pub), unique(sub)
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
)
|
||||
|
||||
// **On a real server, as composed** (novox/hq issue 365): a machine's runtime calls in its own name, and the
|
||||
// server refuses it a call naming another — the grant, not the runtime, is what makes the caller a fact.
|
||||
func TestAServerComposedFromTheGrantsRefusesACallNamingAnother(t *testing.T) {
|
||||
hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ledger := Seat{Name: "issue-tracker", Scope: "mesh", Serves: []string{"open"}}
|
||||
accounts, err := ComposeAccounts([]Principal{
|
||||
{Kind: KindNodeTools, Node: "novox", Module: RuntimeModule, PasswordHash: string(hash),
|
||||
Carries: []Declared{{Module: "mesh-issues", Holds: []Seat{ledger}}}},
|
||||
{Kind: KindNodeTools, Node: "shanks", Module: RuntimeModule, PasswordHash: string(hash)},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conf := filepath.Join(t.TempDir(), "bus.conf")
|
||||
if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+
|
||||
`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts, err := server.ProcessConfigFile(conf)
|
||||
if err != nil {
|
||||
t.Fatalf("the composed accounts do not parse: %v", err)
|
||||
}
|
||||
opts.NoLog, opts.NoSigs = true, true
|
||||
s, err := server.NewServer(opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go s.Start()
|
||||
if !s.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
defer s.Shutdown()
|
||||
|
||||
holder, err := nats.Connect(s.ClientURL(), nats.UserInfo("novox.node-tools", "pw"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer holder.Close()
|
||||
heard := make(chan string, 4)
|
||||
sub, err := holder.Subscribe("mesh.seat.issue-tracker.call.open.*", func(m *nats.Msg) {
|
||||
heard <- m.Subject
|
||||
_ = m.Respond([]byte(`{"result":{}}`))
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = holder.Flush()
|
||||
if !sub.IsValid() {
|
||||
t.Fatal("the holder may not hear the caller-named calls to its seat")
|
||||
}
|
||||
|
||||
refusals := make(chan error, 4)
|
||||
caller, err := nats.Connect(s.ClientURL(), nats.UserInfo("shanks.node-tools", "pw"),
|
||||
nats.CustomInboxPrefix("_INBOX.shanks.node-tools"),
|
||||
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refusals <- err }))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer caller.Close()
|
||||
if _, err := caller.Request("mesh.seat.issue-tracker.call.open.shanks~node-tools", []byte(`{}`), 3*time.Second); err != nil {
|
||||
t.Fatalf("a call in the caller's own name was not answered: %v", err)
|
||||
}
|
||||
if got := <-heard; got != "mesh.seat.issue-tracker.call.open.shanks~node-tools" {
|
||||
t.Fatalf("heard %s", got)
|
||||
}
|
||||
if err := caller.Publish("mesh.seat.issue-tracker.call.open.novox~node-tools", []byte(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = caller.Flush()
|
||||
select {
|
||||
case err := <-refusals:
|
||||
if !errors.Is(err, nats.ErrPermissionViolation) {
|
||||
t.Errorf("the server said %v, want a permissions violation", err)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Error("the server did not refuse a call naming another caller")
|
||||
}
|
||||
select {
|
||||
case got := <-heard:
|
||||
t.Errorf("a call naming another reached the holder: %s", got)
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// **A tool call names its caller, and the bus lets each user name itself alone** (novox/hq issue 365, by ADR
|
||||
// 0259 §3's precedent for asks): wherever a principal may call a tool, it may call it on the caller-named
|
||||
// subject — kind `call`, its own bus user last, dots written `~` — and on no subject naming anybody else.
|
||||
func TestEachCredentialMayCallOnlyInItsOwnName(t *testing.T) {
|
||||
ledger := Seat{Name: "node-desk", Scope: "node", Serves: []string{"who"}}
|
||||
tracker := Seat{Name: "issue-tracker", Scope: "mesh", Serves: []string{"open"}}
|
||||
for _, c := range []struct {
|
||||
p Principal
|
||||
may []string
|
||||
mayNot []string
|
||||
subject []string // what it subscribes, to answer calls naming any caller
|
||||
}{
|
||||
{p: Principal{Kind: KindPerson, Module: "jochen", Invokes: []string{"*"}},
|
||||
may: []string{"mesh.mod.ledger.call.who.person~jochen", "mesh.mod.ledger.call.who.anchor.person~jochen",
|
||||
"mesh.seat.issue-tracker.call.open.person~jochen", "mesh.seat.node-desk.call.who.anchor.person~jochen"},
|
||||
mayNot: []string{"mesh.mod.ledger.call.who.shanks~node-tools", "mesh.mod.ledger.call.who.anchor.controller",
|
||||
"mesh.seat.issue-tracker.call.open.person~somebody", "mesh.mod.ledger.call.who.person"}},
|
||||
{p: Principal{Kind: KindNodeTools, Node: "shanks", Module: RuntimeModule,
|
||||
Carries: []Declared{{Module: "ledger", Holds: []Seat{ledger, tracker}}}},
|
||||
may: []string{"mesh.mod.ledger.call.who.shanks~node-tools", "mesh.seat.issue-tracker.call.open.shanks~node-tools"},
|
||||
mayNot: []string{"mesh.mod.ledger.call.who.novox~node-tools", "mesh.seat.issue-tracker.call.open.controller"},
|
||||
subject: []string{"mesh.mod.ledger.call.who.novox~node-tools", "mesh.seat.node-desk.call.who.shanks.person~jochen", "mesh.seat.issue-tracker.call.open.controller"}},
|
||||
{p: Principal{Kind: KindModule, Node: "two", Module: "shop", Invokes: []string{"ledger.who", "seat:issue-tracker.open"}},
|
||||
may: []string{"mesh.mod.ledger.call.who.two~shop", "mesh.mod.ledger.call.who.anchor.two~shop",
|
||||
"mesh.seat.issue-tracker.call.open.two~shop"},
|
||||
mayNot: []string{"mesh.mod.ledger.call.other.two~shop", "mesh.mod.ledger.call.who.one~shop",
|
||||
"mesh.seat.issue-tracker.call.open.one~telegram"}},
|
||||
{p: Principal{Kind: KindNode, Node: "one", Checks: []string{"ledger.health"}},
|
||||
may: []string{"mesh.mod.ledger.call.health.one.node~one"},
|
||||
mayNot: []string{"mesh.mod.ledger.call.health.two.node~one", "mesh.mod.ledger.call.health.one.node~two"}},
|
||||
{p: Principal{Kind: KindModule, Node: "one", Module: "ledger", Holds: []Seat{ledger, tracker}},
|
||||
subject: []string{"mesh.mod.ledger.call.who.person~jochen", "mesh.mod.ledger.call.who.one.controller",
|
||||
"mesh.seat.node-desk.call.who.one.two~shop", "mesh.seat.issue-tracker.call.open.two~shop"}},
|
||||
} {
|
||||
perms, err := PermissionsFor(c.p)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", c.p.Username(), err)
|
||||
}
|
||||
for _, s := range c.may {
|
||||
if !MayPublish(perms, s) {
|
||||
t.Errorf("%s may not call %s, in its own name", c.p.Username(), s)
|
||||
}
|
||||
}
|
||||
for _, s := range c.mayNot {
|
||||
if MayPublish(perms, s) {
|
||||
t.Errorf("%s may call %s, naming somebody else", c.p.Username(), s)
|
||||
}
|
||||
}
|
||||
for _, s := range c.subject {
|
||||
if !MaySubscribe(perms, s) {
|
||||
t.Errorf("%s does not hear %s, a call to what it serves", c.p.Username(), s)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The subjects that name no caller stay granted beside the caller-named ones for one release, so a caller
|
||||
// moves over without a gap (their retirement: hq issue 464).
|
||||
func TestTheSubjectsThatNameNoCallerStayGrantedForOneRelease(t *testing.T) {
|
||||
perms, err := PermissionsFor(Principal{Kind: KindModule, Node: "two", Module: "shop", Invokes: []string{"ledger.who"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, s := range []string{"mesh.mod.ledger.tool.who", "mesh.mod.ledger.tool.who.anchor"} {
|
||||
if !MayPublish(perms, s) {
|
||||
t.Errorf("the old subject %s is no longer granted", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The desk's hidden prompt is the controller's alone on the caller-named subjects too (the review of
|
||||
// 2026-10-09, M4): a grant of every tool does not reach it there either.
|
||||
func TestTheDesksPromptIsTheControllersAloneUnderCallToo(t *testing.T) {
|
||||
for _, p := range []Principal{{Kind: KindPerson, Module: "jochen", Invokes: []string{"*"}},
|
||||
{Kind: KindNodeTools, Node: "shanks", Module: RuntimeModule}} {
|
||||
perms, err := PermissionsFor(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
token := CallerToken(p.Username())
|
||||
for _, s := range []string{"mesh.seat.node-launcher.call.secret.shanks." + token,
|
||||
"mesh.mod.shell.call.node-launcher.secret.shanks." + token, "mesh.mod.shell.call.node-launcher.secret." + token} {
|
||||
if MayPublish(perms, s) {
|
||||
t.Errorf("%s may publish %s, the desk's hidden prompt", p.Username(), s)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The controller's grants are the installer's first user list too, so they move with hq issue 464: this release
|
||||
// it calls and serves on the subjects that name no caller alone, and nobody may call in its name.
|
||||
func TestTheControllerKeepsTheSubjectsThatNameNoCallerThisRelease(t *testing.T) {
|
||||
perms, err := PermissionsFor(Principal{Kind: KindController})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, s := range append(perms.Publish, perms.Subscribe...) {
|
||||
if strings.Contains(s, "."+CallKind+".") {
|
||||
t.Errorf("the controller is granted %s, which the installer's first user list does not carry", s)
|
||||
}
|
||||
}
|
||||
for _, p := range []Principal{{Kind: KindPerson, Module: "jochen", Invokes: []string{"*"}},
|
||||
{Kind: KindNodeTools, Node: "shanks", Module: RuntimeModule}} {
|
||||
perms, err := PermissionsFor(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if MayPublish(perms, "mesh.mod.ledger.call.who.controller") || MayPublish(perms, "mesh.seat.mesh-controller.call.status.controller") {
|
||||
t.Errorf("%s may call in the controller's name", p.Username())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,8 @@ import (
|
||||
"testing"
|
||||
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/beside"
|
||||
)
|
||||
|
||||
// **The first user list the installer carries must be the one the controller would compose.**
|
||||
@@ -76,13 +78,14 @@ func theCarriedAccounts(t *testing.T) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// theTemplate is the installer's bundle, as resources.
|
||||
// theTemplate is the installer's bundle, as resources: the node-engine's, as a merge check clones it at
|
||||
// the commit the mesh runs, or as captured in testdata/beside (internal/beside, novox/hq issue 432).
|
||||
func theTemplate(t *testing.T) []map[string]any {
|
||||
t.Helper()
|
||||
path := filepath.Join("..", "..", "..", "mesh-host", "examples", "foundation-first-node-nats.lock")
|
||||
path := filepath.Join(beside.Dir(t, "mesh-host"), "examples", "foundation-first-node-nats.lock")
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Skipf("the host's checkout is not beside this one: %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The template is JSON with line comments, which is how every one of them is written.
|
||||
var lines []string
|
||||
|
||||
@@ -42,8 +42,9 @@ func TestInvokingGrantsNothingButTheCall(t *testing.T) {
|
||||
if strings.Contains(p, ".event.") {
|
||||
t.Errorf("a module that only invokes may publish %q, an event it never declared", p)
|
||||
}
|
||||
// A role's tools are tools (ADR 0132); a role's work queue and events are not.
|
||||
if strings.HasPrefix(p, "mesh.seat.") && !strings.Contains(p, ".tool.") {
|
||||
// A role's tools are tools (ADR 0132), named by their caller or not (novox/hq issue 365); a role's work
|
||||
// queue and events are not.
|
||||
if strings.HasPrefix(p, "mesh.seat.") && !strings.Contains(p, ".tool.") && !strings.Contains(p, ".call.") {
|
||||
t.Errorf("a module that only invokes may publish %q, a seat it neither holds nor uses", p)
|
||||
}
|
||||
}
|
||||
|
||||
+36
-1
@@ -163,6 +163,21 @@ type Principal struct {
|
||||
// goes with the retired seat row.
|
||||
var seatsTheControllerAsks = []string{"node-build-agent", "mesh-build-machine"}
|
||||
|
||||
// TheControllersAsk says whether a message of stream, published on subject, is an ask the controller
|
||||
// itself makes: one on the accept subject of a seat in seatsTheControllerAsks, kept in that seat's own
|
||||
// work queue. Such an ask, given up on by the seat's worker, can be delivered again with the authority
|
||||
// the controller already holds — the publish on that seat's accepts and the stream API to remove the
|
||||
// original — and no other can (novox/hq issue 334, ADR 0264's consequences: no grant over the seats'
|
||||
// queues).
|
||||
func TheControllersAsk(stream, subject string) bool {
|
||||
for _, seat := range seatsTheControllerAsks {
|
||||
if stream == seatStreamName(seat) && strings.HasPrefix(subject, "mesh.seat."+seat+".accept.") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// SeatVerb is one verb of one seat, on every machine holding it.
|
||||
type SeatVerb struct{ Seat, Verb string }
|
||||
|
||||
@@ -197,6 +212,10 @@ func ControllerOnly() []string {
|
||||
for _, base := range []string{"mesh.seat." + v.Seat + ".tool." + v.Verb, "mesh.mod.*.tool." + v.Seat + "." + v.Verb} {
|
||||
out = append(out, base, base+".*")
|
||||
}
|
||||
// And where a call names its caller (novox/hq issue 365): any machine, any caller.
|
||||
for _, base := range []string{"mesh.seat." + v.Seat + "." + CallKind + "." + v.Verb, "mesh.mod.*." + CallKind + "." + v.Seat + "." + v.Verb} {
|
||||
out = append(out, base+".>")
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -232,8 +251,11 @@ func MaySubscribe(perms Permissions, subject string) bool {
|
||||
//
|
||||
// And, since novox/hq ADR 0259, `release` and `stop`: the controller asks the operator for them about a
|
||||
// delivery held past its bound, and calls them on the operator's warrant, with its why.
|
||||
//
|
||||
// And, since novox/hq ADR 0282, `times`: the self-check reads the delivery times to say a delivery over its budget.
|
||||
var VerbsTheControllerAsksTheDeliveryOwner = []SeatVerb{{Seat: "mesh-delivery", Verb: "stalled"},
|
||||
{Seat: "mesh-delivery", Verb: "close"}, {Seat: "mesh-delivery", Verb: "release"}, {Seat: "mesh-delivery", Verb: "stop"}}
|
||||
{Seat: "mesh-delivery", Verb: "close"}, {Seat: "mesh-delivery", Verb: "release"}, {Seat: "mesh-delivery", Verb: "stop"},
|
||||
{Seat: "mesh-delivery", Verb: "times"}}
|
||||
|
||||
// VerbsTheControllerActsOnAWarrant are the other seat verbs the controller calls when the operator's warrant
|
||||
// chooses them (novox/hq ADR 0259): a machine's service restarted, and a walk started or stopped through the
|
||||
@@ -857,6 +879,19 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
pub = append(pub, "$JS.ACK."+consumerStream(p)+"."+consumerDurable(p)+".>")
|
||||
}
|
||||
|
||||
// **And every tool grant again, naming its caller** (novox/hq issue 365): a call in this principal's own
|
||||
// name, and an answer to a call naming anybody. The `tool` grants above stay for one release beside these,
|
||||
// so callers and runtimes move without a gap; their retirement is hq issue 464.
|
||||
//
|
||||
// **Not the controller's, this release.** Its grants are also the installer's first user list
|
||||
// (TestTheInstallersFirstUserListIsWhatTheControllerWouldCompose), judged against the node-engine the mesh
|
||||
// runs, so a change to them waits on a node-engine delivery. It calls and serves on the subjects that name
|
||||
// no caller meanwhile — a caller-named call to its seat reaches nobody and is asked again on those at once —
|
||||
// and moves with issue 464.
|
||||
if p.Kind != KindController {
|
||||
pub, sub = callerNamed(p.Username(), pub, sub)
|
||||
}
|
||||
|
||||
sort.Strings(pub)
|
||||
sort.Strings(sub)
|
||||
// One writer per piece of state (novox/hq to-be 45 §1): a grant that would make a second is
|
||||
|
||||
@@ -240,9 +240,9 @@ func TestAPersonReachesNothingButTools(t *testing.T) {
|
||||
perms, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo",
|
||||
Invokes: []string{"*"}, PasswordHash: "x"})
|
||||
for _, p := range perms.Publish {
|
||||
// A tool call, or asking what answers (novox/hq ADR 0197) — a question every service
|
||||
// answers about itself, which claims nothing and controls nothing.
|
||||
if !strings.Contains(p, ".tool.") && !strings.HasPrefix(p, "$SRV.") {
|
||||
// A tool call — named by its caller or not (novox/hq issue 365) — or asking what answers (novox/hq
|
||||
// ADR 0197), a question every service answers about itself, which claims nothing and controls nothing.
|
||||
if !strings.Contains(p, ".tool.") && !strings.Contains(p, ".call.") && !strings.HasPrefix(p, "$SRV.") {
|
||||
t.Errorf("a person may publish %q, which is not a tool call", p)
|
||||
}
|
||||
}
|
||||
|
||||
+4
-4
@@ -24,7 +24,7 @@ accounts {
|
||||
jetstream: enabled
|
||||
users = [
|
||||
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*"] }
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*"] }
|
||||
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
@@ -43,17 +43,17 @@ accounts {
|
||||
} }
|
||||
{ user: "one.telegram", password: "$2a$11$tttttttttttttttttttttt", permissions: {
|
||||
publish: { allow: ["$JS.ACK.EVENTS.one_telegram.>", "$JS.ACK.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker.>", "$JS.API.CONSUMER.INFO.EVENTS.one_telegram", "$JS.API.CONSUMER.INFO.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_telegram", "$JS.API.CONSUMER.MSG.NEXT.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.one.telegram", "mesh.seat.telegram-sender.event.delivered", "mesh.seat.telegram-sender.event.failed"] }
|
||||
subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.telegram", "$SRV.INFO.telegram.>", "$SRV.PING", "$SRV.PING.telegram", "$SRV.PING.telegram.>", "$SRV.STATS", "$SRV.STATS.telegram", "$SRV.STATS.telegram.>", "_INBOX.one.telegram.>", "mesh.assignment.one.telegram", "mesh.mod.telegram.tool.>", "mesh.seat.telegram-sender.accept.send"] }
|
||||
subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.telegram", "$SRV.INFO.telegram.>", "$SRV.PING", "$SRV.PING.telegram", "$SRV.PING.telegram.>", "$SRV.STATS", "$SRV.STATS.telegram", "$SRV.STATS.telegram.>", "_INBOX.one.telegram.>", "mesh.assignment.one.telegram", "mesh.mod.telegram.call.>", "mesh.mod.telegram.tool.>", "mesh.seat.telegram-sender.accept.send"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
{ user: "two.audit", password: "$2a$11$aaaaaaaaaaaaaaaaaaaaaa", permissions: {
|
||||
publish: { allow: ["$JS.ACK.EVENTS.two_audit.>", "$JS.API.CONSUMER.INFO.EVENTS.two_audit", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_audit", "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.two.audit"] }
|
||||
subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.audit", "$SRV.INFO.audit.>", "$SRV.PING", "$SRV.PING.audit", "$SRV.PING.audit.>", "$SRV.STATS", "$SRV.STATS.audit", "$SRV.STATS.audit.>", "_INBOX.two.audit.>", "mesh.assignment.two.audit", "mesh.mod.audit.tool.>", "mesh.mod.shop.event.order.placed"] }
|
||||
subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.audit", "$SRV.INFO.audit.>", "$SRV.PING", "$SRV.PING.audit", "$SRV.PING.audit.>", "$SRV.STATS", "$SRV.STATS.audit", "$SRV.STATS.audit.>", "_INBOX.two.audit.>", "mesh.assignment.two.audit", "mesh.mod.audit.call.>", "mesh.mod.audit.tool.>", "mesh.mod.shop.event.order.placed"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
{ user: "two.shop", password: "$2a$11$ssssssssssssssssssssss", permissions: {
|
||||
publish: { allow: ["$JS.ACK.EVENTS.two_shop.>", "$JS.API.CONSUMER.INFO.EVENTS.two_shop", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_shop", "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.two.shop", "mesh.mod.shop.event.order.placed", "mesh.seat.telegram-sender.accept.send"] }
|
||||
subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.shop", "$SRV.INFO.shop.>", "$SRV.PING", "$SRV.PING.shop", "$SRV.PING.shop.>", "$SRV.STATS", "$SRV.STATS.shop", "$SRV.STATS.shop.>", "_INBOX.two.shop.>", "mesh.assignment.two.shop", "mesh.mod.shop.tool.>"] }
|
||||
subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.shop", "$SRV.INFO.shop.>", "$SRV.PING", "$SRV.PING.shop", "$SRV.PING.shop.>", "$SRV.STATS", "$SRV.STATS.shop", "$SRV.STATS.shop.>", "_INBOX.two.shop.>", "mesh.assignment.two.shop", "mesh.mod.shop.call.>", "mesh.mod.shop.tool.>"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
]
|
||||
|
||||
+107
-6
@@ -7,8 +7,10 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
@@ -159,17 +161,18 @@ func build(ctx context.Context, run Runner, publish Publisher,
|
||||
}
|
||||
// The credential is a file git reads, never an argument: a URL carrying a password in argv
|
||||
// would be readable by anything that can list processes for as long as a clone runs.
|
||||
credentials := ""
|
||||
if forge.URL != "" {
|
||||
credentials = filepath.Join(workspace, "git-credentials")
|
||||
if err := os.WriteFile(credentials, []byte(forge.URL+"\n"), 0o600); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
credentials, forget, err := storeCredential(workspace, forge)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
defer forget()
|
||||
tree := filepath.Join(workspace, "source")
|
||||
if err := os.RemoveAll(tree); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
// Removed when the build ends, whatever happens: the tree holds the .npmrc a build may be handed, in the
|
||||
// workspace a later check's container mounts as its HOME (novox/hq issue 462).
|
||||
defer os.RemoveAll(tree)
|
||||
// A fresh clone every time rather than a fetch into a tree that is already there. A build
|
||||
// that reuses a working tree can succeed because of something a previous build left behind,
|
||||
// and that is a build nobody can reproduce.
|
||||
@@ -177,6 +180,9 @@ func build(ctx context.Context, run Runner, publish Publisher,
|
||||
say("clone", "FAILED: %v", err)
|
||||
return Result{}, fmt.Errorf("cannot clone %s: %w", repository, err)
|
||||
}
|
||||
if err := recordedWithoutUserinfo(ctx, run, tree, repository); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
say("clone", "done")
|
||||
if ref != "" {
|
||||
if _, err := run(ctx, tree, "git", "checkout", "--quiet", ref); err != nil {
|
||||
@@ -248,6 +254,8 @@ func build(ctx context.Context, run Runner, publish Publisher,
|
||||
if err := os.WriteFile(npmrcPath, []byte(content), 0o600); err != nil {
|
||||
return Result{}, fmt.Errorf("cannot write the package-registry credential for the build: %w", err)
|
||||
}
|
||||
// Only for as long as the build: a later check's container mounts this workspace (novox/hq issue 462).
|
||||
defer os.Remove(npmrcPath)
|
||||
say("packages", "resolving %s from the mesh's package registry", npmrc.Scope)
|
||||
src.notPinned("it resolves packages from the mesh's registry at build time")
|
||||
}
|
||||
@@ -441,6 +449,9 @@ func contextFrom(ctx context.Context, run Runner, workspace, artifact, credentia
|
||||
if _, err := run(ctx, workspace, "git", cloneWith(credentials, "clone", "--quiet", url, dir)...); err != nil {
|
||||
return "", fmt.Errorf("cannot clone %s: %w", url, err)
|
||||
}
|
||||
if err := recordedWithoutUserinfo(ctx, run, dir, url); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if from.Ref != "" {
|
||||
if _, err := run(ctx, dir, "git", "checkout", "--quiet", from.Ref); err != nil {
|
||||
return "", fmt.Errorf("%s has no %s: %w", from.Repository, from.Ref, err)
|
||||
@@ -467,6 +478,96 @@ func contextURL(from catalogue.ArtifactContext, seats map[string]string) (string
|
||||
return strings.TrimRight(base, "/") + "/" + strings.TrimSuffix(strings.Trim(from.Repository, "/"), ".git") + ".git", nil
|
||||
}
|
||||
|
||||
// storeCredential writes the forge credential where git's credential store reads it, and the function that
|
||||
// removes it again; "" and nothing to remove when the builder holds none.
|
||||
//
|
||||
// **Never inside the workspace** (novox/hq issue 462): a merge check's toolchain container mounts the
|
||||
// workspace as its HOME, so a credential kept there — even one a build left behind — is readable by any pull
|
||||
// request's merge-check.sh, and what it prints is kept on the bus. So it lives in a directory of its own
|
||||
// outside the workspace, made private, and a credential an older builder left in the workspace is removed.
|
||||
func storeCredential(workspace string, forge GitCredential) (string, func(), error) {
|
||||
if err := os.Remove(filepath.Join(workspace, "git-credentials")); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return "", nil, fmt.Errorf("a credential left in the workspace cannot be removed: %w", err)
|
||||
}
|
||||
if forge.URL == "" {
|
||||
return "", func() {}, nil
|
||||
}
|
||||
dir, err := os.MkdirTemp("", "mesh-forge-credential-")
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
forget := func() { os.RemoveAll(dir) }
|
||||
if withinDir(workspace, dir) {
|
||||
forget()
|
||||
return "", nil, fmt.Errorf("the temporary directory %s is inside the workspace %s, which a check's "+
|
||||
"container mounts: the forge credential is not written where it could read it", dir, workspace)
|
||||
}
|
||||
path := filepath.Join(dir, "git-credentials")
|
||||
if err := os.WriteFile(path, []byte(forge.URL+"\n"), 0o600); err != nil {
|
||||
forget()
|
||||
return "", nil, err
|
||||
}
|
||||
return path, forget, nil
|
||||
}
|
||||
|
||||
// recordedWithoutUserinfo makes a clone record the URL it came from without userinfo: git keeps it as given in
|
||||
// the clone's .git/config, inside the workspace a check's container mounts (novox/hq issue 462).
|
||||
func recordedWithoutUserinfo(ctx context.Context, run Runner, clone, repository string) error {
|
||||
bare, carried := withoutUserinfo(repository)
|
||||
if !carried {
|
||||
return nil
|
||||
}
|
||||
if _, err := run(ctx, clone, "git", "remote", "set-url", "origin", bare); err != nil {
|
||||
return fmt.Errorf("the clone of %s keeps its credential: %w", bare, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// forgetLeftCredentials removes what an earlier build or an older builder left in the workspace that holds
|
||||
// a credential: the forge's git-credentials, and every .npmrc a build wrote into a tree it cloned. Run
|
||||
// before a check, whose container mounts the workspace as its HOME (novox/hq issue 462). The Go caches are
|
||||
// not walked: no build writes a credential there, and they hold more files than everything else.
|
||||
func forgetLeftCredentials(workspace string) error {
|
||||
if err := os.Remove(filepath.Join(workspace, "git-credentials")); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
return filepath.WalkDir(workspace, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if d.IsDir() && p != workspace && (d.Name() == "go-cache" || d.Name() == "go-modules") &&
|
||||
filepath.Dir(p) == workspace {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
if !d.IsDir() && d.Name() == ".npmrc" {
|
||||
if err := os.Remove(p); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("a package-registry credential left at %s cannot be removed: %w", p, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// withinDir is whether path is dir or under it, both made absolute and resolved.
|
||||
func withinDir(dir, path string) bool {
|
||||
d, err1 := filepath.Abs(dir)
|
||||
p, err2 := filepath.Abs(path)
|
||||
if err1 != nil || err2 != nil {
|
||||
return true
|
||||
}
|
||||
if r, err := filepath.EvalSymlinks(d); err == nil {
|
||||
d = r
|
||||
}
|
||||
if r, err := filepath.EvalSymlinks(p); err == nil {
|
||||
p = r
|
||||
}
|
||||
rel, err := filepath.Rel(d, p)
|
||||
return err != nil || rel == "." || filepath.IsLocal(rel)
|
||||
}
|
||||
|
||||
// cloneWith is a git invocation that may offer a stored credential.
|
||||
//
|
||||
// The first `-c credential.helper=` clears every helper the environment might carry, so exactly
|
||||
|
||||
@@ -438,30 +438,34 @@ func TestABuildOffersTheForgesCredentialThroughGitsOwnStore(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stored := filepath.Join(workspace, "git-credentials")
|
||||
clone := r.ran[0]
|
||||
if !strings.Contains(clone, "credential.helper=store --file="+stored) {
|
||||
t.Fatalf("the clone does not name the credential store: %s", clone)
|
||||
}
|
||||
stored := storeNamedIn(t, clone)
|
||||
for _, line := range r.ran {
|
||||
if strings.Contains(line, "sw0rdfi5h") {
|
||||
t.Fatalf("the secret is in a command line, readable by anything that can list processes: %s", line)
|
||||
}
|
||||
}
|
||||
raw, err := os.ReadFile(stored)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
// Outside the workspace a check's container mounts, and gone once the build is (novox/hq issue 462); what
|
||||
// it held while git read it is checked with the check's clones (TestACheckContainerSeesNoForgeCredential).
|
||||
if withinDir(workspace, stored) {
|
||||
t.Fatalf("the credential store %s is inside the workspace %s", stored, workspace)
|
||||
}
|
||||
if strings.TrimSpace(string(raw)) != "http://mesh_novox_builder:sw0rdfi5h@forge.invalid:20000" {
|
||||
t.Fatalf("the store does not hold the credential as given: %q", raw)
|
||||
if _, err := os.Stat(stored); !os.IsNotExist(err) {
|
||||
t.Fatalf("the credential store outlives the build: %v", err)
|
||||
}
|
||||
info, err := os.Stat(stored)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
if _, err := os.Stat(filepath.Join(workspace, "git-credentials")); !os.IsNotExist(err) {
|
||||
t.Fatal("a credential file is left in the workspace")
|
||||
}
|
||||
if info.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("the credential file is readable beyond its owner: %v", info.Mode())
|
||||
}
|
||||
|
||||
// storeNamedIn is the credential store a git command line offers.
|
||||
func storeNamedIn(t *testing.T, line string) string {
|
||||
t.Helper()
|
||||
_, after, ok := strings.Cut(line, "credential.helper=store --file=")
|
||||
if !ok {
|
||||
t.Fatalf("the clone does not name the credential store: %s", line)
|
||||
}
|
||||
return strings.Fields(after)[0]
|
||||
}
|
||||
|
||||
// Without a credential, a clone is exactly the invocation it always was, and no credential file
|
||||
@@ -506,7 +510,7 @@ func TestAContextCloneCarriesTheSameCredentialStore(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stored := filepath.Join(workspace, "git-credentials")
|
||||
stored := storeNamedIn(t, r.ran[0])
|
||||
var contextClone string
|
||||
for _, line := range r.ran {
|
||||
if strings.Contains(line, "clone") && strings.Contains(line, "source.git") {
|
||||
|
||||
+137
-16
@@ -146,6 +146,9 @@ type Layer struct {
|
||||
Verdict string
|
||||
Summary string
|
||||
Modules []string
|
||||
// Failed is what the repository's own check printed that names what failed, picked from the whole of
|
||||
// its output and said at the end of the verdict's report (novox/hq issue 460). Empty when it passed.
|
||||
Failed string
|
||||
}
|
||||
|
||||
// CheckScript is what a repository declares its own merge check as: run from its root, with the
|
||||
@@ -226,7 +229,13 @@ func ScriptToolchain(script []byte) string {
|
||||
// Check runs one merge check. An error is that it could not run; the verdict is then "error".
|
||||
func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry string, forge GitCredential,
|
||||
log Log) (CheckVerdict, error) {
|
||||
say := logging(log)
|
||||
// Everything a check says goes to the build's log, which the bus keeps: said redacted (novox/hq issue 462).
|
||||
redact := redactorFor(forge)
|
||||
say := logging(func(step, message string) {
|
||||
if log != nil {
|
||||
log(step, redact.redact(message))
|
||||
}
|
||||
})
|
||||
began := time.Now()
|
||||
ctx, stop := context.WithTimeout(ctx, CheckTimeout)
|
||||
defer stop()
|
||||
@@ -239,20 +248,26 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
||||
return CheckVerdict{}, err
|
||||
}
|
||||
defer os.RemoveAll(root)
|
||||
credentials := ""
|
||||
if forge.URL != "" {
|
||||
credentials = filepath.Join(workspace, "git-credentials")
|
||||
if err := os.WriteFile(credentials, []byte(forge.URL+"\n"), 0o600); err != nil {
|
||||
return CheckVerdict{}, err
|
||||
}
|
||||
// The forge credential lives outside the workspace the check's containers mount, and only while the
|
||||
// check clones (novox/hq issue 462).
|
||||
credentials, forget, err := storeCredential(workspace, forge)
|
||||
if err != nil {
|
||||
return CheckVerdict{}, err
|
||||
}
|
||||
defer forget()
|
||||
if err := forgetLeftCredentials(workspace); err != nil {
|
||||
return CheckVerdict{}, err
|
||||
}
|
||||
clone := func(repository, ref, dir string) error {
|
||||
if _, err := run(ctx, root, "git", cloneWith(credentials, "clone", "--quiet", repository, dir)...); err != nil {
|
||||
return fmt.Errorf("cannot clone %s: %w", repository, err)
|
||||
return fmt.Errorf("cannot clone %s: %s", redact.redact(repository), redact.redact(err.Error()))
|
||||
}
|
||||
if err := recordedWithoutUserinfo(ctx, run, filepath.Join(root, dir), repository); err != nil {
|
||||
return err
|
||||
}
|
||||
if ref != "" {
|
||||
if _, err := run(ctx, filepath.Join(root, dir), "git", "checkout", "--quiet", ref); err != nil {
|
||||
return fmt.Errorf("%s has no %s: %w", repository, ref, err)
|
||||
return fmt.Errorf("%s has no %s: %w", redact.redact(repository), ref, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
@@ -320,6 +335,9 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
||||
}
|
||||
}
|
||||
|
||||
// Every clone is made: the credential is gone before anything of the check runs (novox/hq issue 462).
|
||||
forget()
|
||||
|
||||
// The facts, and the versions they say the mesh runs.
|
||||
if registry == "" {
|
||||
return CheckVerdict{}, errors.New("no artifact store to read the facts snapshot from")
|
||||
@@ -476,8 +494,11 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
||||
}
|
||||
}
|
||||
|
||||
// What the check printed travels in the verdict to the forge and the controller: redacted as the log is.
|
||||
v.Gate.Summary, v.Repo.Summary, v.Repo.Failed = redact.redact(v.Gate.Summary), redact.redact(v.Repo.Summary),
|
||||
redact.redact(v.Repo.Failed)
|
||||
v.Verdict, v.Summary = v.Gate.Verdict, v.Gate.Summary
|
||||
v.Report, v.Took = out.String(), time.Since(began)
|
||||
v.Report, v.Took = redact.redact(withWhatFailed(out.String(), v.Repo)), time.Since(began)
|
||||
say("check", "gate %s — %s; repository %s — %s (%s)", strings.ToUpper(v.Gate.Verdict), v.Gate.Summary,
|
||||
strings.ToUpper(v.Repo.Verdict), v.Repo.Summary, v.Took.Round(time.Second))
|
||||
return v, nil
|
||||
@@ -515,17 +536,29 @@ func ownCheck(ctx context.Context, spec CheckSpec, parts []ScriptPart, tree stri
|
||||
say("check", "running its %s in the mesh's %s toolchain", part.Script, part.Toolchain)
|
||||
fmt.Fprintf(out, "--- its %s (%s toolchain)\n", part.Script, part.Toolchain)
|
||||
var own tail
|
||||
var picked failures
|
||||
logged := &toTheLog{say: say}
|
||||
cmd := command(image, part.Script)
|
||||
inItsOwnGroup(cmd)
|
||||
w := io.MultiWriter(out, &own)
|
||||
w := io.MultiWriter(out, &own, &picked, logged)
|
||||
cmd.Stdout, cmd.Stderr = w, w
|
||||
switch err := cmd.Run(); {
|
||||
err := cmd.Run()
|
||||
logged.close(picked.String())
|
||||
switch {
|
||||
case timedOut():
|
||||
return &Layer{Verdict: "error", Summary: fmt.Sprintf("its %s ran past %s and was ended", part.Script, CheckTimeout)}
|
||||
return &Layer{Verdict: "error", Summary: fmt.Sprintf("its %s ran past %s and was ended", part.Script, CheckTimeout),
|
||||
Failed: picked.String()}
|
||||
case ctx.Err() != nil:
|
||||
return nil
|
||||
case err != nil:
|
||||
return &Layer{Verdict: "fail", Summary: passedSoFar(ran) + "its " + part.Script + " failed: " + whatFailed(own.String())}
|
||||
// Named from the whole run, not the tail: what failed may be thousands of lines from the end
|
||||
// (novox/hq issue 460).
|
||||
said := picked.said()
|
||||
if said == "" {
|
||||
said = whatFailed(own.String())
|
||||
}
|
||||
return &Layer{Verdict: "fail", Summary: passedSoFar(ran) + "its " + part.Script + " failed: " + said,
|
||||
Failed: picked.String()}
|
||||
}
|
||||
ran = append(ran, fmt.Sprintf("%s (%s)", part.Script, part.Toolchain))
|
||||
}
|
||||
@@ -535,6 +568,86 @@ func ownCheck(ctx context.Context, spec CheckSpec, parts []ScriptPart, tree stri
|
||||
return &Layer{Verdict: "pass", Summary: "its " + CheckScript + " passed, each part in its toolchain: " + strings.Join(ran, ", ")}
|
||||
}
|
||||
|
||||
// withWhatFailed is a check's report with what its repository's own check names as failed said last, whole:
|
||||
// the report's tail is the last reportLines lines, and the controller keeps the end of a report it cuts, so
|
||||
// what failed travels whatever came before or after it (novox/hq issue 460).
|
||||
func withWhatFailed(report string, repo *Layer) string {
|
||||
if repo == nil || repo.Failed == "" {
|
||||
return report
|
||||
}
|
||||
return report + "\n--- what failed, picked from the whole of its output (the whole is in the build's log)\n" +
|
||||
repo.Failed
|
||||
}
|
||||
|
||||
// logLines bounds the lines of a repository's own check that go into the build's log. The bus keeps 10 000
|
||||
// messages per subject (EVENTS' MaxMsgsPerSubject), and a build's log is one subject: past that, its first
|
||||
// lines would be lost. The build's own lines are a few hundred at most, so the check's output takes 8 000.
|
||||
const logLines = 8000
|
||||
|
||||
// logLineBytes bounds one line of it, so a line that pastes a document does not fill a message.
|
||||
const logLineBytes = 4 << 10
|
||||
|
||||
// toTheLog says each line a repository's own check prints in the build's log, as it prints it — the whole
|
||||
// output kept for the build and read with `builds` and its id (novox/hq issue 460), where before only the
|
||||
// verdict's tail of it was kept. Past logLines it says how many lines it left out, and the lines that name
|
||||
// what failed after them.
|
||||
type toTheLog struct {
|
||||
say func(step, format string, args ...any)
|
||||
partial []byte
|
||||
said int
|
||||
dropped int
|
||||
}
|
||||
|
||||
func (l *toTheLog) Write(p []byte) (int, error) {
|
||||
l.partial = append(l.partial, p...)
|
||||
for {
|
||||
i := bytes.IndexByte(l.partial, '\n')
|
||||
if i < 0 {
|
||||
break
|
||||
}
|
||||
l.line(string(bytes.TrimRight(l.partial[:i], "\r")))
|
||||
l.partial = l.partial[i+1:]
|
||||
}
|
||||
if len(l.partial) > logLineBytes {
|
||||
l.line(string(l.partial))
|
||||
l.partial = nil
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func (l *toTheLog) line(line string) {
|
||||
if l.said >= logLines {
|
||||
l.dropped++
|
||||
return
|
||||
}
|
||||
if len(line) > logLineBytes {
|
||||
line = line[:logLineBytes] + fmt.Sprintf(" … (%d bytes more)", len(line)-logLineBytes)
|
||||
}
|
||||
l.said++
|
||||
// Redacted by its shape before the bus keeps it (novox/hq issue 462); Check's own say adds the secrets
|
||||
// the builder knows.
|
||||
l.say("output", "%s", redactor{}.redact(line))
|
||||
}
|
||||
|
||||
// close says the last line, and, when lines were left out, how many and what failed.
|
||||
func (l *toTheLog) close(failed string) {
|
||||
if len(l.partial) > 0 {
|
||||
l.line(string(l.partial))
|
||||
l.partial = nil
|
||||
}
|
||||
if l.dropped == 0 {
|
||||
return
|
||||
}
|
||||
l.say("output", "… %d more line(s) of its output are not in this log, which keeps its first %d", l.dropped, logLines)
|
||||
if failed == "" {
|
||||
return
|
||||
}
|
||||
l.say("output", "--- what failed, picked from the whole of its output")
|
||||
for _, line := range strings.Split(failed, "\n") {
|
||||
l.say("output", "%s", redactor{}.redact(line))
|
||||
}
|
||||
}
|
||||
|
||||
// passedSoFar is what of a check's parts passed before the one that did not, said first.
|
||||
func passedSoFar(ran []string) string {
|
||||
if len(ran) == 0 {
|
||||
@@ -1018,9 +1131,17 @@ func whatFailed(s string) string {
|
||||
return "not gofmt'd by the toolchain's gofmt: " + strings.Join(files, ", ")
|
||||
}
|
||||
}
|
||||
for _, prefix := range []string{"--- FAIL:", "FAIL\t", "panic:"} {
|
||||
// A panic, a data race or a build error outside any test says more than the package's bare FAIL line
|
||||
// (novox/hq issue 460).
|
||||
for _, said := range []func(string) bool{
|
||||
func(l string) bool { return strings.HasPrefix(l, "--- FAIL:") },
|
||||
func(l string) bool { return strings.HasPrefix(l, "panic:") || strings.HasPrefix(l, "fatal error:") },
|
||||
func(l string) bool { return l == "WARNING: DATA RACE" },
|
||||
goError.MatchString,
|
||||
func(l string) bool { return strings.HasPrefix(l, "FAIL\t") },
|
||||
} {
|
||||
for _, line := range lines {
|
||||
if line = strings.TrimSpace(line); strings.HasPrefix(line, prefix) {
|
||||
if line = strings.TrimSpace(line); said(line) {
|
||||
return line
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,251 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/facts"
|
||||
)
|
||||
|
||||
// **A check's container never sees the forge credential** (novox/hq issue 462): the toolchain container
|
||||
// mounts the workspace as HOME, so a credential kept there — or a clone's .git/config carrying one — is
|
||||
// readable by any pull request's merge-check.sh, and printed, kept on the bus for days.
|
||||
|
||||
const (
|
||||
forgeSecret = "sw0rdfi5h-forge"
|
||||
forgeURL = "http://mesh_novox_builder:" + forgeSecret + "@forge.invalid:20000"
|
||||
besideSecret = "b3side-t0ken"
|
||||
npmSecret = "npm-s3cret-t0ken"
|
||||
)
|
||||
|
||||
// aFactsRegistry is an artifact store holding the facts snapshot, and nothing else.
|
||||
func aFactsRegistry(t *testing.T) string {
|
||||
t.Helper()
|
||||
body, err := json.Marshal(facts.Facts{Format: facts.Format, Taken: time.Now().UTC(),
|
||||
Versions: facts.Versions{Bus: "2.11.17", Store: "17.11"}, Machines: []facts.Machine{{Name: "abcdef", Length: 6}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
digest := "sha256:" + hex.EncodeToString(sum[:])
|
||||
manifest, _ := json.Marshal(map[string]any{"schemaVersion": 2, "layers": []map[string]any{
|
||||
{"mediaType": facts.MediaType, "digest": digest, "size": len(body)}}})
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case strings.Contains(r.URL.Path, "/manifests/"):
|
||||
w.Write(manifest)
|
||||
case strings.HasSuffix(r.URL.Path, "/blobs/"+digest):
|
||||
w.Write(body)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return strings.TrimPrefix(srv.URL, "http://")
|
||||
}
|
||||
|
||||
// bareURL is a URL with its userinfo left out.
|
||||
func bareURL(t *testing.T, raw string) string {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
u.User = nil
|
||||
return u.String()
|
||||
}
|
||||
|
||||
func TestACheckContainerSeesNoForgeCredential(t *testing.T) {
|
||||
repo, head := aCheckedRepository(t, map[string]string{CheckScript: "echo checked\n"})
|
||||
besideRepo, besideHead := aCheckedRepository(t, map[string]string{"README": "beside"})
|
||||
// A clone source that carries userinfo, as a forge's clone URL may: git records it as given in the
|
||||
// clone's .git/config, which the container reads.
|
||||
besideURL := "file://beside-user:" + besideSecret + "@" + besideRepo
|
||||
workspace := t.TempDir()
|
||||
|
||||
var stores []string
|
||||
reached := false
|
||||
var leaks []string
|
||||
run := func(ctx context.Context, dir, name string, args ...string) (string, error) {
|
||||
switch name {
|
||||
case "git":
|
||||
for _, a := range args {
|
||||
if f, ok := strings.CutPrefix(a, "credential.helper=store --file="); ok {
|
||||
stores = append(stores, f)
|
||||
raw, err := os.ReadFile(f)
|
||||
if err != nil || strings.TrimSpace(string(raw)) != forgeURL {
|
||||
t.Errorf("git is offered a store that does not hold the credential as given: %q, %v", raw, err)
|
||||
}
|
||||
if info, err := os.Stat(f); err == nil && info.Mode().Perm() != 0o600 {
|
||||
t.Errorf("the credential store is readable beyond its owner: %v", info.Mode())
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(args) >= 2 && args[len(args)-3] == "--quiet" && hasString(args, "clone") {
|
||||
source := args[len(args)-2]
|
||||
if u, err := url.Parse(source); err == nil && u.User != nil {
|
||||
// Git cannot reach a file:// URL with userinfo; clone it without, then record it as git
|
||||
// would have: as given.
|
||||
clone := append(append([]string{}, args[:len(args)-2]...), bareURL(t, source), args[len(args)-1])
|
||||
if out, err := Command(ctx, dir, "git", clone...); err != nil {
|
||||
return out, err
|
||||
}
|
||||
return Command(ctx, filepath.Join(dir, args[len(args)-1]), "git", "remote", "set-url", "origin", source)
|
||||
}
|
||||
}
|
||||
return Command(ctx, dir, name, args...)
|
||||
case "docker":
|
||||
if !reached {
|
||||
reached = true
|
||||
// The first container: everything the workspace holds is what the toolchain container sees.
|
||||
filepath.WalkDir(workspace, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
raw, _ := os.ReadFile(path)
|
||||
s := string(raw)
|
||||
if strings.Contains(s, forgeSecret) || strings.Contains(s, besideSecret) ||
|
||||
strings.Contains(s, npmSecret) || d.Name() == "git-credentials" || d.Name() == ".npmrc" ||
|
||||
strings.Contains(s, "credential.helper") {
|
||||
leaks = append(leaks, path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
for _, f := range stores {
|
||||
if _, err := os.Stat(f); !errors.Is(err, os.ErrNotExist) {
|
||||
leaks = append(leaks, f+" (still there when the first container runs)")
|
||||
}
|
||||
if rel, err := filepath.Rel(workspace, f); err == nil && !strings.HasPrefix(rel, "..") {
|
||||
leaks = append(leaks, f+" (inside the workspace the container mounts)")
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(args) > 0 && args[0] == "ps" {
|
||||
return "", nil
|
||||
}
|
||||
return "", errors.New("no container runtime in this test")
|
||||
}
|
||||
return "", errors.New("unexpected command " + name)
|
||||
}
|
||||
// A credential an older builder left in the workspace is removed too: the forge's, and the .npmrc a
|
||||
// build wrote into the tree it cloned.
|
||||
if err := os.WriteFile(filepath.Join(workspace, "git-credentials"), []byte(forgeURL+"\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, dir := range []string{"source/x", "context-server"} {
|
||||
if err := os.MkdirAll(filepath.Join(workspace, dir), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(workspace, dir, ".npmrc"),
|
||||
[]byte("//forge.invalid/api/packages/novox/npm/:_authToken="+npmSecret+"\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
_, err := Check(t.Context(), run, CheckSpec{ID: "check-462", Repository: repo, Ref: head, Owner: "novox",
|
||||
Repo: "mesh-controller", Number: 1, Toolchain: "golang", Beside: map[string]Beside{
|
||||
"mesh-catalog": {Repository: besideURL, Ref: besideHead}}}, workspace, aFactsRegistry(t),
|
||||
GitCredential{URL: forgeURL}, nil)
|
||||
if err == nil {
|
||||
t.Fatal("the check ran past its first container in a test with none")
|
||||
}
|
||||
if !reached {
|
||||
t.Fatalf("the check never reached its first container: %v", err)
|
||||
}
|
||||
if len(stores) == 0 {
|
||||
t.Fatal("no clone was offered the forge credential")
|
||||
}
|
||||
if len(leaks) > 0 {
|
||||
t.Fatalf("the check's container sees the credential:\n%s", strings.Join(leaks, "\n"))
|
||||
}
|
||||
}
|
||||
|
||||
// Every line a repository's own check prints is published to the build's log redacted.
|
||||
func TestACheckLinePublishedToTheLogIsRedacted(t *testing.T) {
|
||||
var said []string
|
||||
say := func(step, format string, args ...any) {
|
||||
if step == "output" && len(args) > 0 {
|
||||
said = append(said, args[0].(string))
|
||||
}
|
||||
}
|
||||
var out tail
|
||||
layer := ownCheck(t.Context(), CheckSpec{Toolchain: "golang"}, []ScriptPart{{Toolchain: "go", Script: CheckScript}},
|
||||
t.TempDir(), &out, func() bool { return false }, func(string, string) *exec.Cmd {
|
||||
return exec.CommandContext(t.Context(), "sh", "-c", "echo cloning http://mesh_builder:t0ps3cret-forge@forge.invalid/novox/x.git; "+
|
||||
"echo token ghp_abcdefghijklmnopqrstuvwxyz0123456789")
|
||||
}, say)
|
||||
if layer == nil || layer.Verdict != "pass" {
|
||||
t.Fatalf("the check answered %+v\n%s", layer, out.String())
|
||||
}
|
||||
joined := strings.Join(said, "\n")
|
||||
if strings.Contains(joined, "t0ps3cret-forge") || strings.Contains(joined, "ghp_abcdef") {
|
||||
t.Fatalf("a secret the check printed is published to the build's log:\n%s", joined)
|
||||
}
|
||||
if !strings.Contains(joined, "http://mesh_builder:[redacted: a password in a URI]@forge.invalid/novox/x.git") {
|
||||
t.Fatalf("the line is not said with what was there named:\n%s", joined)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheRedactorHidesTheForgeCredentialAndShapes(t *testing.T) {
|
||||
r := redactorFor(GitCredential{URL: forgeURL})
|
||||
for in, want := range map[string]string{
|
||||
"the secret alone: " + forgeSecret: "the secret alone: [redacted: the forge credential]",
|
||||
"go test ./... ok": "go test ./... ok",
|
||||
"--password hunter22 and done": "--password [redacted: the word after --password] and done",
|
||||
"commit 3b6b54a0c1d2e3f4a5b6c7d8e9f0": "commit 3b6b54a0c1d2e3f4a5b6c7d8e9f0",
|
||||
} {
|
||||
if got := r.redact(in); got != want {
|
||||
t.Errorf("%q redacted as %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A build's clone of a URL carrying userinfo records it without, so no build leaves a credential in
|
||||
// workspace/source/.git/config while it runs either (novox/hq issue 462); the tree itself is gone when the
|
||||
// build ends.
|
||||
func TestABuildsCloneRecordsNoUserinfo(t *testing.T) {
|
||||
repo, _ := aCheckedRepository(t, map[string]string{ManifestName: `{"module":"plain","version":"1"}`})
|
||||
source := "file://build-user:" + besideSecret + "@" + repo
|
||||
workspace := t.TempDir()
|
||||
tree := filepath.Join(workspace, "source")
|
||||
var configs []string
|
||||
run := func(ctx context.Context, dir, name string, args ...string) (string, error) {
|
||||
if name == "git" && hasString(args, "clone") && args[len(args)-2] == source {
|
||||
clone := append(append([]string{}, args[:len(args)-2]...), bareURL(t, source), args[len(args)-1])
|
||||
if out, err := Command(ctx, dir, "git", clone...); err != nil {
|
||||
return out, err
|
||||
}
|
||||
return Command(ctx, args[len(args)-1], "git", "remote", "set-url", "origin", source)
|
||||
}
|
||||
if name == "git" && len(args) > 0 && args[0] == "rev-parse" {
|
||||
raw, _ := os.ReadFile(filepath.Join(tree, ".git", "config"))
|
||||
configs = append(configs, string(raw))
|
||||
}
|
||||
return Command(ctx, dir, name, args...)
|
||||
}
|
||||
if _, err := Build(t.Context(), run, &recorded{}, source, "", "", workspace, nil, Npmrc{}, GitCredential{}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(configs) == 0 {
|
||||
t.Fatal("the build never read its clone")
|
||||
}
|
||||
for _, c := range configs {
|
||||
if strings.Contains(c, besideSecret) || strings.Contains(c, "build-user") {
|
||||
t.Fatalf("the build's clone records the credential it was cloned with:\n%s", c)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(tree); !os.IsNotExist(err) {
|
||||
t.Fatalf("the build's tree outlives the build: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// **What failed is picked from the whole of a check's output, as it streams** (novox/hq issue 460).
|
||||
//
|
||||
// A repository's own check kept only the last reportLines lines of what it printed, and named what failed
|
||||
// from those. A long store-backed run prints its logs after the failure: the `--- FAIL:` line of mesh-controller
|
||||
// #218 was pushed out of the tail by the package's own log lines, and the verdict said only
|
||||
// "FAIL <package> 268.072s" — the test that failed could not be named, and the fix was a guess. #220's failure
|
||||
// came a few dozen lines before the end, inside the tail, and was named. The length of what came after the
|
||||
// failure decided it, nothing else.
|
||||
//
|
||||
// So every line is looked at as it is written, and the lines that name what failed are kept whole, wherever
|
||||
// in the output they are: each `--- FAIL:` line with its indented message lines, a `panic:` with its first
|
||||
// frames, a `WARNING: DATA RACE` report, each `FAIL\t<package>` line, a build error with its package, and
|
||||
// the files gofmt lists. The rest is cut as before.
|
||||
|
||||
// Bounds on what is picked, so a run that fails everywhere still travels in a verdict: the controller cuts a
|
||||
// report to its last 60 KiB (maxCheckReport), and what failed is said at the report's end, so it stays inside.
|
||||
const (
|
||||
// failureLines and failureBytes bound everything picked from one run.
|
||||
failureLines = 400
|
||||
failureBytes = 32 << 10
|
||||
// failureLineBytes bounds one picked line: a message that pastes a whole document is cut, said.
|
||||
failureLineBytes = 1 << 10
|
||||
// The lines kept after the line that opens each kind of block.
|
||||
failMessageLines = 30 // a --- FAIL's messages
|
||||
panicLines = 40 // a panic's first frames
|
||||
raceLines = 80 // a data race report, to its closing rule
|
||||
buildLines = 40 // a package's build errors
|
||||
gofmtLines = 50 // the files gofmt lists
|
||||
// failedNames bounds how many further failing tests the summary names after the first.
|
||||
failedNames = 5
|
||||
)
|
||||
|
||||
// goError is a compiler's or vet's line: a Go file, a line, and what is wrong there.
|
||||
var goError = regexp.MustCompile(`^\S+\.go:\d+(:\d+)?: `)
|
||||
|
||||
// raceRule is the line the race detector opens and closes its report with.
|
||||
const raceRule = "=================="
|
||||
|
||||
type failureBlock int
|
||||
|
||||
const (
|
||||
noBlock failureBlock = iota
|
||||
inFail
|
||||
inPanic
|
||||
inRace
|
||||
inBuild
|
||||
inGofmt
|
||||
)
|
||||
|
||||
// failures keeps the lines of a check's output that name what failed. It is an io.Writer, fed the same
|
||||
// bytes as the report's tail.
|
||||
type failures struct {
|
||||
partial []byte
|
||||
kept []string
|
||||
size int
|
||||
dropped int
|
||||
|
||||
block failureBlock
|
||||
indent int // a --- FAIL line's indentation: its messages are indented deeper
|
||||
left int // lines the open block may still keep
|
||||
headers []string
|
||||
tests []string // every failing test's --- FAIL line, in order
|
||||
lastRule bool // the line before was the race detector's rule, which opens its report
|
||||
}
|
||||
|
||||
func (f *failures) Write(p []byte) (int, error) {
|
||||
f.partial = append(f.partial, p...)
|
||||
for {
|
||||
i := bytes.IndexByte(f.partial, '\n')
|
||||
if i < 0 {
|
||||
break
|
||||
}
|
||||
f.line(strings.TrimRight(string(f.partial[:i]), "\r"))
|
||||
f.partial = f.partial[i+1:]
|
||||
}
|
||||
// A line with no end, longer than any line is kept, is judged by its start.
|
||||
if len(f.partial) > failureLineBytes*4 {
|
||||
f.line(string(f.partial))
|
||||
f.partial = nil
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
// flush judges what is left of a last line with no newline.
|
||||
func (f *failures) flush() {
|
||||
if len(f.partial) > 0 {
|
||||
f.line(strings.TrimRight(string(f.partial), "\r"))
|
||||
f.partial = nil
|
||||
}
|
||||
}
|
||||
|
||||
func indentOf(line string) int {
|
||||
return len(line) - len(strings.TrimLeft(line, " \t"))
|
||||
}
|
||||
|
||||
func (f *failures) keep(line string) {
|
||||
if f.size >= failureBytes || len(f.kept) >= failureLines {
|
||||
f.dropped++
|
||||
return
|
||||
}
|
||||
if len(line) > failureLineBytes {
|
||||
line = line[:failureLineBytes] + fmt.Sprintf(" … (%d bytes more)", len(line)-failureLineBytes)
|
||||
}
|
||||
f.kept = append(f.kept, line)
|
||||
f.size += len(line) + 1
|
||||
}
|
||||
|
||||
func (f *failures) open(block failureBlock, lines int) {
|
||||
f.block, f.left = block, lines
|
||||
}
|
||||
|
||||
func (f *failures) line(line string) {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
rule := trimmed == raceRule
|
||||
|
||||
// A line that opens a block ends whatever block was open.
|
||||
if f.opens(line, trimmed) {
|
||||
f.lastRule = rule
|
||||
return
|
||||
}
|
||||
switch f.block {
|
||||
case inFail:
|
||||
if trimmed != "" && indentOf(line) > f.indent && f.left > 0 {
|
||||
f.left--
|
||||
f.keep(line)
|
||||
f.lastRule = rule
|
||||
return
|
||||
}
|
||||
case inPanic:
|
||||
if f.left > 0 && !strings.HasPrefix(line, "FAIL") && !strings.HasPrefix(line, "ok \t") &&
|
||||
!strings.HasPrefix(line, "exit status") {
|
||||
f.left--
|
||||
f.keep(line)
|
||||
f.lastRule = rule
|
||||
return
|
||||
}
|
||||
case inRace:
|
||||
if f.left > 0 {
|
||||
f.left--
|
||||
f.keep(line)
|
||||
if rule {
|
||||
f.block = noBlock
|
||||
}
|
||||
f.lastRule = rule
|
||||
return
|
||||
}
|
||||
case inBuild:
|
||||
if f.left > 0 && trimmed != "" && !strings.HasPrefix(line, "FAIL") && !strings.HasPrefix(line, "ok \t") &&
|
||||
!strings.HasPrefix(line, "? \t") {
|
||||
f.left--
|
||||
f.keep(line)
|
||||
f.lastRule = rule
|
||||
return
|
||||
}
|
||||
case inGofmt:
|
||||
if f.left > 0 && trimmed != "" {
|
||||
f.left--
|
||||
f.keep(line)
|
||||
f.lastRule = rule
|
||||
return
|
||||
}
|
||||
}
|
||||
f.block = noBlock
|
||||
|
||||
switch {
|
||||
case strings.HasPrefix(line, "# "):
|
||||
// A package's name before its build errors — kept only when an error follows it.
|
||||
f.headers = append(f.headers, line)
|
||||
if len(f.headers) > 4 {
|
||||
f.headers = f.headers[1:]
|
||||
}
|
||||
case len(f.headers) > 0 && goError.MatchString(line):
|
||||
for _, h := range f.headers {
|
||||
f.keep(h)
|
||||
}
|
||||
f.headers = nil
|
||||
f.keep(line)
|
||||
f.open(inBuild, buildLines)
|
||||
case strings.HasPrefix(line, "FAIL\t"):
|
||||
f.headers = nil
|
||||
f.keep(line)
|
||||
default:
|
||||
f.headers = nil
|
||||
}
|
||||
f.lastRule = rule
|
||||
}
|
||||
|
||||
// opens keeps a line that opens a block of what failed, and opens the block. False for any other line.
|
||||
func (f *failures) opens(line, trimmed string) bool {
|
||||
switch {
|
||||
case strings.HasPrefix(trimmed, "--- FAIL:"):
|
||||
f.headers = nil
|
||||
f.keep(line)
|
||||
f.tests = append(f.tests, trimmed)
|
||||
f.indent = indentOf(line)
|
||||
f.open(inFail, failMessageLines)
|
||||
case strings.HasPrefix(line, "panic:") || strings.HasPrefix(line, "fatal error:"):
|
||||
f.headers = nil
|
||||
f.keep(line)
|
||||
f.open(inPanic, panicLines)
|
||||
case trimmed == "WARNING: DATA RACE":
|
||||
f.headers = nil
|
||||
if f.lastRule {
|
||||
f.keep(raceRule)
|
||||
}
|
||||
f.keep(line)
|
||||
f.open(inRace, raceLines)
|
||||
case strings.HasPrefix(trimmed, "not gofmt'd:"):
|
||||
f.headers = nil
|
||||
f.keep(line)
|
||||
f.open(inGofmt, gofmtLines)
|
||||
default:
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// String is every line picked, in the order the run printed them, and how many more were left out.
|
||||
func (f *failures) String() string {
|
||||
f.flush()
|
||||
s := strings.Join(f.kept, "\n")
|
||||
if f.dropped > 0 {
|
||||
s += fmt.Sprintf("\n… %d more line(s) naming what failed, past the %d lines or %d KiB kept", f.dropped,
|
||||
failureLines, failureBytes>>10)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// said is what the repository layer's summary says failed: the first thing that failed, whole, then the
|
||||
// further failing tests by name. Empty when nothing was picked.
|
||||
func (f *failures) said() string {
|
||||
picked := f.String()
|
||||
if picked == "" {
|
||||
return ""
|
||||
}
|
||||
first := whatFailed(picked)
|
||||
var more []string
|
||||
for _, t := range f.tests {
|
||||
if t == first {
|
||||
continue
|
||||
}
|
||||
// A subtest's parent fails with it; its name is in the subtest's.
|
||||
name := strings.Fields(strings.TrimPrefix(t, "--- FAIL:"))
|
||||
if len(name) > 0 {
|
||||
more = append(more, name[0])
|
||||
}
|
||||
}
|
||||
more = withoutParents(more)
|
||||
switch {
|
||||
case len(more) == 0:
|
||||
return first
|
||||
case len(more) > failedNames:
|
||||
return fmt.Sprintf("%s; and %s and %d more", first, strings.Join(more[:failedNames], ", "),
|
||||
len(more)-failedNames)
|
||||
default:
|
||||
return first + "; and " + strings.Join(more, ", ")
|
||||
}
|
||||
}
|
||||
|
||||
// withoutParents drops a test whose subtest is also named.
|
||||
func withoutParents(names []string) []string {
|
||||
var out []string
|
||||
for _, n := range names {
|
||||
parent := false
|
||||
for _, m := range names {
|
||||
if strings.HasPrefix(m, n+"/") {
|
||||
parent = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !parent {
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// ownCheckOf runs a repository's own check whose script prints a captured output and fails, as the build
|
||||
// seat runs it, and answers the layer and every line it said in the build's log.
|
||||
func ownCheckOf(t *testing.T, printed string) (*Layer, []string) {
|
||||
t.Helper()
|
||||
tree := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(tree, "printed.txt"), []byte(printed), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(tree, CheckScript), []byte("cat printed.txt\nexit 1\n"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out tail
|
||||
var logged []string
|
||||
layer := ownCheck(t.Context(), CheckSpec{Toolchain: "go-image"}, []ScriptPart{{"go", CheckScript}}, tree, &out,
|
||||
func() bool { return false },
|
||||
func(_, script string) *exec.Cmd {
|
||||
cmd := exec.CommandContext(t.Context(), "sh", script)
|
||||
cmd.Dir = tree
|
||||
return cmd
|
||||
}, func(step, format string, args ...any) {
|
||||
if step == "output" {
|
||||
logged = append(logged, fmt.Sprintf(format, args...))
|
||||
}
|
||||
})
|
||||
if layer == nil || layer.Verdict != "fail" {
|
||||
t.Fatalf("a failing check answered %+v", layer)
|
||||
}
|
||||
return layer, logged
|
||||
}
|
||||
|
||||
func captured(t *testing.T, name string) string {
|
||||
t.Helper()
|
||||
body, err := os.ReadFile(filepath.Join("testdata", "issue460", name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(body)
|
||||
}
|
||||
|
||||
// **novox/hq issue 460**: mesh-controller #218's check failed with "FAIL <package> 268.072s" and no test
|
||||
// name: the package's own logs, printed after its `--- FAIL:` lines, pushed them out of the last 200 lines,
|
||||
// which was all the summary was named from. The first failing test is named, and the others after it,
|
||||
// however much came after them.
|
||||
func TestAFailureEarlyInALongRunIsStillNamed(t *testing.T) {
|
||||
layer, _ := ownCheckOf(t, captured(t, "a-long-run-failing-early.txt"))
|
||||
if !strings.Contains(layer.Summary, "failed: --- FAIL: TestTheEnvelopeIsPinned (0.00s)") {
|
||||
t.Errorf("a failure 600 lines from the end is said as %q", layer.Summary)
|
||||
}
|
||||
if !strings.Contains(layer.Summary, "TestSubtests/the_hub") {
|
||||
t.Errorf("the second failing test is not named: %q", layer.Summary)
|
||||
}
|
||||
}
|
||||
|
||||
// One -race run of every package: the first failure is the earliest, not the last within the tail.
|
||||
func TestARaceRunOfEveryPackageNamesItsFirstFailure(t *testing.T) {
|
||||
layer, _ := ownCheckOf(t, captured(t, "a-race-run-of-every-package.txt"))
|
||||
if !strings.Contains(layer.Summary, "failed: --- FAIL: TestTheEnvelopeIsPinned (0.00s)") ||
|
||||
!strings.Contains(layer.Summary, "TestAMapIsNil") || !strings.Contains(layer.Summary, "TestACounterIsShared") {
|
||||
t.Errorf("a run failing in four packages is said as %q", layer.Summary)
|
||||
}
|
||||
}
|
||||
|
||||
// A package that does not build is named by its error, not by its bare "[build failed]".
|
||||
func TestABuildErrorIsNamedByTheError(t *testing.T) {
|
||||
layer, _ := ownCheckOf(t, captured(t, "a-build-error.txt"))
|
||||
if !strings.HasSuffix(layer.Summary, "failed: broken/broken.go:3:28: undefined: undefinedThing") {
|
||||
t.Errorf("a build error is said as %q", layer.Summary)
|
||||
}
|
||||
}
|
||||
|
||||
// The whole of what the check printed is in the build's log, line by line — where before none of it was.
|
||||
func TestTheWholeOutputIsInTheBuildsLog(t *testing.T) {
|
||||
printed := captured(t, "a-long-run-failing-early.txt")
|
||||
_, logged := ownCheckOf(t, printed)
|
||||
want := strings.Split(strings.TrimRight(printed, "\n"), "\n")
|
||||
if len(logged) != len(want) {
|
||||
t.Fatalf("the build's log holds %d line(s) of the %d printed", len(logged), len(want))
|
||||
}
|
||||
for i := range want {
|
||||
if logged[i] != want[i] {
|
||||
t.Fatalf("line %d is logged as %q, printed as %q", i+1, logged[i], want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// What failed is kept whole in the verdict: each --- FAIL with its messages, a panic with its first frames,
|
||||
// a data race report from rule to rule — and none of the log lines around them.
|
||||
func TestTheVerdictKeepsWhatFailedWhole(t *testing.T) {
|
||||
layer, _ := ownCheckOf(t, captured(t, "a-long-run-failing-early.txt"))
|
||||
want := "--- FAIL: TestTheEnvelopeIsPinned (0.00s)\n" +
|
||||
" early_test.go:9: the envelope's subject is \"mesh.a\", not \"mesh.b\"\n" +
|
||||
" early_test.go:10: a second line of the same failure\n" +
|
||||
"--- FAIL: TestSubtests (0.00s)\n" +
|
||||
" --- FAIL: TestSubtests/the_hub (0.00s)\n" +
|
||||
" early_test.go:14: the hub does not compose\n" +
|
||||
"FAIL\texample.com/cap/early\t"
|
||||
if !strings.HasPrefix(layer.Failed, want) || strings.Contains(layer.Failed, "kept what home-server says") {
|
||||
t.Errorf("the early failure is kept as:\n%s", layer.Failed)
|
||||
}
|
||||
|
||||
layer, _ = ownCheckOf(t, captured(t, "a-panic.txt"))
|
||||
for _, line := range []string{"--- FAIL: TestAMapIsNil (0.00s)", "panic: assignment to entry in nil map",
|
||||
"[running]:", "example.com/cap/panics.TestAMapIsNil(", "/src/cap/panics/panics_test.go:7",
|
||||
"FAIL\texample.com/cap/panics\t"} {
|
||||
if !strings.Contains(layer.Failed, line) {
|
||||
t.Errorf("a panic is kept without %q:\n%s", line, layer.Failed)
|
||||
}
|
||||
}
|
||||
|
||||
layer, _ = ownCheckOf(t, captured(t, "a-data-race.txt"))
|
||||
race := captured(t, "a-data-race.txt")
|
||||
report := race[:strings.Index(race, "--- FAIL:")]
|
||||
if !strings.HasPrefix(layer.Failed, report) ||
|
||||
!strings.Contains(layer.Failed, "--- FAIL: TestACounterIsShared (0.00s)\n testing.go:1865: race detected") {
|
||||
t.Errorf("a data race is kept as:\n%s", layer.Failed)
|
||||
}
|
||||
|
||||
layer, _ = ownCheckOf(t, captured(t, "a-build-error.txt"))
|
||||
if layer.Failed != "# example.com/cap/broken\nbroken/broken.go:3:28: undefined: undefinedThing\n"+
|
||||
"FAIL\texample.com/cap/broken [build failed]" {
|
||||
t.Errorf("a build error is kept as:\n%s", layer.Failed)
|
||||
}
|
||||
}
|
||||
|
||||
// A run that fails everywhere is bounded in what it keeps, and what it keeps survives the controller's cut
|
||||
// of a report to its last 60 KiB, said at the report's end.
|
||||
func TestWhatFailedIsBoundedAndSurvivesTheReportsCut(t *testing.T) {
|
||||
var b strings.Builder
|
||||
for i := range 3000 {
|
||||
fmt.Fprintf(&b, "--- FAIL: TestNumber%d (0.00s)\n x_test.go:1: %s\n", i, strings.Repeat("why ", 600))
|
||||
fmt.Fprintf(&b, "2026/10/11 01:30:03 a log line %d\n", i)
|
||||
}
|
||||
b.WriteString("FAIL\tx/everything\t1s\nFAIL\n")
|
||||
layer, logged := ownCheckOf(t, b.String())
|
||||
if len(layer.Failed) > failureBytes+2*failureLineBytes || !strings.Contains(layer.Failed, "more line(s) naming what failed") {
|
||||
t.Errorf("what failed in a run that fails everywhere is %d bytes, ending %q", len(layer.Failed),
|
||||
layer.Failed[max(0, len(layer.Failed)-200):])
|
||||
}
|
||||
if !strings.Contains(layer.Summary, "--- FAIL: TestNumber0 (0.00s); and TestNumber1, ") ||
|
||||
!strings.Contains(layer.Summary, "and 2994 more") {
|
||||
t.Errorf("a run failing 3000 tests is said as %q", layer.Summary)
|
||||
}
|
||||
|
||||
var out tail
|
||||
out.Write([]byte(b.String()))
|
||||
report := withWhatFailed(out.String(), layer)
|
||||
const maxCheckReport = 60 << 10 // as the controller cuts it, from the front
|
||||
if len(report) > maxCheckReport {
|
||||
report = report[len(report)-maxCheckReport:]
|
||||
}
|
||||
if !strings.Contains(report, "--- what failed") || !strings.Contains(report, "--- FAIL: TestNumber0 (0.00s)") {
|
||||
t.Error("what failed did not survive the controller's cut of the report")
|
||||
}
|
||||
|
||||
// The build's log keeps its first logLines lines, says how many it left out, then what failed.
|
||||
if len(logged) < logLines+2 || logged[logLines] != "… 1002 more line(s) of its output are not in this log, which keeps its first 8000" ||
|
||||
logged[logLines+1] != "--- what failed, picked from the whole of its output" ||
|
||||
logged[logLines+2] != "--- FAIL: TestNumber0 (0.00s)" {
|
||||
t.Errorf("the log past its bound says %q", logged[logLines:min(len(logged), logLines+3)])
|
||||
}
|
||||
}
|
||||
|
||||
// A passing check says nothing of what failed.
|
||||
func TestAPassingCheckReportsAsBefore(t *testing.T) {
|
||||
if got := withWhatFailed("ok\tx\t1s", &Layer{Verdict: "pass"}); got != "ok\tx\t1s" {
|
||||
t.Errorf("a passing report became %q", got)
|
||||
}
|
||||
if got := withWhatFailed("r", nil); got != "r" {
|
||||
t.Errorf("no repository layer became %q", got)
|
||||
}
|
||||
}
|
||||
@@ -429,6 +429,11 @@ func (r Registry) copyBlob(ctx context.Context, src *source, where upstream, dig
|
||||
if response.ContentLength > 0 {
|
||||
put.ContentLength = response.ContentLength
|
||||
}
|
||||
// **Not waited for if refused** (novox/hq issue 457): the body streams from upstream and cannot be
|
||||
// read twice, so a registry held still between the POST above and this PUT fails the copy with
|
||||
// "its body cannot be read twice" rather than waiting. Accepted: the POST a moment before already
|
||||
// waited the registry out, so the window is the length of one upstream fetch, and the build fails
|
||||
// loudly, to be asked again, rather than buffering every base blob in memory.
|
||||
done, err := r.client().Do(put)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot upload blob %s: %w", digest, err)
|
||||
|
||||
@@ -70,7 +70,16 @@ func TestNpmrcDisabledUntilThereIsARegistry(t *testing.T) {
|
||||
func TestAnImageBuildGetsTheCredentialInTheContextAndHostNetwork(t *testing.T) {
|
||||
r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch\nCOPY .npmrc ./", "files/x": "y"})
|
||||
n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm/", Token: "t"}
|
||||
if _, err := Build(context.Background(), r.run, r,
|
||||
npmrc := filepath.Join(workspace, "source", ".npmrc")
|
||||
inContext := false
|
||||
run := func(ctx context.Context, dir, name string, args ...string) (string, error) {
|
||||
if name == "docker" && len(args) > 0 && args[0] == "build" {
|
||||
_, err := os.Stat(npmrc)
|
||||
inContext = err == nil
|
||||
}
|
||||
return r.run(ctx, dir, name, args...)
|
||||
}
|
||||
if _, err := Build(context.Background(), run, r,
|
||||
"https://forge.invalid/meshboard.git", "", "", workspace, nil, n, GitCredential{}, nil); err != nil {
|
||||
t.Fatalf("the build failed: %v", err)
|
||||
}
|
||||
@@ -90,11 +99,14 @@ func TestAnImageBuildGetsTheCredentialInTheContextAndHostNetwork(t *testing.T) {
|
||||
if !strings.Contains(build, "--network host") {
|
||||
t.Fatalf("the build was not given the host network to reach the registry: %s", build)
|
||||
}
|
||||
// The .npmrc is written into the build context (the source tree), where a Dockerfile COPYs it.
|
||||
tree := filepath.Join(workspace, "source")
|
||||
npmrc := filepath.Join(tree, ".npmrc")
|
||||
if _, err := os.Stat(npmrc); err != nil {
|
||||
t.Fatalf("the credential was not written into the build context: %v", err)
|
||||
// The .npmrc is written into the build context (the source tree), where a Dockerfile COPYs it, and
|
||||
// removed with the tree when the build ends: a later check's container mounts the workspace (novox/hq
|
||||
// issue 462).
|
||||
if !inContext {
|
||||
t.Fatal("the credential was not in the build context when the image was built")
|
||||
}
|
||||
if _, err := os.Stat(npmrc); !os.IsNotExist(err) {
|
||||
t.Fatalf("the credential outlives the build in the workspace: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
package builder
|
||||
|
||||
// **Every line of a check's output is redacted before it is kept** (novox/hq issue 462).
|
||||
//
|
||||
// A repository's own check prints into the build's log, which the bus keeps for days and anyone who may read
|
||||
// its events reads, and into the verdict, which the forge shows on the pull request. Software prints what it
|
||||
// was given — a URL carrying a password, a token in a flag — and a pull request may print on purpose.
|
||||
// So a line is said only after every secret the builder knows (the forge credential's password) and every
|
||||
// value whose shape says it is one is replaced by a mark naming what was there, as the journal verb does.
|
||||
//
|
||||
// Copied from the journal tool's redactor (mesh-catalog, modules/systemd/cmd/systemd-tools/secrets.go,
|
||||
// itself a copy of the docker module's), narrowed to a line's shapes, with the token shapes a check's output
|
||||
// may carry added. A third copy: sharing them through mesh-sdk is novox/hq issue 471.
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// secretName is a variable name that says its value is a secret.
|
||||
var secretName = regexp.MustCompile(`(?i)(pass(word|wd|phrase)?|secret|token|api_?key|private_?key|access_?key|credential|auth)`)
|
||||
|
||||
// notAValue is a name that says its value is where a secret is, not the secret: a file or a path.
|
||||
var notAValue = regexp.MustCompile(`(?i)(_FILE|FILE|_PATH|_DIR)$`)
|
||||
|
||||
// uriPassword is a URI carrying a password in its userinfo: scheme://user:password@.
|
||||
var uriPassword = regexp.MustCompile(`[A-Za-z][A-Za-z0-9+.-]*://[^\s/:@'"]*:([^\s/@'"]+)@`)
|
||||
|
||||
// tokenShaped are tokens recognised by their own prefix, whatever surrounds them: a forge's or a host's
|
||||
// access token, a JSON web token, a NATS seed.
|
||||
var tokenShaped = []struct {
|
||||
name string
|
||||
re *regexp.Regexp
|
||||
}{
|
||||
{"an access token", regexp.MustCompile(`\b(gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}|xox[abpr]-[A-Za-z0-9-]{10,}|sk-ant-[A-Za-z0-9_-]{20,})`)},
|
||||
{"a JSON web token", regexp.MustCompile(`\beyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]+`)},
|
||||
{"a NATS seed", regexp.MustCompile(`\bS[ACNOU][A-Z2-7]{56}\b`)},
|
||||
}
|
||||
|
||||
// masked is a password a program already hid: ***, xxx, <redacted>, [REDACTED].
|
||||
var masked = regexp.MustCompile(`^(\*+|x+|X+|<[^>]*>|\[[^\]]*\]|%2A+)$`)
|
||||
|
||||
// ordinary is a value under a secret's name that is not one: a path, an address, a number, a switch.
|
||||
var ordinary = regexp.MustCompile(`^(/.*|[A-Za-z][A-Za-z0-9+.-]*://.*|[0-9.]+[a-z]?|(?i:true|false|yes|no|on|off|none|null))$`)
|
||||
|
||||
// leastSecret is the shortest value compared as a secret: a shorter one matches ordinary words.
|
||||
const leastSecret = 6
|
||||
|
||||
// passwordFlags take a secret as their next word, or after `=`, whatever the program.
|
||||
var passwordFlags = map[string]bool{
|
||||
"-P": true, "--password": true, "--pass": true, "--passwd": true, "--secret": true, "--secret-key": true,
|
||||
"--token": true, "--api-key": true, "--apikey": true, "--auth": true,
|
||||
}
|
||||
|
||||
// knownSecret is one value the builder holds, by the name it is said under.
|
||||
type knownSecret struct {
|
||||
Name string
|
||||
Value string
|
||||
}
|
||||
|
||||
// redactor hides the secrets it knows and those a line's shapes say are secrets.
|
||||
type redactor struct{ known []knownSecret }
|
||||
|
||||
// redactorFor knows the forge credential's password, and its user's name with it, in every form git or a
|
||||
// program may print them.
|
||||
func redactorFor(forge GitCredential) redactor {
|
||||
var r redactor
|
||||
if forge.URL == "" {
|
||||
return r
|
||||
}
|
||||
for _, m := range uriPassword.FindAllStringSubmatch(forge.URL, -1) {
|
||||
r.add("the forge credential", m[1])
|
||||
if dec, err := url.PathUnescape(m[1]); err == nil && dec != m[1] {
|
||||
r.add("the forge credential", dec)
|
||||
}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *redactor) add(name, value string) {
|
||||
if len(value) < leastSecret || masked.MatchString(value) {
|
||||
return
|
||||
}
|
||||
for _, k := range r.known {
|
||||
if k.Value == value {
|
||||
return
|
||||
}
|
||||
}
|
||||
r.known = append(r.known, knownSecret{name, value})
|
||||
}
|
||||
|
||||
// redact is a text with every known secret, every value its shape says is one, and every password inside a
|
||||
// URI replaced by a mark naming what was there. Line by line: a shape is judged within its line.
|
||||
func (r redactor) redact(text string) string {
|
||||
if !strings.ContainsAny(text, "\n") {
|
||||
return r.line(text)
|
||||
}
|
||||
lines := strings.Split(text, "\n")
|
||||
for i, l := range lines {
|
||||
lines[i] = r.line(l)
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
func (r redactor) line(line string) string {
|
||||
replace := func(s knownSecret) {
|
||||
for _, f := range forms(s.Value) {
|
||||
line = strings.ReplaceAll(line, f, "[redacted: "+s.Name+"]")
|
||||
}
|
||||
}
|
||||
for _, s := range r.known {
|
||||
replace(s)
|
||||
}
|
||||
for _, s := range shaped(line) {
|
||||
replace(s)
|
||||
}
|
||||
line = uriPassword.ReplaceAllStringFunc(line, func(m string) string {
|
||||
sub := uriPassword.FindStringSubmatch(m)
|
||||
if masked.MatchString(sub[1]) || strings.HasPrefix(sub[1], "[redacted") {
|
||||
return m
|
||||
}
|
||||
return strings.TrimSuffix(m, sub[1]+"@") + "[redacted: a password in a URI]@"
|
||||
})
|
||||
for _, t := range tokenShaped {
|
||||
line = t.re.ReplaceAllString(line, "[redacted: "+t.name+"]")
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
// forms are the ways a value may appear printed: as given, and URL-encoded.
|
||||
func forms(value string) []string {
|
||||
out := []string{value}
|
||||
for _, f := range []string{url.QueryEscape(value), url.PathEscape(value)} {
|
||||
if f != value && !hasString(out, f) {
|
||||
out = append(out, f)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func hasString(list []string, s string) bool {
|
||||
for _, x := range list {
|
||||
if x == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// shaped are the values a line carries by their shape: the word after a password flag, or the value of one
|
||||
// given with `=`, and a NAME=value whose name says secret.
|
||||
func shaped(line string) []knownSecret {
|
||||
var out []knownSecret
|
||||
add := func(name, value string) {
|
||||
value = strings.Trim(value, `"',;`)
|
||||
if len(value) < leastSecret || masked.MatchString(value) || ordinary.MatchString(value) ||
|
||||
strings.HasPrefix(value, "[redacted") {
|
||||
return
|
||||
}
|
||||
out = append(out, knownSecret{name, value})
|
||||
}
|
||||
words := strings.Fields(line)
|
||||
for i, w := range words {
|
||||
if flag, value, ok := strings.Cut(w, "="); ok && strings.HasPrefix(flag, "-") {
|
||||
if passwordFlags[flag] {
|
||||
add("the value of "+flag, value)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if name, value, ok := strings.Cut(w, "="); ok && name != "" && secretName.MatchString(name) &&
|
||||
!notAValue.MatchString(name) && !strings.ContainsAny(name, "/:") {
|
||||
add("the value of "+name, value)
|
||||
continue
|
||||
}
|
||||
if i+1 < len(words) && passwordFlags[w] {
|
||||
add("the word after "+w, words[i+1])
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// withoutUserinfo is a URL with its userinfo left out, and whether it carried any.
|
||||
func withoutUserinfo(raw string) (string, bool) {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil || u.User == nil {
|
||||
return raw, false
|
||||
}
|
||||
u.User = nil
|
||||
return u.String(), true
|
||||
}
|
||||
@@ -52,7 +52,11 @@ func (r Registry) PublishImage(ctx context.Context, localTag, repository string)
|
||||
if _, err := r.Run(ctx, "", "docker", "tag", localTag, remote); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := r.Run(ctx, "", "docker", "push", remote); err != nil {
|
||||
// The push waits for a registry held still, as every request to it does (novox/hq issue 457).
|
||||
if err := waitForRegistry(ctx, r.Address, "docker push "+remote, func() error {
|
||||
_, err := r.Run(ctx, "", "docker", "push", remote)
|
||||
return err
|
||||
}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
out, err := r.Run(ctx, "", "docker", "inspect", "--format", "{{index .RepoDigests 0}}", remote)
|
||||
@@ -169,11 +173,13 @@ func (r Registry) has(ctx context.Context, url string, accept ...string) (bool,
|
||||
return response.StatusCode == http.StatusOK, nil
|
||||
}
|
||||
|
||||
// client is the client for the registry and for upstream, whose requests to the registry wait out a
|
||||
// registry held still (novox/hq issue 457).
|
||||
func (r Registry) client() *http.Client {
|
||||
if r.HTTP != nil {
|
||||
return r.HTTP
|
||||
return waiting(r.HTTP, r.Address)
|
||||
}
|
||||
return http.DefaultClient
|
||||
return waiting(http.DefaultClient, r.Address)
|
||||
}
|
||||
|
||||
// separator is whether the upload location already carries a query.
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A build waits out a registry that refuses connections, for a bounded time (novox/hq issue 457).
|
||||
//
|
||||
// **Why waiting, and why here.** The store's nightly collection holds the registry still for its run —
|
||||
// about a minute and a half, measured on 2026-10-11 — and a build that reached the registry in that
|
||||
// window failed on "connection refused", and its whole delivery plan with it: a plan failed for a
|
||||
// pause the mesh itself scheduled. The other design weighed was the collection telling the controller
|
||||
// it holds the registry, and the controller holding build asks while it runs. Waiting here is smaller
|
||||
// and covers more: it is local to the one place that talks to the registry, needs no new message
|
||||
// between modules, and also carries a build over any other short outage — a registry restarted by its
|
||||
// own update, say. A refusal is the one error waited for: nothing was sent, so trying again cannot
|
||||
// do anything twice, and it is what a registry that is stopped answers.
|
||||
//
|
||||
// **Bounded, and loud past the bound.** registryWait is longer than the collection holds the registry
|
||||
// (five minutes against about one and a half), so the pause the mesh schedules is always waited out,
|
||||
// and a registry that is really down still fails the build — saying how long it was refused — rather
|
||||
// than holding a build machine for ever. Every wait is said in the build's log, with why, and so is
|
||||
// the registry answering again.
|
||||
|
||||
var (
|
||||
// registryWait is how long a build waits for a registry that refuses, per call that found it so.
|
||||
registryWait = 5 * time.Minute
|
||||
// registryFirstPause is the first pause between tries; each pause doubles, up to registryMostPause.
|
||||
registryFirstPause = time.Second
|
||||
)
|
||||
|
||||
// registryMostPause is the longest pause between two tries: short enough that a build goes on within
|
||||
// seconds of the registry answering again.
|
||||
const registryMostPause = 10 * time.Second
|
||||
|
||||
// refused is whether an error is a connection refused: from a dial here, or as a command such as docker
|
||||
// said it in its output.
|
||||
func refused(err error) bool {
|
||||
return err != nil && (errors.Is(err, syscall.ECONNREFUSED) || strings.Contains(err.Error(), "connection refused"))
|
||||
}
|
||||
|
||||
// waitForRegistry runs try, and while it fails because the registry at address refuses connections,
|
||||
// tries again with a growing pause until registryWait has passed. what names the call, for the log.
|
||||
func waitForRegistry(ctx context.Context, address, what string, try func() error) error {
|
||||
err := try()
|
||||
if !refused(err) {
|
||||
return err
|
||||
}
|
||||
started := time.Now()
|
||||
pause := registryFirstPause
|
||||
tell("registry", "%s: refused; the build waits for the registry at %s, for up to %s — it is held still while "+
|
||||
"the store's nightly collection runs, about a minute and a half (novox/hq issue 457)",
|
||||
what, address, registryWait)
|
||||
for {
|
||||
left := registryWait - time.Since(started)
|
||||
if left <= 0 {
|
||||
tell("registry", "%s: the registry at %s still refuses after %s; the build fails", what, address,
|
||||
time.Since(started).Round(time.Second))
|
||||
return fmt.Errorf("the registry at %s refused every connection for %s, longer than its nightly "+
|
||||
"collection holds it still, so it is down, not paused: %w",
|
||||
address, time.Since(started).Round(time.Millisecond), err)
|
||||
}
|
||||
wait := min(pause, left)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("stopped while waiting for the registry at %s: %w (last: %v)", address, ctx.Err(), err)
|
||||
case <-time.After(wait):
|
||||
}
|
||||
pause = min(pause*2, registryMostPause)
|
||||
if err = try(); !refused(err) {
|
||||
// Said as it is: the registry answering is only the build going on when the call worked.
|
||||
if err == nil {
|
||||
tell("registry", "%s: the registry at %s answers again after %s; the build goes on", what, address,
|
||||
time.Since(started).Round(time.Millisecond))
|
||||
} else {
|
||||
tell("registry", "%s: the registry at %s no longer refuses after %s, and answered with: %v", what,
|
||||
address, time.Since(started).Round(time.Millisecond), err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// waitingTransport waits for the registry on every request to it, and on none to anywhere else: the
|
||||
// same client copies from upstream registries, whose refusals are theirs to answer.
|
||||
type waitingTransport struct {
|
||||
base http.RoundTripper
|
||||
address string
|
||||
}
|
||||
|
||||
func (t waitingTransport) RoundTrip(request *http.Request) (*http.Response, error) {
|
||||
if request.URL.Host != t.address {
|
||||
return t.base.RoundTrip(request)
|
||||
}
|
||||
var response *http.Response
|
||||
tries := 0
|
||||
err := waitForRegistry(request.Context(), t.address, request.Method+" "+request.URL.Path, func() error {
|
||||
attempt := request
|
||||
if tries > 0 && request.Body != nil && request.Body != http.NoBody {
|
||||
// A body is sent again only when it can be read again; one that cannot is not retried.
|
||||
if request.GetBody == nil {
|
||||
return fmt.Errorf("%s %s cannot be sent again: its body cannot be read twice", request.Method, request.URL)
|
||||
}
|
||||
body, err := request.GetBody()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
attempt = request.Clone(request.Context())
|
||||
attempt.Body = body
|
||||
}
|
||||
tries++
|
||||
var err error
|
||||
response, err = t.base.RoundTrip(attempt)
|
||||
return err
|
||||
})
|
||||
return response, err
|
||||
}
|
||||
|
||||
// waiting is a client like c whose requests to the registry wait for it.
|
||||
func waiting(c *http.Client, address string) *http.Client {
|
||||
if _, already := c.Transport.(waitingTransport); already {
|
||||
return c
|
||||
}
|
||||
copied := *c
|
||||
base := c.Transport
|
||||
if base == nil {
|
||||
base = http.DefaultTransport
|
||||
}
|
||||
copied.Transport = waitingTransport{base: base, address: address}
|
||||
return &copied
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A registry held still for a while (novox/hq issue 457): the store's nightly collection stops it for
|
||||
// about a minute and a half, and a build in that window was failed, and its whole plan with it, for a
|
||||
// pause the mesh itself scheduled. A build waits it out — for a bound longer than the collection
|
||||
// holds it — says so in its log, and still fails, loudly, past the bound.
|
||||
|
||||
// heldStill is a registry address that refuses every connection until it starts answering after
|
||||
// pause, or never when pause is negative.
|
||||
func heldStill(t *testing.T, f *fakeRegistry, pause time.Duration) string {
|
||||
t.Helper()
|
||||
handler := f.serve(t).Config.Handler
|
||||
reserved, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
address := reserved.Addr().String()
|
||||
reserved.Close() // refused from here on: nothing listens
|
||||
if pause < 0 {
|
||||
return address
|
||||
}
|
||||
server := &http.Server{Handler: handler}
|
||||
go func() {
|
||||
time.Sleep(pause)
|
||||
l, err := net.Listen("tcp", address)
|
||||
if err != nil {
|
||||
t.Errorf("cannot answer at %s again: %v", address, err)
|
||||
return
|
||||
}
|
||||
_ = server.Serve(l)
|
||||
}()
|
||||
t.Cleanup(func() { _ = server.Close() })
|
||||
return address
|
||||
}
|
||||
|
||||
// saying collects what a build says, as the build machine's per-build Said does.
|
||||
func saying(t *testing.T) func() []string {
|
||||
t.Helper()
|
||||
var mu sync.Mutex
|
||||
var lines []string
|
||||
was := Said
|
||||
Said = func(step, message string) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
lines = append(lines, step+": "+message)
|
||||
}
|
||||
t.Cleanup(func() { Said = was })
|
||||
return func() []string {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return append([]string(nil), lines...)
|
||||
}
|
||||
}
|
||||
|
||||
// waitingFor shortens the bound and the first pause, so a test waits for milliseconds.
|
||||
func waitingFor(t *testing.T, bound time.Duration) {
|
||||
t.Helper()
|
||||
wasBound, wasFirst := registryWait, registryFirstPause
|
||||
registryWait, registryFirstPause = bound, 20*time.Millisecond
|
||||
t.Cleanup(func() { registryWait, registryFirstPause = wasBound, wasFirst })
|
||||
}
|
||||
|
||||
func TestABuildWaitsForARegistryHeldStillAndGoesOn(t *testing.T) {
|
||||
waitingFor(t, 5*time.Second)
|
||||
said := saying(t)
|
||||
f := &fakeRegistry{}
|
||||
r := Registry{Address: heldStill(t, f, 400*time.Millisecond)}
|
||||
body := []byte("a theme")
|
||||
sum := sha256.Sum256(body)
|
||||
digest := "sha256:" + hex.EncodeToString(sum[:])
|
||||
|
||||
if _, err := r.PublishArchive(context.Background(), "shell/config", body, digest); err != nil {
|
||||
t.Fatalf("a registry refusing for 400ms failed the build: %v", err)
|
||||
}
|
||||
if string(f.blobs[digest]) != "a theme" {
|
||||
t.Fatalf("the registry holds %q", f.blobs[digest])
|
||||
}
|
||||
log := strings.Join(said(), "\n")
|
||||
if !strings.Contains(log, "waits for the registry at "+r.Address) || !strings.Contains(log, "nightly collection") {
|
||||
t.Errorf("the build's log does not say it waited for the registry, and why:\n%s", log)
|
||||
}
|
||||
if !strings.Contains(log, "answers again") {
|
||||
t.Errorf("the build's log does not say the registry came back:\n%s", log)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABuildFailsLoudlyOnARegistryRefusingPastTheBound(t *testing.T) {
|
||||
waitingFor(t, 300*time.Millisecond)
|
||||
said := saying(t)
|
||||
f := &fakeRegistry{}
|
||||
r := Registry{Address: heldStill(t, f, -1)}
|
||||
body := []byte("a theme")
|
||||
sum := sha256.Sum256(body)
|
||||
digest := "sha256:" + hex.EncodeToString(sum[:])
|
||||
|
||||
started := time.Now()
|
||||
_, err := r.PublishArchive(context.Background(), "shell/config", body, digest)
|
||||
if err == nil {
|
||||
t.Fatal("a registry that never answered published the archive")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "refused every connection for") || !strings.Contains(err.Error(), "connection refused") {
|
||||
t.Errorf("the failure does not say it waited and was refused throughout: %v", err)
|
||||
}
|
||||
if waited := time.Since(started); waited < 300*time.Millisecond {
|
||||
t.Errorf("failed after %s, before the bound", waited)
|
||||
}
|
||||
if log := strings.Join(said(), "\n"); !strings.Contains(log, "waits for the registry") {
|
||||
t.Errorf("the build's log does not say it waited:\n%s", log)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnImagePushWaitsForARegistryHeldStill(t *testing.T) {
|
||||
waitingFor(t, 5*time.Second)
|
||||
said := saying(t)
|
||||
pushes := 0
|
||||
run := func(_ context.Context, _ string, name string, args ...string) (string, error) {
|
||||
switch args[0] {
|
||||
case "push":
|
||||
pushes++
|
||||
if pushes < 3 {
|
||||
return "", errorString("docker push: dial tcp 127.0.0.1:5000: connect: connection refused")
|
||||
}
|
||||
case "inspect":
|
||||
return "127.0.0.1:5000/m/server@sha256:abc\n", nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
r := Registry{Address: "127.0.0.1:5000", Run: run}
|
||||
if _, err := r.PublishImage(context.Background(), "local", "m/server"); err != nil {
|
||||
t.Fatalf("a push refused twice failed the build: %v", err)
|
||||
}
|
||||
if pushes != 3 {
|
||||
t.Errorf("pushed %d times, want 3", pushes)
|
||||
}
|
||||
if log := strings.Join(said(), "\n"); !strings.Contains(log, "waits for the registry") {
|
||||
t.Errorf("the build's log does not say it waited:\n%s", log)
|
||||
}
|
||||
}
|
||||
|
||||
type errorString string
|
||||
|
||||
func (e errorString) Error() string { return string(e) }
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
Captured 2026-10-11 from real `go test` runs (go1.27.1, linux/amd64) of a throwaway module whose
|
||||
packages fail each way a repository's own check fails: a test failing early in a run its own logs
|
||||
then fill (early: 600 log lines after the failures), a panic, a data race under -race, a build error,
|
||||
and one -race run of every package. Only the module's directory was rewritten to /src/cap.
|
||||
|
||||
They are the output shapes of novox/hq issue 460: mesh-controller #218's check printed its
|
||||
`--- FAIL:` line before the store-backed package's logs, which pushed it out of the report's tail.
|
||||
@@ -0,0 +1,5 @@
|
||||
# example.com/cap/broken
|
||||
broken/broken.go:3:28: undefined: undefinedThing
|
||||
FAIL example.com/cap/broken [build failed]
|
||||
ok example.com/cap/fine (cached)
|
||||
FAIL
|
||||
@@ -0,0 +1,31 @@
|
||||
==================
|
||||
WARNING: DATA RACE
|
||||
Read at 0x00c000018398 by goroutine 8:
|
||||
example.com/cap/race.TestACounterIsShared.func1()
|
||||
/src/cap/race/race_test.go:13 +0x7b
|
||||
|
||||
Previous write at 0x00c000018398 by goroutine 9:
|
||||
example.com/cap/race.TestACounterIsShared.func1()
|
||||
/src/cap/race/race_test.go:13 +0x8d
|
||||
|
||||
Goroutine 8 (running) created at:
|
||||
example.com/cap/race.TestACounterIsShared()
|
||||
/src/cap/race/race_test.go:13 +0x78
|
||||
testing.tRunner()
|
||||
/usr/lib/go/src/testing/testing.go:2193 +0x21c
|
||||
testing.(*T).Run.gowrap1()
|
||||
/usr/lib/go/src/testing/testing.go:2258 +0x38
|
||||
|
||||
Goroutine 9 (finished) created at:
|
||||
example.com/cap/race.TestACounterIsShared()
|
||||
/src/cap/race/race_test.go:13 +0x78
|
||||
testing.tRunner()
|
||||
/usr/lib/go/src/testing/testing.go:2193 +0x21c
|
||||
testing.(*T).Run.gowrap1()
|
||||
/usr/lib/go/src/testing/testing.go:2258 +0x38
|
||||
==================
|
||||
--- FAIL: TestACounterIsShared (0.00s)
|
||||
testing.go:1865: race detected during execution of test
|
||||
FAIL
|
||||
FAIL example.com/cap/race 0.008s
|
||||
FAIL
|
||||
@@ -0,0 +1,609 @@
|
||||
--- FAIL: TestTheEnvelopeIsPinned (0.00s)
|
||||
early_test.go:9: the envelope's subject is "mesh.a", not "mesh.b"
|
||||
early_test.go:10: a second line of the same failure
|
||||
--- FAIL: TestSubtests (0.00s)
|
||||
--- FAIL: TestSubtests/the_hub (0.00s)
|
||||
early_test.go:14: the hub does not compose
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d0
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d1
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d2
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d3
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d4
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d5
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d6
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d7
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d8
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d9
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d10
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d11
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d12
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d13
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d14
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d15
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d16
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d17
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d18
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d19
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d20
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d21
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d22
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d23
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d24
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d25
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d26
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d27
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d28
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d29
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d30
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d31
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d32
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d33
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d34
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d35
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d36
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d37
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d38
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d39
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d40
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d41
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d42
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d43
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d44
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d45
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d46
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d47
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d48
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d49
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d50
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d51
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d52
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d53
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d54
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d55
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d56
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d57
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d58
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d59
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d60
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d61
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d62
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d63
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d64
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d65
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d66
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d67
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d68
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d69
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d70
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d71
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d72
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d73
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d74
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d75
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d76
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d77
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d78
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d79
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d80
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d81
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d82
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d83
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d84
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d85
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d86
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d87
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d88
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d89
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d90
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d91
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d92
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d93
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d94
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d95
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d96
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d97
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d98
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d99
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d100
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d101
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d102
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d103
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d104
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d105
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d106
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d107
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d108
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d109
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d110
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d111
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d112
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d113
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d114
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d115
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d116
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d117
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d118
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d119
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d120
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d121
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d122
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d123
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d124
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d125
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d126
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d127
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d128
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d129
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d130
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d131
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d132
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d133
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d134
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d135
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d136
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d137
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d138
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d139
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d140
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d141
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d142
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d143
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d144
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d145
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d146
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d147
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d148
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d149
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d150
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d151
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d152
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d153
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d154
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d155
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d156
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d157
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d158
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d159
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d160
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d161
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d162
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d163
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d164
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d165
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d166
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d167
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d168
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d169
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d170
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d171
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d172
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d173
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d174
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d175
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d176
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d177
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d178
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d179
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d180
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d181
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d182
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d183
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d184
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d185
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d186
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d187
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d188
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d189
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d190
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d191
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d192
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d193
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d194
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d195
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d196
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d197
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d198
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d199
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d200
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d201
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d202
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d203
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d204
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d205
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d206
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d207
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d208
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d209
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d210
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d211
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d212
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d213
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d214
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d215
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d216
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d217
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d218
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d219
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d220
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d221
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d222
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d223
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d224
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d225
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d226
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d227
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d228
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d229
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d230
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d231
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d232
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d233
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d234
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d235
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d236
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d237
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d238
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d239
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d240
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d241
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d242
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d243
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d244
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d245
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d246
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d247
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d248
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d249
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d250
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d251
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d252
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d253
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d254
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d255
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d256
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d257
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d258
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d259
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d260
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d261
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d262
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d263
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d264
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d265
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d266
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d267
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d268
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d269
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d270
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d271
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d272
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d273
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d274
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d275
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d276
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d277
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d278
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d279
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d280
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d281
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d282
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d283
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d284
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d285
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d286
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d287
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d288
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d289
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d290
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d291
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d292
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d293
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d294
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d295
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d296
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d297
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d298
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d299
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d300
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d301
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d302
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d303
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d304
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d305
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d306
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d307
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d308
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d309
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d310
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d311
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d312
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d313
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d314
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d315
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d316
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d317
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d318
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d319
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d320
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d321
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d322
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d323
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d324
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d325
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d326
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d327
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d328
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d329
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d330
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d331
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d332
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d333
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d334
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d335
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d336
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d337
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d338
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d339
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d340
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d341
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d342
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d343
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d344
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d345
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d346
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d347
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d348
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d349
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d350
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d351
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d352
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d353
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d354
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d355
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d356
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d357
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d358
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d359
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d360
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d361
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d362
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d363
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d364
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d365
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d366
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d367
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d368
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d369
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d370
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d371
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d372
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d373
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d374
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d375
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d376
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d377
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d378
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d379
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d380
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d381
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d382
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d383
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d384
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d385
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d386
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d387
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d388
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d389
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d390
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d391
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d392
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d393
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d394
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d395
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d396
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d397
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d398
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d399
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d400
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d401
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d402
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d403
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d404
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d405
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d406
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d407
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d408
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d409
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d410
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d411
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d412
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d413
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d414
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d415
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d416
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d417
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d418
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d419
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d420
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d421
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d422
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d423
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d424
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d425
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d426
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d427
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d428
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d429
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d430
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d431
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d432
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d433
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d434
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d435
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d436
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d437
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d438
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d439
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d440
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d441
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d442
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d443
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d444
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d445
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d446
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d447
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d448
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d449
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d450
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d451
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d452
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d453
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d454
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d455
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d456
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d457
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d458
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d459
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d460
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d461
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d462
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d463
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d464
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d465
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d466
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d467
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d468
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d469
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d470
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d471
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d472
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d473
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d474
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d475
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d476
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d477
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d478
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d479
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d480
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d481
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d482
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d483
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d484
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d485
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d486
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d487
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d488
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d489
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d490
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d491
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d492
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d493
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d494
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d495
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d496
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d497
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d498
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d499
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d500
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d501
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d502
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d503
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d504
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d505
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d506
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d507
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d508
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d509
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d510
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d511
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d512
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d513
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d514
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d515
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d516
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d517
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d518
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d519
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d520
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d521
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d522
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d523
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d524
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d525
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d526
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d527
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d528
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d529
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d530
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d531
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d532
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d533
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d534
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d535
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d536
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d537
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d538
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d539
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d540
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d541
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d542
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d543
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d544
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d545
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d546
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d547
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d548
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d549
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d550
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d551
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d552
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d553
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d554
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d555
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d556
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d557
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d558
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d559
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d560
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d561
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d562
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d563
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d564
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d565
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d566
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d567
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d568
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d569
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d570
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d571
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d572
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d573
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d574
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d575
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d576
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d577
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d578
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d579
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d580
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d581
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d582
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d583
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d584
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d585
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d586
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d587
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d588
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d589
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d590
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d591
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d592
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d593
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d594
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d595
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d596
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d597
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d598
|
||||
2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d599
|
||||
FAIL
|
||||
FAIL example.com/cap/early 0.003s
|
||||
FAIL
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
--- FAIL: TestAMapIsNil (0.00s)
|
||||
panic: assignment to entry in nil map [recovered, repanicked]
|
||||
|
||||
goroutine 18 [running]:
|
||||
testing.tRunner.func1.2({0x6b6810, 0x6ee000})
|
||||
/usr/lib/go/src/testing/testing.go:2123 +0x232
|
||||
testing.tRunner.func1()
|
||||
/usr/lib/go/src/testing/testing.go:2126 +0x329
|
||||
panic({0x6b6810?, 0x6ee000?})
|
||||
/usr/lib/go/src/runtime/panic.go:859 +0x125
|
||||
example.com/cap/panics.TestAMapIsNil(0x1e97d59dc248?)
|
||||
/src/cap/panics/panics_test.go:7 +0x28
|
||||
testing.tRunner(0x1e97d59dc248, 0x6d41f8)
|
||||
/usr/lib/go/src/testing/testing.go:2193 +0xea
|
||||
created by testing.(*T).Run in goroutine 1
|
||||
/usr/lib/go/src/testing/testing.go:2258 +0x4d4
|
||||
FAIL example.com/cap/panics 0.004s
|
||||
FAIL
|
||||
@@ -0,0 +1,660 @@
|
||||
# example.com/cap/broken
|
||||
broken/broken.go:3:28: undefined: undefinedThing
|
||||
FAIL example.com/cap/broken [build failed]
|
||||
--- FAIL: TestTheEnvelopeIsPinned (0.00s)
|
||||
early_test.go:9: the envelope's subject is "mesh.a", not "mesh.b"
|
||||
early_test.go:10: a second line of the same failure
|
||||
--- FAIL: TestSubtests (0.00s)
|
||||
--- FAIL: TestSubtests/the_hub (0.00s)
|
||||
early_test.go:14: the hub does not compose
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d0
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d1
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d2
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d3
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d4
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d5
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d6
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d7
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d8
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d9
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d10
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d11
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d12
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d13
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d14
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d15
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d16
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d17
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d18
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d19
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d20
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d21
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d22
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d23
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d24
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d25
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d26
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d27
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d28
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d29
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d30
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d31
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d32
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d33
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d34
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d35
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d36
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d37
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d38
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d39
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d40
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d41
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d42
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d43
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d44
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d45
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d46
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d47
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d48
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d49
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d50
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d51
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d52
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d53
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d54
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d55
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d56
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d57
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d58
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d59
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d60
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d61
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d62
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d63
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d64
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d65
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d66
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d67
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d68
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d69
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d70
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d71
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d72
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d73
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d74
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d75
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d76
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d77
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d78
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d79
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d80
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d81
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d82
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d83
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d84
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d85
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d86
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d87
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d88
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d89
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d90
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d91
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d92
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d93
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d94
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d95
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d96
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d97
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d98
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d99
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d100
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d101
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d102
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d103
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d104
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d105
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d106
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d107
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d108
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d109
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d110
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d111
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d112
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d113
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d114
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d115
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d116
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d117
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d118
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d119
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d120
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d121
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d122
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d123
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d124
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d125
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d126
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d127
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d128
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d129
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d130
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d131
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d132
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d133
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d134
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d135
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d136
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d137
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d138
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d139
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d140
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d141
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d142
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d143
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d144
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d145
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d146
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d147
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d148
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d149
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d150
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d151
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d152
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d153
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d154
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d155
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d156
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d157
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d158
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d159
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d160
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d161
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d162
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d163
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d164
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d165
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d166
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d167
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d168
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d169
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d170
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d171
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d172
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d173
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d174
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d175
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d176
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d177
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d178
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d179
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d180
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d181
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d182
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d183
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d184
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d185
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d186
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d187
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d188
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d189
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d190
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d191
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d192
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d193
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d194
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d195
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d196
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d197
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d198
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d199
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d200
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d201
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d202
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d203
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d204
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d205
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d206
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d207
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d208
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d209
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d210
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d211
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d212
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d213
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d214
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d215
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d216
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d217
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d218
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d219
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d220
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d221
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d222
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d223
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d224
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d225
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d226
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d227
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d228
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d229
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d230
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d231
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d232
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d233
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d234
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d235
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d236
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d237
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d238
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d239
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d240
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d241
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d242
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d243
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d244
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d245
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d246
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d247
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d248
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d249
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d250
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d251
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d252
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d253
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d254
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d255
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d256
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d257
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d258
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d259
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d260
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d261
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d262
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d263
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d264
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d265
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d266
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d267
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d268
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d269
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d270
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d271
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d272
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d273
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d274
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d275
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d276
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d277
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d278
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d279
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d280
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d281
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d282
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d283
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d284
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d285
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d286
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d287
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d288
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d289
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d290
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d291
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d292
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d293
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d294
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d295
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d296
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d297
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d298
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d299
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d300
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d301
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d302
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d303
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d304
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d305
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d306
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d307
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d308
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d309
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d310
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d311
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d312
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d313
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d314
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d315
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d316
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d317
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d318
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d319
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d320
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d321
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d322
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d323
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d324
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d325
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d326
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d327
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d328
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d329
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d330
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d331
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d332
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d333
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d334
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d335
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d336
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d337
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d338
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d339
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d340
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d341
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d342
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d343
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d344
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d345
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d346
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d347
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d348
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d349
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d350
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d351
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d352
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d353
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d354
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d355
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d356
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d357
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d358
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d359
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d360
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d361
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d362
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d363
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d364
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d365
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d366
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d367
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d368
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d369
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d370
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d371
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d372
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d373
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d374
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d375
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d376
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d377
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d378
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d379
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d380
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d381
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d382
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d383
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d384
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d385
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d386
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d387
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d388
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d389
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d390
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d391
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d392
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d393
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d394
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d395
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d396
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d397
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d398
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d399
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d400
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d401
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d402
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d403
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d404
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d405
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d406
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d407
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d408
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d409
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d410
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d411
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d412
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d413
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d414
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d415
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d416
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d417
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d418
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d419
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d420
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d421
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d422
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d423
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d424
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d425
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d426
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d427
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d428
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d429
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d430
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d431
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d432
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d433
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d434
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d435
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d436
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d437
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d438
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d439
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d440
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d441
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d442
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d443
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d444
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d445
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d446
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d447
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d448
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d449
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d450
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d451
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d452
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d453
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d454
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d455
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d456
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d457
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d458
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d459
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d460
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d461
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d462
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d463
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d464
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d465
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d466
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d467
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d468
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d469
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d470
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d471
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d472
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d473
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d474
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d475
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d476
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d477
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d478
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d479
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d480
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d481
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d482
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d483
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d484
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d485
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d486
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d487
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d488
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d489
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d490
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d491
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d492
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d493
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d494
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d495
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d496
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d497
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d498
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d499
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d500
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d501
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d502
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d503
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d504
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d505
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d506
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d507
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d508
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d509
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d510
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d511
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d512
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d513
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d514
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d515
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d516
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d517
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d518
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d519
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d520
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d521
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d522
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d523
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d524
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d525
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d526
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d527
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d528
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d529
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d530
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d531
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d532
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d533
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d534
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d535
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d536
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d537
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d538
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d539
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d540
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d541
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d542
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d543
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d544
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d545
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d546
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d547
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d548
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d549
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d550
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d551
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d552
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d553
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d554
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d555
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d556
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d557
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d558
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d559
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d560
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d561
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d562
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d563
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d564
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d565
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d566
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d567
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d568
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d569
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d570
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d571
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d572
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d573
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d574
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d575
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d576
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d577
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d578
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d579
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d580
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d581
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d582
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d583
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d584
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d585
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d586
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d587
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d588
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d589
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d590
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d591
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d592
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d593
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d594
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d595
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d596
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d597
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d598
|
||||
2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d599
|
||||
FAIL
|
||||
FAIL example.com/cap/early 0.013s
|
||||
ok example.com/cap/fine (cached)
|
||||
--- FAIL: TestAMapIsNil (0.00s)
|
||||
panic: assignment to entry in nil map [recovered, repanicked]
|
||||
|
||||
goroutine 35 [running]:
|
||||
testing.tRunner.func1.2({0x7c6e60, 0x806640})
|
||||
/usr/lib/go/src/testing/testing.go:2123 +0x419
|
||||
testing.tRunner.func1()
|
||||
/usr/lib/go/src/testing/testing.go:2126 +0x65f
|
||||
panic({0x7c6e60?, 0x806640?})
|
||||
/usr/lib/go/src/runtime/panic.go:859 +0x125
|
||||
example.com/cap/panics.TestAMapIsNil(0xc0001d6248?)
|
||||
/src/cap/panics/panics_test.go:7 +0x32
|
||||
testing.tRunner(0xc0001d6248, 0x7e5f78)
|
||||
/usr/lib/go/src/testing/testing.go:2193 +0x21d
|
||||
created by testing.(*T).Run in goroutine 1
|
||||
/usr/lib/go/src/testing/testing.go:2258 +0xb13
|
||||
FAIL example.com/cap/panics 0.011s
|
||||
==================
|
||||
WARNING: DATA RACE
|
||||
Read at 0x00c000018398 by goroutine 9:
|
||||
example.com/cap/race.TestACounterIsShared.func1()
|
||||
/src/cap/race/race_test.go:13 +0x7b
|
||||
|
||||
Previous write at 0x00c000018398 by goroutine 8:
|
||||
example.com/cap/race.TestACounterIsShared.func1()
|
||||
/src/cap/race/race_test.go:13 +0x8d
|
||||
|
||||
Goroutine 9 (running) created at:
|
||||
example.com/cap/race.TestACounterIsShared()
|
||||
/src/cap/race/race_test.go:13 +0x78
|
||||
testing.tRunner()
|
||||
/usr/lib/go/src/testing/testing.go:2193 +0x21c
|
||||
testing.(*T).Run.gowrap1()
|
||||
/usr/lib/go/src/testing/testing.go:2258 +0x38
|
||||
|
||||
Goroutine 8 (finished) created at:
|
||||
example.com/cap/race.TestACounterIsShared()
|
||||
/src/cap/race/race_test.go:13 +0x78
|
||||
testing.tRunner()
|
||||
/usr/lib/go/src/testing/testing.go:2193 +0x21c
|
||||
testing.(*T).Run.gowrap1()
|
||||
/usr/lib/go/src/testing/testing.go:2258 +0x38
|
||||
==================
|
||||
--- FAIL: TestACounterIsShared (0.00s)
|
||||
testing.go:1865: race detected during execution of test
|
||||
FAIL
|
||||
FAIL example.com/cap/race 0.009s
|
||||
FAIL
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/beside"
|
||||
)
|
||||
|
||||
// **The word does not come back through a manifest** (novox/hq ADR 0131). A module that wants
|
||||
@@ -28,12 +30,12 @@ func TestAManifestRequiringAmqpIsRefused(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// And the catalogue as checked out beside this repository names it nowhere — the three modules that
|
||||
// did are removed under design 28 task 5.4, not converted.
|
||||
// And the catalogue (internal/beside) names it nowhere — the three modules that did are removed under
|
||||
// design 28 task 5.4, not converted.
|
||||
func TestNoCatalogueManifestNamesAmqp(t *testing.T) {
|
||||
modules, err := filepath.Glob("../../../mesh-catalog/modules/*/module.json")
|
||||
modules, err := filepath.Glob(filepath.Join(beside.Catalogue(t), "*", "module.json"))
|
||||
if err != nil || len(modules) == 0 {
|
||||
t.Skip("the catalogue is not checked out beside this repository")
|
||||
t.Fatalf("no manifests in the catalogue, so this proved nothing: %v", err)
|
||||
}
|
||||
for _, path := range modules {
|
||||
raw, err := os.ReadFile(path)
|
||||
|
||||
@@ -2,8 +2,11 @@ package catalogue
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/beside"
|
||||
)
|
||||
|
||||
// The module holding mesh-broker is the bus, and its account is granted the bus's snapshot API and
|
||||
@@ -34,9 +37,9 @@ func TestTheBussAccountSaysNothingOnTheBus(t *testing.T) {
|
||||
// dump into its snapshots, it has the account the dump runs as, and the dump runs the snapshot
|
||||
// program in the bus's own container.
|
||||
func TestTheCataloguesBusIsBackedUpBySnapshot(t *testing.T) {
|
||||
raw, err := os.ReadFile("../../../mesh-catalog/modules/nats/module.json")
|
||||
raw, err := os.ReadFile(filepath.Join(beside.Catalogue(t), "nats", "module.json"))
|
||||
if err != nil {
|
||||
t.Skip("the catalogue is not checked out beside this repository")
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := ParseManifest(raw)
|
||||
if err != nil {
|
||||
|
||||
@@ -5,26 +5,21 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/beside"
|
||||
)
|
||||
|
||||
// TestEveryCatalogueManifestParses runs the real catalogue through the real gate.
|
||||
//
|
||||
// Not a fixture: the point is whether the manifests as written are accepted by the control plane that
|
||||
// will read them, and a copy of one manifest proves nothing about the other seventy-one.
|
||||
// catalogueRoot is the catalogue these checks run over: MESH_CATALOGUE when set, else the checkout
|
||||
// beside this one, the way the main layout has it. A check that only ran when somebody remembered a
|
||||
// variable was a check nobody ran (novox/hq issue 134, 2026-09-30); it skips only when there is no
|
||||
// catalogue to be found at all.
|
||||
// catalogueRoot is the catalogue these checks run over: in a merge check the clone the build seat put
|
||||
// beside this one, elsewhere the copy captured in testdata/beside (internal/beside). A check that only
|
||||
// ran when somebody remembered a variable was a check nobody ran (novox/hq issue 134), and one that read
|
||||
// whatever checkout sat beside judged the machine, not the change (novox/hq issue 432): it never skips.
|
||||
func catalogueRoot(t *testing.T) string {
|
||||
t.Helper()
|
||||
if root := os.Getenv("MESH_CATALOGUE"); root != "" {
|
||||
return root
|
||||
}
|
||||
sibling := filepath.Join("..", "..", "..", "mesh-catalog")
|
||||
if _, err := os.Stat(filepath.Join(sibling, "modules")); err != nil {
|
||||
t.Skip("no catalogue beside this checkout and MESH_CATALOGUE unset")
|
||||
}
|
||||
return sibling
|
||||
return beside.Dir(t, "mesh-catalog")
|
||||
}
|
||||
|
||||
func TestEveryCatalogueManifestParses(t *testing.T) {
|
||||
|
||||
@@ -2,8 +2,11 @@ package catalogue
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/beside"
|
||||
)
|
||||
|
||||
// **A machine trusts the mesh's authority because a module put its root there** (novox/hq ADR
|
||||
@@ -16,9 +19,9 @@ import (
|
||||
// itself — a plain client trusting an internal name on a machine holding this, and failing on one
|
||||
// that does not — is the lab's, and cannot be had here.
|
||||
func TestCaTrustRendersTheAuthorityItWasBoundTo(t *testing.T) {
|
||||
raw, err := os.ReadFile("../../../mesh-catalog/modules/ca-trust/module.json")
|
||||
raw, err := os.ReadFile(filepath.Join(beside.Catalogue(t), "ca-trust", "module.json"))
|
||||
if err != nil {
|
||||
t.Skipf("the catalogue is not beside this checkout: %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := ParseManifest(raw)
|
||||
if err != nil {
|
||||
|
||||
@@ -606,7 +606,18 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
"mode": "0600",
|
||||
})
|
||||
}
|
||||
for _, name := range sortedKeys(m.OwnSecrets) {
|
||||
// **A secret family's members, as given** (novox/hq ADR 0283): one file each at the family's path, and
|
||||
// nothing for a member not given — the mesh never makes one, and the module says it waits for it.
|
||||
for _, name := range sortedKeys(with.Needed[m.Module]) {
|
||||
own, family, ok := m.OwnSecrets.Lookup(name)
|
||||
if !ok || family == "" || with.Needed[m.Module][name] == "" {
|
||||
continue
|
||||
}
|
||||
first = append(first, ownedBy(m.SecretsOwner, map[string]any{
|
||||
"id": NeedID(name), "type": "file", "path": own.Path, "sealed": with.Needed[m.Module][name],
|
||||
}))
|
||||
}
|
||||
for _, name := range sortedKeys(m.OwnSecrets.Plain()) {
|
||||
sealed := with.Needed[m.Module][name]
|
||||
if sealed == "" && with.Foreseen[m.Module][name] {
|
||||
// Not made, and the next send makes it: composed with a stand-in so that whatever
|
||||
@@ -900,6 +911,15 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
// **And every placeholder no pass fills where it stands is refused** (novox/hq issue 231):
|
||||
// judged here, over the definition as written and before any pass, by the function the
|
||||
// catalogue check runs — so a manifest registered by an older binary is judged too, and
|
||||
// what a setting or a binding later puts into a file is its software's text, never swept.
|
||||
for _, own := range m.Resources {
|
||||
if problems := unconsumedPlaceholders(m.Module, own); len(problems) > 0 {
|
||||
return nil, fmt.Errorf("%s", problems[0])
|
||||
}
|
||||
}
|
||||
|
||||
// Which of this module's files carry a secret, for the rule that a container may not read
|
||||
// one of them as its environment without saying so (ADR 0086, issue 041).
|
||||
@@ -1056,7 +1076,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
// credential the mesh had replaced, because its manifest restarted it on its
|
||||
// environment file and nobody had thought to name the credential too. Composed here so
|
||||
// no manifest has to say it, for a container or a daemon that names the secret's path.
|
||||
if reads := secretsReadBy(copied, m); len(reads) > 0 {
|
||||
if reads := secretsReadBy(copied, m, with.Needed[m.Module]); len(reads) > 0 {
|
||||
copied["restart-on"] = withRestartOn(copied["restart-on"], reads)
|
||||
}
|
||||
// **A version prepares its state before it runs** (novox/hq ADR 0135). Derived from the
|
||||
@@ -2372,7 +2392,12 @@ func portOfEndpoint(values map[string]any, ports map[string]int) {
|
||||
// — named in its volumes, its environment or its env-files by the secret's placed path — as
|
||||
// restart-on ids. Nothing for other shapes, and nothing for a scheduled or run-once process, which
|
||||
// the host refuses a restart-on for (it runs again anyway, and reads the file afresh).
|
||||
func secretsReadBy(resource map[string]any, m Manifest) []string {
|
||||
//
|
||||
// **A member of a secret family given here is read the same way** (novox/hq issue 405, ADR 0283 decision 6):
|
||||
// it is an own secret placed at its own path, and a container that mounted it would keep the value it
|
||||
// started with when the operator gives it again. given is the module's own secrets sealed to this
|
||||
// machine; a member not given is no file, so nothing restarts on it.
|
||||
func secretsReadBy(resource map[string]any, m Manifest, given map[string]string) []string {
|
||||
kind := fmt.Sprint(resource["type"])
|
||||
if kind != "container" && kind != "process" {
|
||||
return nil
|
||||
@@ -2384,9 +2409,18 @@ func secretsReadBy(resource map[string]any, m Manifest) []string {
|
||||
for _, key := range []string{"volumes", "env", "env-file"} {
|
||||
mentioned = append(mentioned, stringsIn(resource[key])...)
|
||||
}
|
||||
paths := map[string]string{}
|
||||
for name, own := range m.OwnSecrets.Plain() {
|
||||
paths[name] = own.Path
|
||||
}
|
||||
for name, sealed := range given {
|
||||
if own, family, ok := m.OwnSecrets.Lookup(name); ok && family != "" && sealed != "" {
|
||||
paths[name] = own.Path
|
||||
}
|
||||
}
|
||||
var out []string
|
||||
for _, name := range sortedKeys(m.OwnSecrets) {
|
||||
path := m.OwnSecrets[name].Path
|
||||
for _, name := range sortedKeys(paths) {
|
||||
path := paths[name]
|
||||
if path == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -18,6 +18,15 @@ func deliveryVerbs() []Verb {
|
||||
Input: schema(map[string]string{"state": "one state, e.g. delivering or held",
|
||||
"repository": "owner/repository", "group": "a group's id (its branch name)",
|
||||
"all": "\"true\": the final ones of the last thirty days too"}, nil, "all")},
|
||||
// Delivery time against the delivery budgets (novox/hq ADR 0282): what probe D16 reads, optional until its
|
||||
// holder serves it.
|
||||
{Name: "times", Description: "Delivery time: from a merge to every machine of its walk running the build. " +
|
||||
"Each class's delivery budget (a leaf module five minutes, a core module ten), the last days' median and " +
|
||||
"worst, how many within and over, the newest delivery of each class, every delivery over its budget with its " +
|
||||
"longest phase, and the delivered ones whose time is not known. Given a delivery's id, its time phase by phase.",
|
||||
Input: schema(map[string]string{"days": "how many days back (default 7)", "id": "one delivery's id: its phases"}, nil),
|
||||
// Optional until mesh-delivery serves it: its holder lives in the catalogue (design 33 §7).
|
||||
Optional: true},
|
||||
{Name: "show", Description: "One delivery or group whole: its delivery plan (what it builds, what each " +
|
||||
"machine receives, what is not an ordinary send), every transition with when and why, the machine " +
|
||||
"steps of its walk, its group and its order.",
|
||||
|
||||
@@ -97,3 +97,39 @@ func TestTheDeliverySeatPromisesRetireHistoryOptionally(t *testing.T) {
|
||||
t.Fatalf("a holder serving retire-history: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The seat says delivery times (novox/hq ADR 0282, issue 382): `times`, over some days or for one delivery. Added
|
||||
// after the holder shipped, it is optional, so the holder that does not serve it yet still holds the seat, and one
|
||||
// that does is not refused for a verb the seat lacks.
|
||||
func TestTheDeliverySeatPromisesTimesOptionally(t *testing.T) {
|
||||
seat, _ := SeatNamed(DeliverySeat)
|
||||
var times *Verb
|
||||
for i := range seat.Serves {
|
||||
if seat.Serves[i].Name == "times" {
|
||||
times = &seat.Serves[i]
|
||||
}
|
||||
}
|
||||
if times == nil || !times.Optional {
|
||||
t.Fatalf("the delivery seat promises %v, times optionally", VerbNames(seat.Serves))
|
||||
}
|
||||
props, _ := times.Input["properties"].(map[string]any)
|
||||
for _, arg := range []string{"days", "id"} {
|
||||
if _, has := props[arg]; !has {
|
||||
t.Errorf("times takes no %q", arg)
|
||||
}
|
||||
}
|
||||
var without []string
|
||||
for _, v := range VerbNames(seat.Serves) {
|
||||
if v != "times" {
|
||||
without = append(without, v)
|
||||
}
|
||||
}
|
||||
m := Manifest{Module: "mesh-delivery", Claims: []Claim{{Name: DeliverySeat, Scope: ScopeMesh, Serves: without}}}
|
||||
if err := CanHold(m, seat); err != nil {
|
||||
t.Fatalf("a holder without times yet: %v", err)
|
||||
}
|
||||
m.Claims[0].Serves = VerbNames(seat.Serves)
|
||||
if err := CanHold(m, seat); err != nil {
|
||||
t.Fatalf("a holder serving times: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -226,6 +226,9 @@ func (m Manifest) contributionPlaceholderProblems() []string {
|
||||
for _, r := range m.Resources {
|
||||
problems = append(problems, placeholderProblems(m, r)...)
|
||||
problems = append(problems, seatPlaceholderProblems(m, r)...)
|
||||
// And every placeholder no pass fills where it stands: a misspelt namespace, a key its
|
||||
// namespace cannot take, or a field its pass does not read (novox/hq issue 231).
|
||||
problems = append(problems, unconsumedPlaceholders(m.Module, r)...)
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
@@ -134,27 +134,61 @@ func TestThePOSIXEnvironmentSourcedTwiceLeavesPATHAsOnce(t *testing.T) {
|
||||
|
||||
// The environment.d rendering, read by the service manager's own generator where this machine has
|
||||
// one — the same reader an account's user manager runs, so the PATH it composes is the one asserted.
|
||||
//
|
||||
// The generator also reads the machine's own environment.d (/etc, /run, /usr/lib, /usr/local/lib), and
|
||||
// no option points it elsewhere: a desktop whose snapd appends its bin directory failed this, on main,
|
||||
// for a file the mesh never wrote (novox/hq issue 432). So the generator is run twice, once without the
|
||||
// mesh's file, and what the machine's own files make of PATH and set is held out of the verdict.
|
||||
//
|
||||
// A machine without the generator — the build seat's toolchain image holds no systemd — cannot judge
|
||||
// this and says so: there the rendering is held byte for byte by
|
||||
// TestTheEnvironmentRendersForTheServiceManagerByteForByte, and this reading only where systemd runs.
|
||||
func TestTheServiceManagerReadsTheSystemdRenderingAsMeant(t *testing.T) {
|
||||
generator := "/usr/lib/systemd/user-environment-generators/30-systemd-environment-d-generator"
|
||||
if _, err := os.Stat(generator); err != nil {
|
||||
t.Skip("no environment.d generator on this machine")
|
||||
t.Skip("NOT JUDGED: no environment.d generator on this machine, so the service manager's reading " +
|
||||
"of the rendering is judged only where systemd runs; the rendering itself is held byte for byte " +
|
||||
"by TestTheEnvironmentRendersForTheServiceManagerByteForByte")
|
||||
}
|
||||
config := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(config, "environment.d"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
const base = "/usr/bin:/bin"
|
||||
read := func(conf string) map[string]string {
|
||||
t.Helper()
|
||||
config := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(config, "environment.d"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if conf != "" {
|
||||
if err := os.WriteFile(filepath.Join(config, "environment.d", "50-mesh.conf"), []byte(conf), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
cmd := exec.Command(generator)
|
||||
cmd.Env = []string{"PATH=" + base, "HOME=" + config, "XDG_CONFIG_HOME=" + config}
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("%v\n%s", err, out)
|
||||
}
|
||||
vars := map[string]string{}
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
if k, v, ok := strings.Cut(line, "="); ok {
|
||||
vars[k] = v
|
||||
}
|
||||
}
|
||||
return vars
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(config, "environment.d", "50-mesh.conf"), []byte(composedSystemd), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
machine, read50 := read(""), read(composedSystemd)
|
||||
|
||||
// What the machine's own files do to PATH: nothing, or append to it. One that replaces or prepends
|
||||
// leaves nothing this test can say about the mesh's file, and says so rather than guess.
|
||||
added, ok := strings.CutPrefix(machine["PATH"], base)
|
||||
if machine["PATH"] != "" && !ok {
|
||||
t.Skipf("NOT JUDGED: this machine's own environment.d sets PATH to %s, not %s with something after it, so "+
|
||||
"what the mesh's file adds cannot be told apart from it", machine["PATH"], base)
|
||||
}
|
||||
cmd := exec.Command(generator)
|
||||
cmd.Env = []string{"PATH=/usr/bin:/bin", "HOME=" + config, "XDG_CONFIG_HOME=" + config}
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("%v\n%s", err, out)
|
||||
}
|
||||
want := "PATH=/home/op/go/bin:/usr/local/go/bin:/home/op/.local/bin:/home/op/bin:/usr/bin:/bin:/opt/agent/bin:/opt/scripts"
|
||||
if !strings.Contains(string(out), want+"\n") || !strings.Contains(string(out), "GOPATH=/home/op/go\n") {
|
||||
t.Fatalf("the service manager read\n%s", out)
|
||||
want := "/home/op/go/bin:/usr/local/go/bin:/home/op/.local/bin:/home/op/bin:/usr/bin:/bin:/opt/agent/bin:/opt/scripts" + added
|
||||
if read50["PATH"] != want || read50["GOPATH"] != "/home/op/go" {
|
||||
t.Fatalf("the service manager read PATH=%s GOPATH=%s, not PATH=%s GOPATH=/home/op/go (the machine's own "+
|
||||
"files add %q to PATH)", read50["PATH"], read50["GOPATH"], want, added)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,19 +4,24 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/beside"
|
||||
)
|
||||
|
||||
// The catalogue's foundation modules as they are, parsed by the real parser (novox/hq ADR 0100):
|
||||
// The catalogue's foundation modules as they are — in a merge check the catalogue cloned beside it,
|
||||
// elsewhere the copy captured in testdata/beside (internal/beside, novox/hq issue 432) — parsed by the
|
||||
// real parser (novox/hq ADR 0100):
|
||||
// the store and the broker say which of their ports the mesh guards on an adopted node, and the
|
||||
// filter module loads its table through a unit of its own whose stop deletes only that table.
|
||||
func catalogueManifest(t *testing.T, module string) Manifest {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile("../../../mesh-catalog/modules/" + module + "/module.json")
|
||||
raw, err := os.ReadFile(filepath.Join(beside.Catalogue(t), module, "module.json"))
|
||||
if err != nil {
|
||||
t.Skipf("the catalogue is not beside this checkout: %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := ParseManifest(raw)
|
||||
if err != nil {
|
||||
@@ -125,8 +130,11 @@ func TestTheForgesPortIsGivenLikeAnyOtherProvidersPort(t *testing.T) {
|
||||
// seat's holder the answer when more than one module provides it — so a carried copy would be a
|
||||
// second answer to the same question, free to drift from the first. Asserted gone, not merely
|
||||
// unused.
|
||||
//
|
||||
// The builder is the build-agent on every machine since novox/hq ADR 0190; this read the retired
|
||||
// `builder` and, finding no manifest, skipped unseen from then until novox/hq issue 432.
|
||||
func TestTheBuilderRequiresTheRegistryTheNpmSeatDelivers(t *testing.T) {
|
||||
builder := catalogueManifest(t, "builder")
|
||||
builder := catalogueManifest(t, "build-agent")
|
||||
seat, _ := SeatNamed("npm-package-registry")
|
||||
var requires bool
|
||||
for _, r := range builder.Requires {
|
||||
|
||||
@@ -1942,6 +1942,37 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
||||
problems = append(problems, whileStoppedProblems(m, r, hasSchedule(r))...)
|
||||
}
|
||||
for name, own := range m.OwnSecrets {
|
||||
// **A family is issued outside the mesh, and lands one file per member** (novox/hq ADR 0283): the mesh
|
||||
// never makes a member, so a family the mesh may make would be one nothing ever fills.
|
||||
if IsFamily(name) {
|
||||
if !memberRest.MatchString(strings.TrimSuffix(FamilyPrefix(name), "-")) {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s declares the secret family %q, whose prefix is not a name", m.Module, name))
|
||||
}
|
||||
if own.IssuedBy != IssuedOutside {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s declares the secret family %q without \"issued-by\": %q; the mesh never makes a "+
|
||||
"member of a family, so only a party outside the mesh can fill one (novox/hq ADR 0283)",
|
||||
m.Module, name, IssuedOutside))
|
||||
}
|
||||
if strings.Count(own.Path, "*") != 1 {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s keeps the secret family %q at %q, which does not hold exactly one *: each member lands "+
|
||||
"where the * is replaced by its name (novox/hq ADR 0283)", m.Module, name, own.Path))
|
||||
}
|
||||
for other := range m.OwnSecrets {
|
||||
if other != name && !IsFamily(other) {
|
||||
if rest := strings.TrimPrefix(other, FamilyPrefix(name)); rest != other && memberRest.MatchString(rest) {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s declares the secret %q, which is also a member of its family %q — a name names one",
|
||||
m.Module, other, name))
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if strings.Contains(name, "*") {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s declares the secret %q: a * only ends a family's name, as \"<prefix>-*\"", m.Module, name))
|
||||
}
|
||||
if !placedOrAbsolute(own.Path) {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s needs %q at %q, which is neither an absolute path nor a placed one", m.Module, name, own.Path))
|
||||
@@ -2549,6 +2580,69 @@ func (o OwnSecrets) MarshalJSON() ([]byte, error) {
|
||||
return json.Marshal(entries)
|
||||
}
|
||||
|
||||
// A secret family (novox/hq ADR 0283): an own secret declared under a name ending in FamilySuffix is one
|
||||
// member per part the module's settings name — `smb-password-*` holds `smb-password-games`,
|
||||
// `smb-password-library` — each given by its full name, placed at the family's path with its one `*` replaced
|
||||
// by what follows the prefix. The mesh never makes a member: a family is issued outside the mesh, and a member
|
||||
// not given is no file.
|
||||
const FamilySuffix = "-*"
|
||||
|
||||
// memberRest is what may follow a family's prefix: a name's characters.
|
||||
var memberRest = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`)
|
||||
|
||||
// IsFamily says an own secret's declared name is a family's.
|
||||
func IsFamily(name string) bool { return strings.HasSuffix(name, FamilySuffix) }
|
||||
|
||||
// FamilyPrefix is what every member of a family begins with: the declared name without its `*`.
|
||||
func FamilyPrefix(family string) string { return strings.TrimSuffix(family, "*") }
|
||||
|
||||
// Lookup resolves an own secret by the name it is given under: declared by that name, or a member of a family
|
||||
// (the longest prefix that fits), with the family's path filled for the member. family is the family's
|
||||
// declared name, or "" for a secret declared by name.
|
||||
func (o OwnSecrets) Lookup(name string) (s OwnSecret, family string, ok bool) {
|
||||
if s, ok := o[name]; ok && !IsFamily(name) {
|
||||
return s, "", true
|
||||
}
|
||||
for declared, f := range o {
|
||||
if !IsFamily(declared) {
|
||||
continue
|
||||
}
|
||||
prefix := FamilyPrefix(declared)
|
||||
rest := strings.TrimPrefix(name, prefix)
|
||||
if !strings.HasPrefix(name, prefix) || !memberRest.MatchString(rest) {
|
||||
continue
|
||||
}
|
||||
if family == "" || len(declared) > len(family) {
|
||||
s, family, ok = OwnSecret{Path: strings.Replace(f.Path, "*", rest, 1), Taken: f.Taken, IssuedBy: f.IssuedBy}, declared, true
|
||||
}
|
||||
}
|
||||
return s, family, ok
|
||||
}
|
||||
|
||||
// Plain is every own secret declared by its own name: what the mesh makes when not given, and places by
|
||||
// name. A family is not one, and is never made.
|
||||
func (o OwnSecrets) Plain() OwnSecrets {
|
||||
out := make(OwnSecrets, len(o))
|
||||
for name, s := range o {
|
||||
if !IsFamily(name) {
|
||||
out[name] = s
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Families is every family's declared name, sorted.
|
||||
func (o OwnSecrets) Families() []string {
|
||||
var out []string
|
||||
for name := range o {
|
||||
if IsFamily(name) {
|
||||
out = append(out, name)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// Paths is each own secret's path by name — the shape every placement and file walk reads.
|
||||
func (o OwnSecrets) Paths() map[string]string {
|
||||
out := make(map[string]string, len(o))
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/beside"
|
||||
)
|
||||
|
||||
// A bind mount the module never declared is refused where it is written (novox/hq 04-ISSUES/026,
|
||||
@@ -67,16 +69,12 @@ func TestTheRuntimeSocketIsGrantedByTheCapabilityAndNotOtherwise(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// **Every manifest in the catalogue beside this checkout passes**, so the rule is not one the
|
||||
// catalogue is already breaking. Skipped, aloud, where the catalogue is not there.
|
||||
// **Every manifest in the catalogue (internal/beside) passes**, so the rule is not one the catalogue
|
||||
// is already breaking.
|
||||
func TestEveryCatalogueManifestDeclaresWhatItMounts(t *testing.T) {
|
||||
root := os.Getenv("MESH_CATALOG")
|
||||
if root == "" {
|
||||
root = "../../../mesh-catalog"
|
||||
}
|
||||
files, _ := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
files, _ := filepath.Glob(filepath.Join(beside.Catalogue(t), "*", "module.json"))
|
||||
if len(files) == 0 {
|
||||
t.Skipf("no catalogue at %s (set MESH_CATALOG to a checkout)", root)
|
||||
t.Fatal("no manifests in the catalogue, so this proved nothing")
|
||||
}
|
||||
for _, file := range files {
|
||||
raw, err := os.ReadFile(file)
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/beside"
|
||||
)
|
||||
|
||||
// **A manifest holds no subject** (novox/hq design 29 §1).
|
||||
@@ -17,10 +19,10 @@ import (
|
||||
// held by construction is one a later field breaks quietly, with the symptom appearing as a
|
||||
// permission that does not match a subject rather than as a manifest that was wrong.
|
||||
func TestNoManifestContainsASubject(t *testing.T) {
|
||||
root := filepath.Join("..", "..", "..", "mesh-catalog", "modules")
|
||||
root := beside.Catalogue(t)
|
||||
entries, err := os.ReadDir(root)
|
||||
if err != nil {
|
||||
t.Skipf("catalogue sibling not present: %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Anything in the mesh's own subject space, and anything shaped like a wire address.
|
||||
@@ -63,7 +65,7 @@ func TestNoManifestContainsASubject(t *testing.T) {
|
||||
walk(e.Name(), "", m)
|
||||
}
|
||||
if checked == 0 {
|
||||
t.Skip("no manifests read")
|
||||
t.Fatal("no manifests read, so this proved nothing")
|
||||
}
|
||||
if len(found) > 0 {
|
||||
t.Errorf("a manifest names a subject, so reorganising the subject space would mean "+
|
||||
@@ -91,13 +93,14 @@ func TestEveryManifestsEventNamesAreLocal(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// theCatalogue is every manifest beside this checkout, parsed the way registration parses one.
|
||||
// theCatalogue is every manifest of the catalogue internal/beside finds, parsed the way registration
|
||||
// parses one.
|
||||
func theCatalogue(t *testing.T) []Manifest {
|
||||
t.Helper()
|
||||
root := filepath.Join("..", "..", "..", "mesh-catalog", "modules")
|
||||
root := beside.Catalogue(t)
|
||||
entries, err := os.ReadDir(root)
|
||||
if err != nil {
|
||||
t.Skipf("catalogue sibling not present: %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out []Manifest
|
||||
for _, e := range entries {
|
||||
@@ -115,7 +118,7 @@ func theCatalogue(t *testing.T) []Manifest {
|
||||
out = append(out, m)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
t.Skip("no manifests found beside this checkout")
|
||||
t.Fatalf("no manifests under %s, so this proved nothing", root)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -2,7 +2,10 @@ package catalogue
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/beside"
|
||||
)
|
||||
|
||||
// The public issuer is the proxy's own fact (novox/hq ADR 0226), and the folding of it must not
|
||||
@@ -86,22 +89,22 @@ func TestRouteProxyKeepsItsPublicAccountDirectory(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The modules ADR 0226 retired stay retired, and nothing asks for what they provided.
|
||||
// The modules ADR 0226 retired stay retired, and nothing asks for what they provided. A module is
|
||||
// in the catalogue when its manifest is: a directory left behind with no manifest in it (a build's
|
||||
// leftovers, untracked by git) is not a module, and failed this on a desktop (novox/hq issue 432).
|
||||
func TestTheRetiredNetworkingModulesAreNotInTheCatalogue(t *testing.T) {
|
||||
if _, err := os.Stat("../../../mesh-catalog/modules"); err != nil {
|
||||
t.Skipf("the catalogue is not beside this checkout: %v", err)
|
||||
}
|
||||
modules := beside.Catalogue(t)
|
||||
for _, gone := range []string{"public-acme", "dhcpcd", "cloudflare-dns"} {
|
||||
if _, err := os.Stat("../../../mesh-catalog/modules/" + gone); err == nil {
|
||||
if _, err := os.Stat(filepath.Join(modules, gone, "module.json")); err == nil {
|
||||
t.Errorf("%s is in the catalogue again; ADR 0226 retired it", gone)
|
||||
}
|
||||
}
|
||||
entries, err := os.ReadDir("../../../mesh-catalog/modules")
|
||||
entries, err := os.ReadDir(modules)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, e := range entries {
|
||||
raw, err := os.ReadFile("../../../mesh-catalog/modules/" + e.Name() + "/module.json")
|
||||
raw, err := os.ReadFile(filepath.Join(modules, e.Name(), "module.json"))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -161,7 +161,9 @@ func TestTheCataloguesBarRendersEveryExampleOfTheShape(t *testing.T) {
|
||||
holder := catalogueManifest(t, "i3status-rust")
|
||||
s, _ := SeatNamed(BarSeat)
|
||||
if !placesKind(holder, s, BarKindBlock) {
|
||||
t.Skip("the catalogue beside this checkout has a bar that places no blocks yet")
|
||||
// It places them since the catalogue's bar took the seat; a skip here hid a bar that stopped
|
||||
// (novox/hq issue 432).
|
||||
t.Fatal("the catalogue's bar places no blocks, so none of the shape's examples would render")
|
||||
}
|
||||
tmpl, err := holderTemplate(holder, s, BarKindBlock)
|
||||
if err != nil {
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/beside"
|
||||
)
|
||||
|
||||
// Defends novox/hq ADR 0110: a seat is a module assignment from a closed set.
|
||||
@@ -193,13 +195,13 @@ func TestAClaimThatIsMalformedIsRefusedOnceForThat(t *testing.T) {
|
||||
|
||||
// Every module in use claims a seat in the set, so closing it refuses nothing that runs.
|
||||
//
|
||||
// Read from the catalogue beside this checkout and from this repository's own manifest, the two
|
||||
// places a manifest lives (ADR 0069). The private-network module's manifest is composed in code,
|
||||
// and its claim is checked where it is composed.
|
||||
// Read from the catalogue (internal/beside) and from this repository's own manifest, the two places a
|
||||
// manifest lives (ADR 0069). The private-network module's manifest is composed in code, and its claim
|
||||
// is checked where it is composed.
|
||||
func TestEveryManifestInUseClaimsASeatTheMeshDefines(t *testing.T) {
|
||||
paths, _ := filepath.Glob("../../../mesh-catalog/modules/*/module.json")
|
||||
paths, _ := filepath.Glob(filepath.Join(beside.Catalogue(t), "*", "module.json"))
|
||||
if len(paths) == 0 {
|
||||
t.Skip("the catalogue is not beside this checkout")
|
||||
t.Fatal("no manifests in the catalogue, so this proved nothing")
|
||||
}
|
||||
paths = append(paths, "../../module.json")
|
||||
var checked int
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A secret family (novox/hq ADR 0283): one own secret per part the settings name, issued outside the mesh, each
|
||||
// given by its full name, never made by the mesh.
|
||||
|
||||
func familyManifest(t *testing.T, ownSecrets string) (Manifest, error) {
|
||||
t.Helper()
|
||||
return ParseManifest([]byte(`{"module":"mounts","version":"1","own-secrets":{` + ownSecrets + `}}`))
|
||||
}
|
||||
|
||||
func TestAFamilyIsDeclaredIssuedOutsideWithOneStar(t *testing.T) {
|
||||
m, err := familyManifest(t, `"smb-password-*":{"path":"/var/lib/mounts/smb-password-*.secret","issued-by":"outside"}`)
|
||||
if err != nil {
|
||||
t.Fatalf("a well-formed family was refused: %v", err)
|
||||
}
|
||||
if got := m.OwnSecrets.Families(); len(got) != 1 || got[0] != "smb-password-*" {
|
||||
t.Fatalf("families: %v", got)
|
||||
}
|
||||
if len(m.OwnSecrets.Plain()) != 0 {
|
||||
t.Fatalf("a family counted as a secret declared by name: %v", m.OwnSecrets.Plain())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMalformedFamilyIsRefused(t *testing.T) {
|
||||
for name, c := range map[string]struct{ own, says string }{
|
||||
"made by the mesh": {`"smb-password-*":{"path":"/s/smb-password-*.secret","taken":"at-start"}`, "issued-by"},
|
||||
"no star in its path": {`"smb-password-*":{"path":"/s/smb-password.secret","issued-by":"outside"}`,
|
||||
"exactly one *"},
|
||||
"two stars in its path": {`"smb-password-*":{"path":"/s/*/smb-password-*.secret","issued-by":"outside"}`,
|
||||
"exactly one *"},
|
||||
"a star inside a name": {`"smb*password":{"path":"/s/x","issued-by":"outside"}`, "only ends a family"},
|
||||
"a name that is also a member": {`"smb-password-*":{"path":"/s/p-*","issued-by":"outside"},
|
||||
"smb-password-games":{"path":"/s/games","issued-by":"outside"}`, "also a member"},
|
||||
} {
|
||||
if _, err := familyManifest(t, c.own); err == nil || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%s: %v; want a refusal saying %q", name, err, c.says)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMemberIsFoundByItsFullNameAndLandsWhereTheStarIs(t *testing.T) {
|
||||
o := OwnSecrets{
|
||||
"smb-password-*": {Path: "/s/smb-password-*.secret", IssuedBy: IssuedOutside},
|
||||
"smb-password-big-*": {Path: "/big/*.secret", IssuedBy: IssuedOutside},
|
||||
"token": {Path: "/s/token", IssuedBy: IssuedOutside},
|
||||
}
|
||||
s, family, ok := o.Lookup("smb-password-games")
|
||||
if !ok || family != "smb-password-*" || s.Path != "/s/smb-password-games.secret" || s.IssuedBy != IssuedOutside {
|
||||
t.Fatalf("a member: %+v %q %v", s, family, ok)
|
||||
}
|
||||
if s, family, ok := o.Lookup("smb-password-big-one"); !ok || family != "smb-password-big-*" || s.Path != "/big/one.secret" {
|
||||
t.Fatalf("the longest family that fits: %+v %q %v", s, family, ok)
|
||||
}
|
||||
if s, family, ok := o.Lookup("token"); !ok || family != "" || s.Path != "/s/token" {
|
||||
t.Fatalf("a secret declared by name: %+v %q %v", s, family, ok)
|
||||
}
|
||||
for _, name := range []string{"smb-password-*", "smb-password-", "smb-password-../etc", "smb-password-a/b",
|
||||
"smb-password-a.b", "smb-password-Games", "smb-password--x", "other"} {
|
||||
if _, _, ok := o.Lookup(name); ok {
|
||||
t.Errorf("%q was found as a secret", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A member given is one file at its path; one not given is nothing, and the machine still composes — the mesh never
|
||||
// makes a member.
|
||||
func TestAMemberGivenIsPlacedAndOneNotGivenIsNothing(t *testing.T) {
|
||||
m, err := familyManifest(t, `"smb-password-*":{"path":"/var/lib/mounts/smb-password-*.secret","issued-by":"outside"}`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := Resolution{Node: "workstation", Modules: []Manifest{m}}
|
||||
out, err := r.Declaration(Rendering{Needed: map[string]map[string]string{"mounts": {"smb-password-games": "sealed"}}})
|
||||
if err != nil {
|
||||
t.Fatalf("a module with one member given did not compose: %v", err)
|
||||
}
|
||||
var paths []string
|
||||
for _, res := range out {
|
||||
if res["sealed"] != nil {
|
||||
paths = append(paths, res["path"].(string))
|
||||
}
|
||||
}
|
||||
if len(paths) != 1 || paths[0] != "/var/lib/mounts/smb-password-games.secret" {
|
||||
t.Fatalf("placed: %v", paths)
|
||||
}
|
||||
out, err = r.Declaration(Rendering{})
|
||||
if err != nil {
|
||||
t.Fatalf("a module with no member given did not compose: %v", err)
|
||||
}
|
||||
for _, res := range out {
|
||||
if res["sealed"] != nil {
|
||||
t.Fatalf("a member nobody gave was placed: %v", res)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -50,3 +50,41 @@ func TestAContainerReadingAnOwnSecretIsRestartedWhenItChanges(t *testing.T) {
|
||||
t.Fatalf("a container that reads no secret was given one to restart on: %v", by["agent.other"]["restart-on"])
|
||||
}
|
||||
}
|
||||
|
||||
// A member of a secret family is an own secret too (novox/hq issue 405, ADR 0283 decision 6): a container that
|
||||
// reads a member given is restarted when it changes, as for a secret declared by name. A member not given is no
|
||||
// file, so nothing is restarted on it.
|
||||
func TestAContainerReadingAFamilyMemberIsRestartedWhenItChanges(t *testing.T) {
|
||||
m := Manifest{Module: "mounts", Version: "1",
|
||||
OwnSecrets: OwnSecrets{"smb-password-*": {Path: "/var/lib/mesh/mounts/smb-password-*.secret", IssuedBy: IssuedOutside}},
|
||||
Resources: []map[string]any{
|
||||
{"id": "games", "type": "container", "name": "mounts-games", "network": "host",
|
||||
"image": "registry.example/mounts@sha256:" + strings.Repeat("a", 64),
|
||||
"volumes": []any{"/var/lib/mesh/mounts/smb-password-games.secret:/run/password:ro"}},
|
||||
{"id": "library", "type": "container", "name": "mounts-library", "network": "host",
|
||||
"image": "registry.example/mounts@sha256:" + strings.Repeat("a", 64),
|
||||
"volumes": []any{"/var/lib/mesh/mounts/smb-password-library.secret:/run/password:ro"}},
|
||||
}}
|
||||
got, err := Resolve(shelf(m), []string{m.Module},
|
||||
Node{Name: "anchor", At: "10.0.0.1", Capabilities: map[string]bool{"container-runtime": true}}, World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := got.Declaration(Rendering{Needed: map[string]map[string]string{"mounts": {"smb-password-games": "SEALED"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
by := map[string]map[string]any{}
|
||||
for _, r := range out {
|
||||
by[r["id"].(string)] = r
|
||||
}
|
||||
if want := []any{"mounts.needs-smb-password-games"}; !reflect.DeepEqual(by["mounts.games"]["restart-on"], want) {
|
||||
t.Fatalf("a container reading a member given is not restarted on it: %v", by["mounts.games"]["restart-on"])
|
||||
}
|
||||
if _, placed := by["mounts.needs-smb-password-games"]; !placed {
|
||||
t.Fatalf("the member given is not placed, so a restart-on names nothing: %v", out)
|
||||
}
|
||||
if _, has := by["mounts.library"]["restart-on"]; has {
|
||||
t.Fatalf("a container reading a member not given was given a restart-on naming nothing: %v", by["mounts.library"]["restart-on"])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,7 +56,7 @@ func secretsUsed(content string) []string {
|
||||
// name would end that, to save writing a file.
|
||||
func sealedFor(m Manifest, needs []Needed, with Rendering) (map[string]string, error) {
|
||||
sealed := map[string]string{}
|
||||
for name := range m.OwnSecrets {
|
||||
for name := range m.OwnSecrets.Plain() {
|
||||
if value := with.Needed[m.Module][name]; value != "" {
|
||||
sealed[name] = value
|
||||
}
|
||||
|
||||
@@ -2,7 +2,10 @@ package catalogue
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/beside"
|
||||
)
|
||||
|
||||
// **The showcase module is parsed by the real parser, in the real test suite.**
|
||||
@@ -11,9 +14,9 @@ import (
|
||||
// Written as a test rather than a script so it runs whenever anything about manifests changes —
|
||||
// which is exactly when a module using all of it would quietly stop being valid.
|
||||
func TestTheShowcaseModuleIsAValidManifest(t *testing.T) {
|
||||
raw, err := os.ReadFile("../../../mesh-catalog/modules/showcase/module.json")
|
||||
raw, err := os.ReadFile(filepath.Join(beside.Catalogue(t), "showcase", "module.json"))
|
||||
if err != nil {
|
||||
t.Skipf("the catalogue is not beside this checkout: %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := ParseManifest(raw)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// A placeholder no pass consumes is refused, never written out as text (novox/hq issue 231).
|
||||
//
|
||||
// Each namespace is filled by its own pass with its own pattern, in the fields that pass reads. A
|
||||
// word in that shape that no pattern matched — `${machnie:address}`, `${shel:zsh:first}`, a setting
|
||||
// key no definition could declare such as `${setting:Undeclared}` — was left in the file as it was
|
||||
// written and reached a machine as a value: the predecessor's failure the namespaced placeholders
|
||||
// were meant to end (novox/hq ADR 0164). So the definition is swept, field by field, against the
|
||||
// same table of what each pass fills where.
|
||||
//
|
||||
// **The definition, never the values.** Swept as the manifest wrote it, at the catalogue check and
|
||||
// again at composition before any pass has run: what an operator's setting, a binding or a merged
|
||||
// JSON setting puts into a file is its own software's text — `${env:HOME}` for Log4j, `${timeout:30}`
|
||||
// for Spring — and refusing it there would refuse something its author never wrote, on a machine the
|
||||
// check had passed (novox/hq issue 231, review of mesh-controller #211).
|
||||
//
|
||||
// What is refused:
|
||||
// - a placeholder of a namespace the mesh knows, in a field that namespace's pass does not read: it
|
||||
// would reach the machine as the same text;
|
||||
// - any other `${<known namespace>:`, case-insensitively, unless a shell's parameter operator follows
|
||||
// its colon — `${Machine:address}`, `${machine:.address}`, `${machine: address}`, an unclosed
|
||||
// `${machine:address` — because no pass's pattern takes it;
|
||||
// - a namespace-shaped placeholder of a word the mesh does not know: a lower-case word, a colon, and
|
||||
// a key that begins with a letter or a digit and holds no space, brace or `$`.
|
||||
//
|
||||
// **The shell's own syntax is not that shape, and passes.** `${NAME:-…}` has an upper-case word,
|
||||
// `${(%):-…}` and `${1:-.}` begin with no letter, and a lower-case variable with an operator after its
|
||||
// colon — `${count:-}`, `${trial:+…}`, `${state:=…}` — has no key that begins with a letter or a digit.
|
||||
// And an operator — `:-`, `:=`, `:+`, `:?` — after a word the mesh also uses is the shell's too:
|
||||
// `${PORT:-8080}`, `${SHELL:-/bin/sh}`, `${SECRET:?unset}` and `${dir:-/tmp}` are among the commonest
|
||||
// lines of a script or an env file, and pass whatever the word's case.
|
||||
// What the shape does catch is a zsh modifier (`${path:t}`) or a substring (`${where:0:12}`) in a
|
||||
// resource's own text: shell code of that kind belongs in the module's contributed shell code, which
|
||||
// is not a resource and is never swept (novox/hq ADR 0204).
|
||||
|
||||
// filler is one namespace's pass: the patterns it fills with, and the fields it reads them in, by
|
||||
// resource type ("*" for any type).
|
||||
type filler struct {
|
||||
namespace string
|
||||
patterns []*regexp.Regexp
|
||||
fields map[string][]string
|
||||
// why, when set, is the rule that keeps the namespace out of every other field, said with a
|
||||
// refusal of one written there.
|
||||
why string
|
||||
}
|
||||
|
||||
// fillers is the table of what each pass fills where — read from the passes themselves: settingInto,
|
||||
// dirInto, accessInto, intoFile (whose ${secret:…} the node-engine fills), boundInto, portInto,
|
||||
// seatInto, machineInto and contributionsInto. A pass that comes to read another field adds it here,
|
||||
// or the sweep refuses the placeholder it would have filled.
|
||||
var fillers = []filler{
|
||||
{namespace: "setting", patterns: []*regexp.Regexp{settingRef}, fields: map[string][]string{"file": {"content"}, "service": {"unit"}}},
|
||||
{namespace: "dir", patterns: []*regexp.Regexp{dirRef}, fields: map[string][]string{"*": {"path", "content", "volumes", "env", "env-file"}}},
|
||||
{namespace: "access", patterns: []*regexp.Regexp{accessRef}, fields: map[string][]string{"*": {"path", "content", "volumes", "env", "env-file"}}},
|
||||
{namespace: "secret", patterns: []*regexp.Regexp{placeholder}, fields: map[string][]string{"file": {"content"}},
|
||||
why: "a secret is never filled into an environment variable or any other field: put it in a file the " +
|
||||
"module declares and mount that (novox/hq ADR 0086)"},
|
||||
{namespace: "bound", patterns: []*regexp.Regexp{bound}, fields: map[string][]string{"file": {"content"}}},
|
||||
{namespace: "port", patterns: []*regexp.Regexp{ofPort}, fields: map[string][]string{"file": {"content"}, "container": {"env"}, "process": {"env"}}},
|
||||
{namespace: "seat", patterns: []*regexp.Regexp{ofSeat, ofSeatReach}, fields: map[string][]string{"file": {"content"}, "container": {"env"}, "process": {"env"}}},
|
||||
{namespace: "machine", patterns: []*regexp.Regexp{ofMachine}, fields: map[string][]string{"*": {"path", "owner", "content", "name", "user", "root", "home"}}},
|
||||
// Placed in a file's content by their holders, and judged there by their own rules
|
||||
// (placeholderProblems, seatPlaceholderProblems).
|
||||
{namespace: "environment", patterns: []*regexp.Regexp{ofEnvironment}, fields: map[string][]string{"*": {"content"}}},
|
||||
{namespace: "shell", patterns: []*regexp.Regexp{ofShell}, fields: map[string][]string{"*": {"content"}}},
|
||||
{namespace: "contribution", patterns: []*regexp.Regexp{ofContribution}, fields: map[string][]string{"*": {"content"}}},
|
||||
// Filled in what a provider serves (consumer_into_serves.go), never in a resource.
|
||||
{namespace: "consumer", patterns: []*regexp.Regexp{consumerFact}},
|
||||
}
|
||||
|
||||
// reads is whether this pass fills a field of a resource of this type.
|
||||
func (f filler) reads(kind, field string) bool {
|
||||
return oneOf(f.fields[kind], field) || oneOf(f.fields["*"], field)
|
||||
}
|
||||
|
||||
// ofKnownNamespace is any `${<known namespace>:` and what follows it up to its brace or the end of
|
||||
// its line, whatever its case, unless what follows the colon is a shell's parameter operator (`-`,
|
||||
// `=`, `+`, `?`): what remains of one once every pass's pattern is set aside is a misspelling.
|
||||
var ofKnownNamespace = regexp.MustCompile(`(?im)\$\{(?:` + strings.Join(namespacesOf(fillers), "|") +
|
||||
`):(?:[^-=+?}\n][^}\n]*\}?|\}|$)`)
|
||||
|
||||
// namespaceShaped is a placeholder in the mesh's shape: a lower-case word, a colon, and a key that
|
||||
// begins with a letter or a digit and holds no space, brace or `$`.
|
||||
var namespaceShaped = regexp.MustCompile(`\$\{[a-z][a-z0-9_-]*:[A-Za-z0-9][^\s{}$]*\}`)
|
||||
|
||||
func namespacesOf(fs []filler) []string {
|
||||
out := make([]string, len(fs))
|
||||
for i, f := range fs {
|
||||
out[i] = f.namespace
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// unconsumedPlaceholders is every placeholder in one resource of a definition that no pass fills
|
||||
// where it stands, each named with the module, the resource, the field and the token. The same
|
||||
// function at the catalogue check and at composition, over the resource as the manifest wrote it.
|
||||
func unconsumedPlaceholders(module string, r map[string]any) []string {
|
||||
kind := fmt.Sprint(r["type"])
|
||||
var problems []string
|
||||
var sweep func(top, field string, v any)
|
||||
sweep = func(top, field string, v any) {
|
||||
switch v := v.(type) {
|
||||
case string:
|
||||
rest := v
|
||||
for _, f := range fillers {
|
||||
for _, p := range f.patterns {
|
||||
if !f.reads(kind, top) {
|
||||
for _, token := range p.FindAllString(rest, -1) {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s's resource %v holds %s in its %s, and ${%s:…} is not filled in this field: "+
|
||||
"it would reach the machine as that text (novox/hq issue 231)%s",
|
||||
module, r["id"], token, field, f.namespace, whereFilled(f)))
|
||||
}
|
||||
}
|
||||
rest = p.ReplaceAllString(rest, "")
|
||||
}
|
||||
}
|
||||
for _, token := range ofKnownNamespace.FindAllString(rest, -1) {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s's resource %v holds %s in its %s, which is no placeholder the mesh fills: a "+
|
||||
"misspelt key, or a namespace in the wrong case, would reach the machine as that "+
|
||||
"text (novox/hq issue 231)", module, r["id"], token, field))
|
||||
}
|
||||
rest = ofKnownNamespace.ReplaceAllString(rest, "")
|
||||
for _, token := range namespaceShaped.FindAllString(rest, -1) {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s's resource %v holds %s in its %s, and no pass of the mesh fills it: a misspelt "+
|
||||
"placeholder would reach the machine as that text (novox/hq issue 231). The mesh "+
|
||||
"fills ${%s:…}; the shell's own syntax belongs in the module's shell code (ADR 0204)",
|
||||
module, r["id"], token, field, strings.Join(namespacesOf(fillers), ":…}, ${")))
|
||||
}
|
||||
case []any:
|
||||
for i, e := range v {
|
||||
sweep(top, fmt.Sprintf("%s[%d]", field, i), e)
|
||||
}
|
||||
case map[string]any:
|
||||
for _, k := range sortedKeys(v) {
|
||||
sweep(top, field+"."+k, v[k])
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, k := range sortedKeys(r) {
|
||||
sweep(k, k, r[k])
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// whereFilled says where a namespace's pass does fill, for a refusal of one written elsewhere.
|
||||
func whereFilled(f filler) string {
|
||||
if f.why != "" {
|
||||
return ". " + strings.ToUpper(f.why[:1]) + f.why[1:]
|
||||
}
|
||||
if len(f.fields) == 0 {
|
||||
return "; it is filled only in what a provider serves"
|
||||
}
|
||||
var where []string
|
||||
for _, kind := range sortedKeys(f.fields) {
|
||||
of := "a " + kind + "'s"
|
||||
if kind == "*" {
|
||||
of = "any resource's"
|
||||
}
|
||||
where = append(where, of+" "+strings.Join(f.fields[kind], ", "))
|
||||
}
|
||||
return "; it is filled in " + strings.Join(where, "; ")
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// novox/hq issue 231 — a misspelled placeholder is written out as text.
|
||||
//
|
||||
// The four placeholders of the issue, in one file: two misspelled namespaces, a setting key no
|
||||
// definition could declare, and the shell's own syntax. The first three are refused by name, at the
|
||||
// catalogue check and at composition; the fourth reaches the file as written.
|
||||
const (
|
||||
misspelledShell = "${shel:zsh:first}"
|
||||
undeclaredSetting = "${setting:Undeclared}"
|
||||
misspelledMachine = "${machnie:address}"
|
||||
shellsOwn = "${XDG_CACHE_HOME:-x}"
|
||||
// The shell's operators after a word the mesh also uses, in any case: the shell's, and passed.
|
||||
shellsOperators = "${PORT:-8080} ${SHELL:-/bin/sh} ${SECRET:?unset} ${dir:-/tmp}"
|
||||
)
|
||||
|
||||
// The catalogue check — the strict parse registration runs too — refuses each by name, with the
|
||||
// module and the field it stands in, and says nothing about the shell's own syntax.
|
||||
func TestAMisspelledPlaceholderIsRefusedAtTheCheck(t *testing.T) {
|
||||
raw := `{"module":"speller","version":"1","resources":[{"id":"rc","type":"file","path":"/etc/speller.rc",` +
|
||||
`"content":"a=` + misspelledShell + `\nb=` + undeclaredSetting + `\nc=` + misspelledMachine +
|
||||
`\nd=` + shellsOwn + `\n"}]}`
|
||||
_, err := ParseManifest([]byte(raw))
|
||||
if err == nil {
|
||||
t.Fatal("a file holding three placeholders no pass consumes was accepted")
|
||||
}
|
||||
for _, token := range []string{misspelledShell, undeclaredSetting, misspelledMachine} {
|
||||
if !strings.Contains(err.Error(), "speller's resource rc holds "+token+" in its content") {
|
||||
t.Errorf("the refusal does not name %s with its module and field: %v", token, err)
|
||||
}
|
||||
}
|
||||
if strings.Contains(err.Error(), "XDG_CACHE_HOME") {
|
||||
t.Errorf("the shell's own syntax was refused: %v", err)
|
||||
}
|
||||
|
||||
// A namespace the mesh knows, misspelt in any way its own pattern does not take, and the same
|
||||
// namespace in a field its pass does not read: refused at the check as at composition.
|
||||
for _, c := range []struct{ resource, token, field string }{
|
||||
{`{"id":"rc","type":"file","path":"/etc/rc","content":"${Machine:address}"}`, "${Machine:address}", "content"},
|
||||
{`{"id":"rc","type":"file","path":"/etc/rc","content":"${machine:.address}"}`, "${machine:.address}", "content"},
|
||||
{`{"id":"rc","type":"file","path":"/etc/rc","content":"${machine: address}"}`, "${machine: address}", "content"},
|
||||
{`{"id":"rc","type":"file","path":"/etc/rc","content":"${Dir:x}"}`, "${Dir:x}", "content"},
|
||||
{`{"id":"rc","type":"file","path":"/etc/rc","content":"a=${machine:address\nb=1"}`, "${machine:address", "content"},
|
||||
{`{"id":"rc","type":"process","name":"speller","env":{"NAME":"${machine:name}"}}`, "${machine:name}", "env.NAME"},
|
||||
} {
|
||||
raw := `{"module":"speller","version":"1","resources":[` + c.resource + `]}`
|
||||
_, err := ParseManifest([]byte(raw))
|
||||
if err == nil || !strings.Contains(err.Error(), "speller's resource rc holds "+c.token+" in its "+c.field) {
|
||||
t.Errorf("%s in its %s was accepted at the check, or not named: %v", c.token, c.field, err)
|
||||
}
|
||||
}
|
||||
|
||||
// And the shell's syntax alone, beside placeholders every pass knows, is accepted — as is the
|
||||
// shape a lower-case shell variable takes with an operator after its colon.
|
||||
raw = `{"module":"speller","version":"1","resources":[{"id":"rc","type":"file","path":"${machine:account-home}/.rc",` +
|
||||
`"content":"` + shellsOwn + ` ${(%):-%n} ${1:-.} ${count:-} ${trial:+ on trial} ${machine:address} ` +
|
||||
shellsOperators + `\n"},` +
|
||||
`{"id":"server","type":"container","name":"server","env":{"PORT":"${PORT:-80}"}}]}`
|
||||
if _, err := ParseManifest([]byte(raw)); err != nil {
|
||||
t.Fatalf("the shell's own syntax was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Composition refuses the same placeholders in the same words — a manifest the store already holds
|
||||
// was never parsed by this binary — and a namespace the mesh knows, written in a field its pass does
|
||||
// not read, is refused there too, because it would reach the machine as the same literal text.
|
||||
func TestAMisspelledPlaceholderIsRefusedAtComposition(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
field string
|
||||
r map[string]any
|
||||
token string
|
||||
}{
|
||||
{"content", map[string]any{"id": "rc", "type": "file", "path": "/etc/speller.rc", "content": "a=" + misspelledShell + "\n"}, misspelledShell},
|
||||
{"content", map[string]any{"id": "rc", "type": "file", "path": "/etc/speller.rc", "content": "b=" + undeclaredSetting + "\n"}, undeclaredSetting},
|
||||
{"content", map[string]any{"id": "rc", "type": "file", "path": "/etc/speller.rc", "content": "c=" + misspelledMachine + "\n"}, misspelledMachine},
|
||||
{"env.NAME", map[string]any{"id": "rc", "type": "process", "name": "speller", "env": map[string]any{"NAME": "${machine:name}"}}, "${machine:name}"},
|
||||
{"content", map[string]any{"id": "rc", "type": "file", "path": "/etc/speller.rc", "content": "${Machine:address}"}, "${Machine:address}"},
|
||||
{"content", map[string]any{"id": "rc", "type": "file", "path": "/etc/speller.rc", "content": "${machine:.address}"}, "${machine:.address}"},
|
||||
{"content", map[string]any{"id": "rc", "type": "file", "path": "/etc/speller.rc", "content": "${machine: address}"}, "${machine: address}"},
|
||||
{"content", map[string]any{"id": "rc", "type": "file", "path": "/etc/speller.rc", "content": "${Dir:x}"}, "${Dir:x}"},
|
||||
{"content", map[string]any{"id": "rc", "type": "file", "path": "/etc/speller.rc", "content": "a=${machine:address\nb=1"}, "${machine:address"},
|
||||
} {
|
||||
m := Manifest{Module: "speller", Version: "1", Resources: []map[string]any{c.r}}
|
||||
r := Resolution{Node: "workstation", Account: "op", Modules: []Manifest{m}}
|
||||
_, err := r.Declaration(Rendering{})
|
||||
if err == nil || !strings.Contains(err.Error(), "speller's resource rc holds "+c.token+" in its "+c.field) {
|
||||
t.Errorf("%s in its %s was composed rather than refused by name: %v", c.token, c.field, err)
|
||||
}
|
||||
}
|
||||
|
||||
m := Manifest{Module: "speller", Version: "1", Resources: []map[string]any{
|
||||
{"id": "rc", "type": "file", "path": "/etc/speller.rc", "content": "d=" + shellsOwn + " " + shellsOperators + "\n"},
|
||||
{"id": "server", "type": "process", "name": "server", "env": map[string]any{"PORT": "${PORT:-80}"}},
|
||||
}}
|
||||
out, err := Resolution{Node: "workstation", Account: "op", Modules: []Manifest{m}}.Declaration(Rendering{})
|
||||
if err != nil {
|
||||
t.Fatalf("the shell's own syntax was refused: %v", err)
|
||||
}
|
||||
if got := contentOf(t, out, "speller.rc"); got != "d="+shellsOwn+" "+shellsOperators+"\n" {
|
||||
t.Fatalf("the shell's own syntax did not pass through as written: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// contentOf is the content of the composed resource with this id, failing when it was not composed.
|
||||
func contentOf(t *testing.T, out []map[string]any, id string) string {
|
||||
t.Helper()
|
||||
for _, res := range out {
|
||||
if res["id"] == id {
|
||||
return plainly(res["content"])
|
||||
}
|
||||
}
|
||||
t.Fatalf("%s was not composed: %v", id, out)
|
||||
return ""
|
||||
}
|
||||
|
||||
// What a value puts into a file is its software's text, not the definition's, and is never swept
|
||||
// (review of mesh-controller #211): an operator's setting holding `${labels:instance}` filled through
|
||||
// ${setting:…}, and a JSON setting `${level:upper}` merged into a mergeable file, reach the machine as
|
||||
// set — software that templates its own configuration (Log4j's `${env:…}`, Spring's `${timeout:30}`)
|
||||
// is configured exactly this way.
|
||||
func TestAValueHoldingAPlaceholderShapeComposesUnchanged(t *testing.T) {
|
||||
m := Manifest{Module: "templater", Version: "1", Resources: []map[string]any{
|
||||
{"id": "conf", "type": "file", "path": "/etc/templater.conf", "content": "template=${setting:template}\n"},
|
||||
{"id": "json", "type": "file", "path": "/etc/templater.json", "merge": MergeJSON, "content": `{"fmt":"plain"}`},
|
||||
}}
|
||||
settings := SettingsBy{"templater": {{From: "the operator", Values: map[string]any{
|
||||
"template": "${labels:instance}", "fmt": "${level:upper}",
|
||||
}}}}
|
||||
out, err := Resolution{Node: "workstation", Account: "op", Modules: []Manifest{m}}.Declaration(Rendering{Settings: settings})
|
||||
if err != nil {
|
||||
t.Fatalf("a value holding a placeholder's shape was refused as the definition's: %v", err)
|
||||
}
|
||||
if got := contentOf(t, out, "templater.conf"); got != "template=${labels:instance}\n" {
|
||||
t.Errorf("the setting did not reach the file as set: %q", got)
|
||||
}
|
||||
if got := contentOf(t, out, "templater.json"); !strings.Contains(got, `${level:upper}`) {
|
||||
t.Errorf("the merged setting did not reach the file as set: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Contributed shell code is the shell's, and no pass reads it (novox/hq ADR 0204): zsh's own
|
||||
// `${path:t}` is namespace-shaped, and reaches the holder's file untouched.
|
||||
func TestContributedShellCodeIsNotSwept(t *testing.T) {
|
||||
modules := []Manifest{zshHolder(), {Module: "modifier", Shell: []ShellCode{
|
||||
{For: "zsh", Slot: "normal", Code: "echo ${path:t} ${shel:zsh:first}"},
|
||||
}}}
|
||||
out, err := Resolution{Node: "workstation", Account: "op", Modules: modules}.Declaration(Rendering{})
|
||||
if err != nil {
|
||||
t.Fatalf("contributed shell code was swept: %v", err)
|
||||
}
|
||||
if got := contentOf(t, out, "zsh.zshrc"); !strings.Contains(got, "echo ${path:t} ${shel:zsh:first}") {
|
||||
t.Fatalf("the contributed code did not reach the holder's file as written: %q", got)
|
||||
}
|
||||
}
|
||||
+27
-10
@@ -164,8 +164,9 @@ var ControllerVerbs = []Verb{
|
||||
Input: schema(map[string]string{"plan": "the walk's id", "why": "why", "by": "who stopped the delivery"},
|
||||
[]string{"plan", "why"})},
|
||||
{Name: "delivery-walks", Description: "The walks the controller keeps (novox/hq ADR 0239): every open one and the " +
|
||||
"last ended ones, each whole — its tiers, each module's state, first machines and gate — and whether the " +
|
||||
"delivery seat has a holder on record. Given a plan, that one.",
|
||||
"last ended ones, each whole — its tiers, each module's state, first machines and first-node gate with its readings, and its " +
|
||||
"phases from its merge to every machine running it (novox/hq ADR 0282) — and whether the delivery seat has a " +
|
||||
"holder on record. Given a plan, that one.",
|
||||
Input: schema(map[string]string{"plan": "one walk's id", "limit": "how many ended walks beside the open ones (default 50)"},
|
||||
nil)},
|
||||
{Name: "plan", Description: "What one machine would run, and why: the declaration the mesh would send it — " +
|
||||
@@ -231,17 +232,32 @@ var ControllerVerbs = []Verb{
|
||||
"cause": "with why: the cause in a word, the word a second rotation for the same reason uses (optional)",
|
||||
}, nil)},
|
||||
{Name: "give", Description: "Take a module's own secret from the operator at their desk (novox/hq ADR 0259 " +
|
||||
"§10): a prompt that does not show what is typed opens on the machine named by at, its answer comes " +
|
||||
"back sealed to this call alone, and is sealed to the module's machine as `secret accept` seals it. " +
|
||||
"The value is never an argument and never in the answer: the answer says it was taken, or why not. " +
|
||||
"Recorded in the hand-act log as a value given at the desk. The prompt waits 25 seconds; dismissed " +
|
||||
"or unanswered, nothing changes. Then push the machine.",
|
||||
"§10): the same as secret-ask with the desk named. A prompt that does not show what is typed opens on " +
|
||||
"the machine named by at, its answer comes back sealed to this call alone, and is sealed to the " +
|
||||
"module's machine as `secret accept` seals it. The value is never an argument and never in the answer: " +
|
||||
"the answer says it was taken, or why not. Recorded in the hand-act log as a value given at the desk. " +
|
||||
"The prompt waits 25 seconds; dismissed or unanswered, nothing changes. Then push the machine.",
|
||||
Input: schema(map[string]string{
|
||||
"node": "the machine the module runs on, which the secret is sealed to",
|
||||
"module": "the module's name",
|
||||
"secret": "the own secret's name in the module's definition",
|
||||
"at": "the machine the operator sits at, where the prompt opens",
|
||||
}, []string{"node", "module", "secret", "at"})},
|
||||
{Name: "secret-ask", Description: "Ask the operator for a module's own secret (novox/hq ADR 0277): a prompt " +
|
||||
"that shows nothing of what is typed opens at the desk — the module's machine, or the one at names — " +
|
||||
"naming the module, the secret and who asked; the operator types the value there, never on a channel " +
|
||||
"and never in a verb's argument; the answer comes back sealed to this call alone and is sealed to the " +
|
||||
"module's machine as `secret accept` seals it. The answer says the value was taken, or why not. " +
|
||||
"Refused for a secret the mesh issues itself (the bus account, one the mesh may make) and for a module " +
|
||||
"that runs as an account of its own, whose value is typed at the controller's terminal. Bounded: one " +
|
||||
"open prompt per secret, three asks an hour. Recorded in the hand-act log, with who asked, and " +
|
||||
"announced on every channel. Then push the machine.",
|
||||
Input: schema(map[string]string{
|
||||
"node": "the machine the module runs on, which the secret is sealed to",
|
||||
"module": "the module's name",
|
||||
"secret": "the own secret's name in the module's definition",
|
||||
"at": "the machine the operator sits at, where the prompt opens; the module's machine when absent",
|
||||
}, []string{"node", "module", "secret"})},
|
||||
{Name: "issue", Description: "Give a module on a machine its account on the bus: minted, and sealed to the " +
|
||||
"machine as the module's own secret named broker, read at the next push of that machine. For a module " +
|
||||
"whose definition declares that secret; refused with the reason otherwise. Issued again, it replaces the account.",
|
||||
@@ -360,10 +376,11 @@ var ControllerVerbs = []Verb{
|
||||
"recorded — who, why and the cause of each, and which causes repeat: each repeat is a healer the mesh lacks.",
|
||||
Input: schema(map[string]string{"days": "how many days back (default 14)"}, nil)},
|
||||
{Name: "durations", Description: "How long things take, as the controller measured them: a send to its machine's " +
|
||||
"report (apply), a machine's silence between words (heartbeat-gap), a plan's tier, a build — per machine, " +
|
||||
"repository or module, with median, p90 and max. What the core's bounds are set from (novox/hq to-be 45 Phase 0).",
|
||||
"report (apply), a machine's silence between words (heartbeat-gap), a plan's tier, a build, and each phase of an " +
|
||||
"ended walk per class (walk-phase, novox/hq ADR 0282) — per machine, repository, module or class/phase, with " +
|
||||
"median, p90 and max. What the core's bounds are set from (novox/hq to-be 45 Phase 0).",
|
||||
Input: schema(map[string]string{
|
||||
"kind": "one kind: apply, heartbeat-gap, plan-tier or build; every kind when absent",
|
||||
"kind": "one kind: apply, heartbeat-gap, plan-tier, build or walk-phase; every kind when absent",
|
||||
"days": "how many days back (default 14)",
|
||||
}, nil)},
|
||||
// What is wrong, and the self-check (novox/hq to-be 45 §2, §4).
|
||||
|
||||
@@ -152,7 +152,11 @@ type Condition struct {
|
||||
// reopened after a send was there at the send unless the send fell in one of its gaps.
|
||||
First time.Time `json:"first,omitzero"`
|
||||
Gaps []Gap `json:"gaps,omitempty"`
|
||||
// Observations is how many times it was observed since raised.
|
||||
// Observations is how many times it was observed since raised: a look that sees it, for a source
|
||||
// that looks at the present; for one that reads a record of what happened (Observation.Happened),
|
||||
// how many times it happened, and LastObserved when it last did (novox/hq issue 402). That count is a
|
||||
// running total since raised, never lowered: a consumer's dead letters count each message given up on
|
||||
// while the condition is open, not the number held now (novox/hq issue 440).
|
||||
Observations int `json:"observations"`
|
||||
// Count is how many times it has been raised, a reopening within ReopenWithin counted.
|
||||
Count int `json:"count"`
|
||||
@@ -208,6 +212,15 @@ type Observation struct {
|
||||
Said string
|
||||
Source string
|
||||
Resolver string
|
||||
// Happened is set by a source that reads a record of what happened rather than looking at the
|
||||
// present — a bus advisory, remembered for an hour and read again on every look — to when it last
|
||||
// happened; Times is how many times the record counts it. The condition then counts what happened,
|
||||
// not the looks (novox/hq issue 402): one refusal looked at every half minute read as "observed 15
|
||||
// time(s)", with a line of evidence every 30 seconds, and was taken for a refusal going on. A look
|
||||
// that brings nothing newer than the condition's last observation adds no observation and no
|
||||
// evidence. Zero for a source that looks at the present, whose every look is an observation.
|
||||
Happened time.Time
|
||||
Times int
|
||||
// Confirm says a single look can be wrong about this finding — a question over the network that
|
||||
// went unanswered, a time measured once on a loaded machine. The keeper does not read it: the
|
||||
// source that looks again does, and raises it only when the next look sees it too, or while it is
|
||||
@@ -265,6 +278,10 @@ func (o Observation) check() error {
|
||||
return fmt.Errorf("the condition %s says nothing", o.Key())
|
||||
case strings.TrimSpace(o.Source) == "":
|
||||
return fmt.Errorf("the condition %s does not say what raised it", o.Key())
|
||||
case o.Times != 0 && o.Happened.IsZero():
|
||||
// Times is what a record counts beside when it last happened: alone, the raise would ignore it
|
||||
// and an update count it, so the count would mean two things (novox/hq issue 440).
|
||||
return fmt.Errorf("the condition %s says how often it happened (%d) but not when", o.Key(), o.Times)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -191,11 +191,16 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
|
||||
if err != nil {
|
||||
return Condition{}, fmt.Errorf("reading the condition %s: %w", key, err)
|
||||
}
|
||||
// What was observed and when: the look, or what the record says happened (novox/hq issue 402).
|
||||
at, observations := now, 1
|
||||
if !o.Happened.IsZero() {
|
||||
at, observations = o.Happened.UTC(), max(1, o.Times)
|
||||
}
|
||||
if !found {
|
||||
c := Condition{Key: key, Kind: o.Kind, Subject: Subject{Scope: o.Scope, ID: o.ID, Machine: o.Machine, Also: o.Also},
|
||||
Severity: o.Severity, Summary: o.Summary, Headline: o.Headline, Explanation: o.Explanation,
|
||||
Resolved: o.Resolved, Needs: o.Needs, Actions: o.Actions, Evidence: []Evidence{{At: now, Said: said}},
|
||||
Source: o.Source, Raised: now, LastObserved: now, Observations: 1, Count: 1,
|
||||
Resolved: o.Resolved, Needs: o.Needs, Actions: o.Actions, Evidence: []Evidence{{At: at, Said: said}},
|
||||
Source: o.Source, Raised: now, LastObserved: at, Observations: observations, Count: 1,
|
||||
Resolver: orSelf(o.Resolver)}
|
||||
change := ChangeRaised
|
||||
k.mu.Lock()
|
||||
@@ -244,7 +249,10 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
|
||||
}
|
||||
// The kind as the source says it now: a source that gave the same key a kind of its own since
|
||||
// (a probe's finding split out for a healer) is read by that kind from its next observation.
|
||||
c.Kind, c.Summary, c.Source, c.LastObserved = o.Kind, o.Summary, o.Source, now
|
||||
// A record read again with nothing newer in it is not observed again: its words are refreshed,
|
||||
// its count, evidence and last observation stay (novox/hq issue 402).
|
||||
fresh := o.Happened.IsZero() || at.After(c.LastObserved)
|
||||
c.Kind, c.Summary, c.Source = o.Kind, o.Summary, o.Source
|
||||
wasNeeds, wasActions := c.Needs, c.Actions
|
||||
c.Headline, c.Explanation, c.Resolved, c.Needs, c.Actions = o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions
|
||||
escalatedWords(&c)
|
||||
@@ -257,10 +265,15 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
|
||||
if len(o.Also) > 0 {
|
||||
c.Subject.Also = o.Also
|
||||
}
|
||||
c.Observations++
|
||||
c.Evidence = append([]Evidence{{At: now, Said: said}}, c.Evidence...)
|
||||
if len(c.Evidence) > KeptEvidence {
|
||||
c.Evidence = c.Evidence[:KeptEvidence]
|
||||
if fresh {
|
||||
// A record counts what happened since it began; the condition never counts down, so a
|
||||
// record begun again (a controller restarted) still adds the one it shows.
|
||||
c.Observations = max(c.Observations+1, o.Times)
|
||||
c.LastObserved = at
|
||||
c.Evidence = append([]Evidence{{At: at, Said: said}}, c.Evidence...)
|
||||
if len(c.Evidence) > KeptEvidence {
|
||||
c.Evidence = c.Evidence[:KeptEvidence]
|
||||
}
|
||||
}
|
||||
if err := k.stamp(&c); err != nil {
|
||||
return Condition{}, err
|
||||
|
||||
@@ -494,3 +494,19 @@ func TestASecondReopeningKeepsBothGaps(t *testing.T) {
|
||||
t.Fatalf("open at the wrong moments: %+v", g)
|
||||
}
|
||||
}
|
||||
|
||||
// **Times is how often a record says it happened, never alone** (novox/hq issue 440). The raise read
|
||||
// Times only beside Happened while an update read it always, so a source setting Times alone would have
|
||||
// jumped the count on its second look and not its first. The keeper refuses it, saying why.
|
||||
func TestTimesWithoutWhenItHappenedIsRefused(t *testing.T) {
|
||||
k, store, _, _ := keeper(t)
|
||||
o := Observation{Scope: ScopeMachine, ID: "ace", Kind: "silent", Machine: "ace", Severity: Warning,
|
||||
Summary: "ace has not been heard from", Source: "S1", Times: 5}
|
||||
_, err := k.Observe(t.Context(), o)
|
||||
if err == nil || !strings.Contains(err.Error(), "when") {
|
||||
t.Fatalf("Times without Happened was taken: %v", err)
|
||||
}
|
||||
if _, found, _ := ReadOne(t.Context(), store, o.Key()); found {
|
||||
t.Fatal("a refused observation raised its condition")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -94,7 +94,9 @@ type DataChange struct {
|
||||
}
|
||||
|
||||
// readingEvery is how often a measurement is kept as a reading: the shrink is read over days, and a
|
||||
// row every five minutes would say the same thing sixty times an hour.
|
||||
// row every five minutes would say the same thing sixty times an hour. A reading keeps the item's path
|
||||
// as it stands after the measurement — the holder's, or the last one known when it named none — and an
|
||||
// item at a path with no recent reading is read at once (novox/hq issue 368).
|
||||
const readingEvery = 55 * time.Minute
|
||||
|
||||
// readingsKept is how long readings are kept.
|
||||
@@ -187,10 +189,14 @@ func (i *Inventory) RecordData(ctx context.Context, machine string, declared []D
|
||||
}
|
||||
if m.Size != nil && m.MeasuredAt != nil && Comparable(m.Precision) {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
insert into data_reading (machine, module, item, at, size_bytes, last_write)
|
||||
select $1, $2, $3, $4, $5, $6
|
||||
where not exists (select 1 from data_reading
|
||||
where machine = $1 and module = $2 and item = $3 and at > $4::timestamptz - $7::interval)`,
|
||||
insert into data_reading (machine, module, item, at, size_bytes, last_write, path)
|
||||
select $1, $2, $3, $4, $5, $6, d.path
|
||||
from data_item d
|
||||
where d.machine = $1 and d.module = $2 and d.item = $3
|
||||
and not exists (select 1 from data_reading
|
||||
where machine = $1 and module = $2 and item = $3 and path = d.path
|
||||
and at > $4::timestamptz - $7::interval)
|
||||
on conflict (machine, module, item, at) do nothing`,
|
||||
machine, d.Module, d.Item, *m.MeasuredAt, *m.Size, m.LastWrite,
|
||||
fmt.Sprintf("%d seconds", int(readingEvery.Seconds()))); err != nil {
|
||||
return change, err
|
||||
@@ -281,10 +287,14 @@ func (i *Inventory) DataOf(ctx context.Context, machine, module, item string) (D
|
||||
return r, err
|
||||
}
|
||||
|
||||
// DataPeaks is each item's largest reading since a moment, keyed by DataRecord.Key.
|
||||
// DataPeaks is each item's largest reading since a moment, keyed by DataRecord.Key — read only from
|
||||
// readings at the item's path now (novox/hq issue 368): a directory is never compared with another one
|
||||
// that once held the same item, so a moved item starts its size history again at its new path.
|
||||
func (i *Inventory) DataPeaks(ctx context.Context, since time.Time) (map[string]int64, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, `select machine, module, item, max(size_bytes) from data_reading
|
||||
where at >= $1 group by machine, module, item`, since)
|
||||
rows, err := i.store.Pool().Query(ctx, `select r.machine, r.module, r.item, max(r.size_bytes)
|
||||
from data_reading r
|
||||
join data_item d on d.machine = r.machine and d.module = r.module and d.item = r.item and d.path = r.path
|
||||
where r.at >= $1 group by r.machine, r.module, r.item`, since)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -128,3 +128,66 @@ func TestAPartialMeasurementIsNeverAReading(t *testing.T) {
|
||||
t.Fatalf("%+v, %v", r, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A shrink is read against what the same directory held (novox/hq issue 368): an item whose path moved
|
||||
// — an agent's home moved to its own account — starts its size history again at the new path, and a
|
||||
// genuine shrink at the new path is still read against what that path held.
|
||||
func TestThePeakIsReadAtTheItemsPathOnly(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
ctx := t.Context()
|
||||
now := time.Date(2026, 10, 10, 0, 0, 0, 0, time.UTC)
|
||||
declared := []DeclaredData{{Module: "claude-code", Item: "agent-home", Class: "valuable", Owned: true}}
|
||||
measure := func(when time.Time, path string, s int64) {
|
||||
t.Helper()
|
||||
if _, err := inv.RecordData(ctx, "novox", declared, map[string]map[string]Measurement{"claude-code": {
|
||||
"agent-home": {Path: path, Size: size(s), MeasuredAt: at(when)}}}, "", when); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
measure(now, "/home/operator/.claude", 94_700_000)
|
||||
// The path moves ten minutes later: the new directory is measured at once, not an hour on.
|
||||
measure(now.Add(10*time.Minute), "/home/agent/.claude", 490)
|
||||
peaks, err := inv.DataPeaks(ctx, now.Add(-time.Hour))
|
||||
if err != nil || peaks["novox/claude-code/agent-home"] != 490 {
|
||||
t.Fatalf("after the path moved the peak is %v (%v), want 490: the old directory's size is no shrink "+
|
||||
"of the new one", peaks, err)
|
||||
}
|
||||
// The new directory grows, then genuinely loses most of it: that is read against the new path's peak.
|
||||
measure(now.Add(2*time.Hour), "/home/agent/.claude", 80_000_000)
|
||||
measure(now.Add(4*time.Hour), "/home/agent/.claude", 1_000)
|
||||
peaks, err = inv.DataPeaks(ctx, now.Add(-time.Hour))
|
||||
if err != nil || peaks["novox/claude-code/agent-home"] != 80_000_000 {
|
||||
t.Fatalf("a shrink at the new path is read against %v (%v), want 80000000", peaks, err)
|
||||
}
|
||||
// A measurement that names no path is the item's last known path's, not a new history.
|
||||
measure(now.Add(6*time.Hour), "", 2_000)
|
||||
peaks, err = inv.DataPeaks(ctx, now.Add(-time.Hour))
|
||||
if err != nil || peaks["novox/claude-code/agent-home"] != 80_000_000 {
|
||||
t.Fatalf("a measurement with no path started a new history: peak %v (%v)", peaks, err)
|
||||
}
|
||||
// Moving back to a directory measured before reads it against what it held then.
|
||||
measure(now.Add(8*time.Hour), "/home/operator/.claude", 94_000_000)
|
||||
peaks, err = inv.DataPeaks(ctx, now.Add(-time.Hour))
|
||||
if err != nil || peaks["novox/claude-code/agent-home"] != 94_700_000 {
|
||||
t.Fatalf("back at the first path the peak is %v (%v), want 94700000", peaks, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Two measurements at the same moment at two paths keep one reading and fail nothing: the second
|
||||
// would otherwise break the reading's key and lose the machine's whole record (issue 368 review).
|
||||
func TestTwoPathsAtOneMomentFailNothing(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
ctx := t.Context()
|
||||
now := time.Date(2026, 10, 10, 0, 0, 0, 0, time.UTC)
|
||||
declared := []DeclaredData{{Module: "claude-code", Item: "agent-home", Class: "valuable", Owned: true}}
|
||||
for _, path := range []string{"/home/operator/.claude", "/home/agent/.claude"} {
|
||||
if _, err := inv.RecordData(ctx, "novox", declared, map[string]map[string]Measurement{"claude-code": {
|
||||
"agent-home": {Path: path, Size: size(100), MeasuredAt: at(now)}}}, "", now); err != nil {
|
||||
t.Fatalf("a measurement at %s failed: %v", path, err)
|
||||
}
|
||||
}
|
||||
r, err := inv.DataOf(ctx, "novox", "claude-code", "agent-home")
|
||||
if err != nil || r.Path != "/home/agent/.claude" {
|
||||
t.Fatalf("%+v, %v", r, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,10 +17,13 @@ const (
|
||||
DurationHeartbeatGap = "heartbeat-gap"
|
||||
DurationPlanTier = "plan-tier"
|
||||
DurationBuild = "build"
|
||||
// DurationWalkPhase is one phase of an ended walk, per class (novox/hq ADR 0282 decision 6): subject
|
||||
// "<class>/<phase>", and "<class>/total" for its merge to every machine running it.
|
||||
DurationWalkPhase = "walk-phase"
|
||||
)
|
||||
|
||||
// DurationKinds are every kind, in the order `durations` shows them.
|
||||
var DurationKinds = []string{DurationApply, DurationHeartbeatGap, DurationPlanTier, DurationBuild}
|
||||
var DurationKinds = []string{DurationApply, DurationHeartbeatGap, DurationPlanTier, DurationBuild, DurationWalkPhase}
|
||||
|
||||
// DurationsKeptFor is how long a duration is kept: long enough to set a bound from, and to correct it
|
||||
// in Phase 1's first live week.
|
||||
@@ -100,6 +103,14 @@ func (i *Inventory) Durations(ctx context.Context, kind string, since time.Time)
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// WalkPhasesKept says whether any phase of a walk is kept as a walk-phase duration, and whether its total is.
|
||||
func (i *Inventory) WalkPhasesKept(ctx context.Context, plan string) (anyKept, totalKept bool, err error) {
|
||||
err = i.store.Pool().QueryRow(ctx,
|
||||
`select count(*) > 0, count(*) filter (where ref = $2) > 0 from duration where kind = $1 and ref like $3`,
|
||||
DurationWalkPhase, plan+"/total", plan+"/%").Scan(&anyKept, &totalKept)
|
||||
return anyKept, totalKept, err
|
||||
}
|
||||
|
||||
// ForgetOldDurations removes what is older than DurationsKeptFor, and says how many.
|
||||
func (i *Inventory) ForgetOldDurations(ctx context.Context) (int64, error) {
|
||||
tag, err := i.store.Pool().Exec(ctx, `delete from duration where recorded < $1`,
|
||||
|
||||
@@ -155,7 +155,7 @@ func (i *Inventory) ReplaceGivenAfterStart(ctx context.Context, node, declared s
|
||||
}
|
||||
// The definition is asked again now, not only when the value was given: one that has since
|
||||
// said the value is an outside party's, or applied, keeps it as given, and the mark stays gone.
|
||||
if own, ok := m.OwnSecrets[d.name]; !ok || !own.MeshMayMake() {
|
||||
if own, _, ok := m.OwnSecrets.Lookup(d.name); !ok || !own.MeshMayMake() {
|
||||
continue
|
||||
}
|
||||
if err := i.remakeOwn(ctx, d.nodeID, key, m, d.module, d.name); err != nil {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user