Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
dc5a8208a2 | ||
|
|
a5a355aeec |
@@ -3,6 +3,3 @@
|
|||||||
# A built binary. The lab writes one here when pointed at the repository root by mistake;
|
# A built binary. The lab writes one here when pointed at the repository root by mistake;
|
||||||
# built artifacts belong in build/, which is already ignored.
|
# built artifacts belong in build/, which is already ignored.
|
||||||
/mesh-builder
|
/mesh-builder
|
||||||
/mesh-controller
|
|
||||||
/postgres-provisioner
|
|
||||||
/redis-provisioner
|
|
||||||
|
|||||||
@@ -99,22 +99,16 @@ proxy-image:
|
|||||||
@echo
|
@echo
|
||||||
@docker image inspect $(PROXY_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
@docker image inspect $(PROXY_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||||
|
|
||||||
# The whole check. Raises a database, runs the repository's own check against it -- merge-check.sh,
|
# The whole gate. Raises a database, runs everything against it, and takes it down again --
|
||||||
# the very script `mesh/repo-check` runs -- and takes the database down again, including when the
|
# including when the tests fail, which is why the teardown is not conditional.
|
||||||
# check fails, which is why the teardown is not conditional and the script's exit status is the
|
|
||||||
# target's.
|
|
||||||
#
|
#
|
||||||
# **It calls the script rather than restating it** (novox/hq issue 431): this target used to list its
|
# **Packages in parallel, under the race detector, each test on a bus of its own** (internal/testbus).
|
||||||
# own steps, its formatting step walked vendor/ where the script's does not, and it failed on
|
# It was one package at a time against one shared bus, because the live tests assert, read and remove
|
||||||
# third-party code no change could fix -- so a developer's check and the gate disagreed, and the steps
|
# the mesh's own objects by their fixed names, and two packages at once deleted what the other read; the
|
||||||
# after formatting never ran through it. The script is the one list of steps; this target only gives it
|
# suite was red run as Go runs it and read as noise. A bus per test, of the release the mesh runs, made
|
||||||
# a database (MESH_TEST_POSTGRES, exported above).
|
# it the same in any order. The timeout bounds a hang to a failure with a stack, never a stalled gate.
|
||||||
#
|
check: fmt vet postgres
|
||||||
# **Packages in parallel, under the race detector, each test on a bus of its own** (internal/testbus),
|
@go test -race -timeout 15m ./... ; status=$$? ; $(MAKE) postgres-stop ; exit $$status
|
||||||
# as the script runs them: a bus per test, of the release the mesh runs, makes the suite the same in any
|
|
||||||
# order, and the timeout bounds a hang to a failure with a stack, never a stalled gate.
|
|
||||||
check: postgres
|
|
||||||
@sh merge-check.sh ; status=$$? ; docker rm -f $(PG_CONTAINER) >/dev/null 2>&1 || true ; exit $$status
|
|
||||||
|
|
||||||
# Without a database the store's tests skip rather than fail, so this is the honest subset and not
|
# Without a database the store's tests skip rather than fail, so this is the honest subset and not
|
||||||
# the gate.
|
# the gate.
|
||||||
@@ -124,10 +118,8 @@ test:
|
|||||||
vet:
|
vet:
|
||||||
go vet ./...
|
go vet ./...
|
||||||
|
|
||||||
# Quick steps for a developer, not part of `check`. The directories gofmt reads must be the ones
|
|
||||||
# merge-check.sh lists, never `.`, which walks vendor/ (novox/hq issue 431).
|
|
||||||
fmt:
|
fmt:
|
||||||
@unformatted=$$(gofmt -l cmd internal examples) ; \
|
@unformatted=$$(gofmt -l . 2>/dev/null) ; \
|
||||||
if [ -n "$$unformatted" ] ; then echo "not gofmt'd:" ; echo "$$unformatted" ; exit 1 ; fi
|
if [ -n "$$unformatted" ] ; then echo "not gofmt'd:" ; echo "$$unformatted" ; exit 1 ; fi
|
||||||
|
|
||||||
postgres:
|
postgres:
|
||||||
|
|||||||
@@ -408,9 +408,6 @@ func comparisonOf(module string, c comparison, opts takeOptions) (preview string
|
|||||||
switch {
|
switch {
|
||||||
case sec.Origin == inventory.OriginAccepted:
|
case sec.Origin == inventory.OriginAccepted:
|
||||||
fmt.Fprintf(&b, " %s %s: accepted from a person, carried in as it is\n", what, name)
|
fmt.Fprintf(&b, " %s %s: accepted from a person, carried in as it is\n", what, name)
|
||||||
case sec.Origin == inventory.OriginIssued:
|
|
||||||
// Taken from the provider's software (novox/hq ADR 0305): the value the running service uses.
|
|
||||||
fmt.Fprintf(&b, " %s %s: issued by the provider's software, carried in as it is\n", what, name)
|
|
||||||
case opts.Mint[sec.Name]:
|
case opts.Mint[sec.Name]:
|
||||||
fmt.Fprintf(&b, " %s %s: minted by the mesh; the service takes the new value, as --mint said\n", what, name)
|
fmt.Fprintf(&b, " %s %s: minted by the mesh; the service takes the new value, as --mint said\n", what, name)
|
||||||
case foundData:
|
case foundData:
|
||||||
|
|||||||
@@ -1,180 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **A refusal the bus said once is counted once** (novox/hq issue 402). The bus refused the controller
|
|
||||||
// one publish at 18:52:59 UTC on 2026-10-10 and never again, yet S9 looked at the one remembered
|
|
||||||
// advisory every half minute for the hour it is kept, and the condition said "observed 15 time(s), last
|
|
||||||
// 13s ago" with a refusal in its evidence every 30 seconds: two of us read it as a refusal going on and
|
|
||||||
// went looking for a missing grant. The condition counts what the bus said, with when it last said it,
|
|
||||||
// never how often the controller looked.
|
|
||||||
func TestARefusalSaidOnceIsCountedOnceHoweverOftenItIsLookedAt(t *testing.T) {
|
|
||||||
refused := time.Date(2026, 10, 10, 18, 52, 59, 0, time.UTC)
|
|
||||||
now := refused.Add(10 * time.Second)
|
|
||||||
store := conditions.NewInMemory()
|
|
||||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
|
||||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
|
||||||
t.Cleanup(func() { k.Close(t.Context()) })
|
|
||||||
|
|
||||||
// The words the live condition bus.controller.refused carried on 2026-10-10.
|
|
||||||
said := "the bus refused the controller: nats: permissions violation: Permissions Violation for " +
|
|
||||||
`Publish to "mesh.seat.mesh-delivery.tool.times"`
|
|
||||||
advisory := link.Advisory{Kind: link.AdvisoryRefused, ID: "controller", Said: said,
|
|
||||||
First: refused, Last: refused, Count: 1}
|
|
||||||
look := func() conditions.Condition {
|
|
||||||
t.Helper()
|
|
||||||
f := &signalFacts{now: now, host: "novox", advisories: []link.Advisory{advisory}}
|
|
||||||
if err := k.Reconcile(t.Context(), "S9", watchAdvisories(f)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
c, found, err := conditions.ReadOne(t.Context(), store, "bus.controller.refused")
|
|
||||||
if err != nil || !found {
|
|
||||||
t.Fatalf("bus.controller.refused: found %v, %v", found, err)
|
|
||||||
}
|
|
||||||
return c
|
|
||||||
}
|
|
||||||
|
|
||||||
var c conditions.Condition
|
|
||||||
for range 15 {
|
|
||||||
c = look()
|
|
||||||
now = now.Add(30 * time.Second)
|
|
||||||
}
|
|
||||||
if c.Observations != 1 || len(c.Evidence) != 1 {
|
|
||||||
t.Fatalf("one refusal, looked at 15 times, reads as observed %d time(s) with %d evidence lines: %+v",
|
|
||||||
c.Observations, len(c.Evidence), c.Evidence)
|
|
||||||
}
|
|
||||||
if !c.LastObserved.Equal(refused) || !c.Evidence[0].At.Equal(refused) {
|
|
||||||
t.Fatalf("last observed %s, evidence at %s: the refusal was at %s", c.LastObserved, c.Evidence[0].At, refused)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The bus refuses it three times more between two looks: the condition counts four refusals, says
|
|
||||||
// the newest, and adds one line of evidence for the look that saw them.
|
|
||||||
again := now.Add(-5 * time.Second)
|
|
||||||
advisory.Last, advisory.Count = again, 4
|
|
||||||
c = look()
|
|
||||||
if c.Observations != 4 || len(c.Evidence) != 2 || !c.LastObserved.Equal(again) || !c.Evidence[0].At.Equal(again) {
|
|
||||||
t.Fatalf("four refusals read as observed %d time(s), last %s, evidence %+v", c.Observations, c.LastObserved, c.Evidence)
|
|
||||||
}
|
|
||||||
if !strings.Contains(c.Summary, "(4 times since 18:52 UTC)") {
|
|
||||||
t.Fatalf("summary %q", c.Summary)
|
|
||||||
}
|
|
||||||
now = now.Add(30 * time.Second)
|
|
||||||
if c = look(); c.Observations != 4 || len(c.Evidence) != 2 {
|
|
||||||
t.Fatalf("a look with nothing new counted: observed %d time(s), evidence %+v", c.Observations, c.Evidence)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **A condition that looks at the present still counts its looks** (novox/hq issue 402 changes only
|
|
||||||
// what reads a record): a machine silent for three looks was observed three times, and says so.
|
|
||||||
func TestAConditionOfThePresentCountsItsLooks(t *testing.T) {
|
|
||||||
now := time.Date(2026, 10, 10, 19, 0, 0, 0, time.UTC)
|
|
||||||
store := conditions.NewInMemory()
|
|
||||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
|
||||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
|
||||||
t.Cleanup(func() { k.Close(t.Context()) })
|
|
||||||
var c conditions.Condition
|
|
||||||
for range 3 {
|
|
||||||
var err error
|
|
||||||
if c, err = k.Observe(t.Context(), conditions.Observation{Scope: conditions.ScopeMachine, ID: "ace",
|
|
||||||
Kind: "silent", Machine: "ace", Severity: conditions.Warning, Summary: "ace has not been heard from",
|
|
||||||
Source: "S1"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
now = now.Add(30 * time.Second)
|
|
||||||
}
|
|
||||||
if c.Observations != 3 || len(c.Evidence) != 3 || !c.LastObserved.Equal(now.Add(-30*time.Second)) {
|
|
||||||
t.Fatalf("observed %d time(s), %d evidence lines, last %s", c.Observations, len(c.Evidence), c.LastObserved)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **One key, one watcher** (novox/hq issue 440). A consumer's max-deliveries condition is said from
|
|
||||||
// DEAD_LETTERS while it holds a message the consumer gave up on (issue 330). A max-deliveries advisory
|
|
||||||
// without a token names the same key; read beside it, each look added an observation and a line of
|
|
||||||
// evidence through the dead-letter half, and the two summaries overwrote each other on every look. The
|
|
||||||
// dead-letter row alone says that key, and counts the messages given up on, never the looks.
|
|
||||||
func TestAHeldDeadLetterIsCountedByWhatWasGivenUpNotByTheLooks(t *testing.T) {
|
|
||||||
gaveUp := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
|
|
||||||
now := gaveUp.Add(20 * time.Second)
|
|
||||||
store := conditions.NewInMemory()
|
|
||||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
|
||||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
|
||||||
t.Cleanup(func() { k.Close(t.Context()) })
|
|
||||||
|
|
||||||
const key = "bus.EVENTS.media_sonarr.max-deliveries"
|
|
||||||
held := map[string]int{"EVENTS.media_sonarr": 1}
|
|
||||||
newest := map[string]time.Time{"EVENTS.media_sonarr": gaveUp}
|
|
||||||
advisory := link.Advisory{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.media_sonarr", Stream: "EVENTS",
|
|
||||||
Consumer: "media_sonarr", Said: "sonarr on media handed message 7 over 5 times and gave up on it",
|
|
||||||
First: gaveUp, Last: gaveUp, Count: 1}
|
|
||||||
look := func() conditions.Condition {
|
|
||||||
t.Helper()
|
|
||||||
f := &signalFacts{now: now, host: "novox", advisories: []link.Advisory{advisory},
|
|
||||||
deadLetters: held, deadLettersNewest: newest}
|
|
||||||
if err := k.Reconcile(t.Context(), "S9", watchAdvisories(f)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
c, found, err := conditions.ReadOne(t.Context(), store, key)
|
|
||||||
if err != nil || !found {
|
|
||||||
t.Fatalf("%s: found %v, %v", key, found, err)
|
|
||||||
}
|
|
||||||
if !strings.Contains(c.Summary, "dead-letters verb") {
|
|
||||||
t.Fatalf("the summary is not the dead-letter row's: %q", c.Summary)
|
|
||||||
}
|
|
||||||
return c
|
|
||||||
}
|
|
||||||
|
|
||||||
var c conditions.Condition
|
|
||||||
for range 5 {
|
|
||||||
c = look()
|
|
||||||
now = now.Add(30 * time.Second)
|
|
||||||
}
|
|
||||||
if c.Observations != 1 || len(c.Evidence) != 1 || !c.LastObserved.Equal(gaveUp) {
|
|
||||||
t.Fatalf("one message given up on, looked at five times, reads as observed %d time(s), last %s, "+
|
|
||||||
"with %d evidence lines: %+v", c.Observations, c.LastObserved, len(c.Evidence), c.Evidence)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The consumer gives up on a second message between two looks: two given up on, one more line.
|
|
||||||
again := now.Add(-10 * time.Second)
|
|
||||||
held["EVENTS.media_sonarr"], newest["EVENTS.media_sonarr"] = 2, again
|
|
||||||
c = look()
|
|
||||||
if c.Observations != 2 || len(c.Evidence) != 2 || !c.LastObserved.Equal(again) {
|
|
||||||
t.Fatalf("two given up on read as observed %d time(s), last %s, evidence %+v", c.Observations,
|
|
||||||
c.LastObserved, c.Evidence)
|
|
||||||
}
|
|
||||||
now = now.Add(30 * time.Second)
|
|
||||||
if c = look(); c.Observations != 2 || len(c.Evidence) != 2 {
|
|
||||||
t.Fatalf("a look with nothing new counted: observed %d time(s), evidence %+v", c.Observations, c.Evidence)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The advisory still says the max-deliveries it alone knows: a message given up on that could not be
|
|
||||||
// kept, under a key of its own (issue 330), which issue 440's change leaves as it was.
|
|
||||||
func TestAMessageThatCouldNotBeKeptIsStillSaidFromTheAdvisory(t *testing.T) {
|
|
||||||
at := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
|
|
||||||
f := &signalFacts{now: at, host: "novox", advisories: []link.Advisory{{Kind: link.AdvisoryMaxDeliveries,
|
|
||||||
ID: "EVENTS.media_sonarr", Stream: "EVENTS", Consumer: "media_sonarr", Token: link.AdvisoryNotKept,
|
|
||||||
Said: "sonarr on media gave up on message 7, and it could not be kept", First: at, Last: at, Count: 1}}}
|
|
||||||
said := watchAdvisories(f)
|
|
||||||
if len(said) != 1 || said[0].Key() != "bus.EVENTS.media_sonarr.not-kept" {
|
|
||||||
t.Fatalf("%+v", said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A max-deliveries advisory without a token names a consumer's dead-letter key, which only DEAD_LETTERS
|
|
||||||
// says (novox/hq issues 330 and 440): with nothing held, the advisory alone raises nothing.
|
|
||||||
func TestAMaxDeliveriesAdvisoryAloneRaisesNothing(t *testing.T) {
|
|
||||||
at := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
|
|
||||||
f := &signalFacts{now: at, host: "novox", advisories: []link.Advisory{{Kind: link.AdvisoryMaxDeliveries,
|
|
||||||
ID: "EVENTS.media_sonarr", Stream: "EVENTS", Consumer: "media_sonarr",
|
|
||||||
Said: "sonarr on media handed message 7 over 5 times and gave up on it", First: at, Last: at, Count: 1}}}
|
|
||||||
if said := watchAdvisories(f); len(said) != 0 {
|
|
||||||
t.Fatalf("said %+v", said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -367,13 +367,7 @@ func keepBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan,
|
|||||||
}
|
}
|
||||||
closes, latest := windowOf(merges, b.Created, window, atMost)
|
closes, latest := windowOf(merges, b.Created, window, atMost)
|
||||||
b.Delivery.Merges = named
|
b.Delivery.Merges = named
|
||||||
var closed bool
|
b.Delivery.Batch = &inventory.PlanBatch{ClosesAt: closes, AtMost: latest, Behind: behind, Own: b.OwnPath()}
|
||||||
var withheld []string
|
|
||||||
if was := b.Delivery.Batch; was != nil {
|
|
||||||
closed, withheld = was.Closed, was.Withheld // a stop's, which no look takes back (novox/hq ADR 0299)
|
|
||||||
}
|
|
||||||
b.Delivery.Batch = &inventory.PlanBatch{ClosesAt: closes, AtMost: latest, Behind: behind, Own: b.OwnPath(),
|
|
||||||
Closed: closed, Withheld: withheld}
|
|
||||||
b.State = inventory.PlanAssembling
|
b.State = inventory.PlanAssembling
|
||||||
if behind != "" && windowClosed(*b, now) {
|
if behind != "" && windowClosed(*b, now) {
|
||||||
b.State = inventory.PlanQueued
|
b.State = inventory.PlanQueued
|
||||||
@@ -408,7 +402,7 @@ func windowClosed(b inventory.Plan, now time.Time) bool {
|
|||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
w := b.Delivery.Batch
|
w := b.Delivery.Batch
|
||||||
return w.Closed || !now.Before(w.ClosesAt) || !now.Before(w.AtMost)
|
return !now.Before(w.ClosesAt) || !now.Before(w.AtMost)
|
||||||
}
|
}
|
||||||
|
|
||||||
// carriedOf is a set of merges as a walk carries them: the latest merge of each repository's branch, and
|
// carriedOf is a set of merges as a walk carries them: the latest merge of each repository's branch, and
|
||||||
@@ -484,8 +478,7 @@ func spelledOf(m inventory.BatchedMerge) string {
|
|||||||
// namedOf is one merge as a walk or batch names it: its commit, and its pull request and moves as announced.
|
// namedOf is one merge as a walk or batch names it: its commit, and its pull request and moves as announced.
|
||||||
func namedOf(m inventory.BatchedMerge, repository string) inventory.PlanMerge {
|
func namedOf(m inventory.BatchedMerge, repository string) inventory.PlanMerge {
|
||||||
k := announcedOf(m)
|
k := announcedOf(m)
|
||||||
return inventory.PlanMerge{Repository: repository, Commit: m.Commit, Number: k.Number, Title: k.Title, Moves: k.Moves,
|
return inventory.PlanMerge{Repository: repository, Commit: m.Commit, Number: k.Number, Title: k.Title, Moves: k.Moves}
|
||||||
Merged: m.Merged, Heard: m.Heard}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// laterMerge says a was merged after b: by the forge's merge time, then by when each was heard.
|
// laterMerge says a was merged after b: by the forge's merge time, then by when each was heard.
|
||||||
@@ -768,18 +761,7 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
|
|||||||
}
|
}
|
||||||
sort.Strings(also)
|
sort.Strings(also)
|
||||||
ordered := groupOrderEdges(members, entries, read, edges)
|
ordered := groupOrderEdges(members, entries, read, edges)
|
||||||
var withheld []string
|
|
||||||
plan := planOfMoves(moved, append(append([]inventory.Edge{}, edges...), ordered...))
|
plan := planOfMoves(moved, append(append([]inventory.Edge{}, edges...), ordered...))
|
||||||
// **Withheld, a stopped merge's modules** (novox/hq ADR 0299): a module is built at its branch, which holds the
|
|
||||||
// merge a stop left out of the walk these merges were walked again from — built here only as a dependent, it
|
|
||||||
// would deliver that merge. One a merge of this batch moves is built: that merge carries the stopped change,
|
|
||||||
// and the stop named it.
|
|
||||||
if batch.Delivery != nil && batch.Delivery.Batch != nil {
|
|
||||||
if out := withhold(&plan, batch.Delivery.Batch.Withheld, moved); len(out) > 0 {
|
|
||||||
withheld = out
|
|
||||||
fmt.Printf(" withheld, a stopped merge's modules built only as dependents: %s\n", strings.Join(out, ", "))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
plan.ID, plan.Revision, plan.Created, plan.Epoch = batch.ID, batch.Revision, now, batch.Epoch
|
plan.ID, plan.Revision, plan.Created, plan.Epoch = batch.ID, batch.Revision, now, batch.Epoch
|
||||||
plan.Commits = commits
|
plan.Commits = commits
|
||||||
newest := newestCommit(commits)
|
newest := newestCommit(commits)
|
||||||
@@ -790,9 +772,6 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
|
|||||||
}
|
}
|
||||||
plan.Delivery.Merges = named
|
plan.Delivery.Merges = named
|
||||||
plan.Delivery.Alone = batch.Delivery != nil && batch.Delivery.Alone
|
plan.Delivery.Alone = batch.Delivery != nil && batch.Delivery.Alone
|
||||||
plan.Delivery.Withheld = withheld
|
|
||||||
// **Its moments and its class** (novox/hq ADR 0282 decision 6): measured, never acted on.
|
|
||||||
plan.Times = walkTimesAtCut(*batch, plan, entries, now)
|
|
||||||
if len(moved) == 0 {
|
if len(moved) == 0 {
|
||||||
plan.State = inventory.PlanDone
|
plan.State = inventory.PlanDone
|
||||||
plan.Tiers = [][]string{}
|
plan.Tiers = [][]string{}
|
||||||
@@ -841,83 +820,6 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// withhold takes from a plan every module named that its merges do not move, and the tiers left empty: the
|
|
||||||
// modules taken.
|
|
||||||
func withhold(plan *inventory.Plan, names, moved []string) []string {
|
|
||||||
var out []string
|
|
||||||
for _, n := range names {
|
|
||||||
if _, in := plan.Modules[n]; in && !slices.Contains(moved, n) {
|
|
||||||
delete(plan.Modules, n)
|
|
||||||
out = append(out, n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(out) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
var tiers [][]string
|
|
||||||
for _, t := range plan.Tiers {
|
|
||||||
var kept []string
|
|
||||||
for _, n := range t {
|
|
||||||
if !slices.Contains(out, n) {
|
|
||||||
kept = append(kept, n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(kept) > 0 {
|
|
||||||
tiers = append(tiers, kept)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if tiers == nil {
|
|
||||||
tiers = [][]string{}
|
|
||||||
}
|
|
||||||
plan.Tiers = tiers
|
|
||||||
sort.Strings(out)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// walkTimesAtCut is a walk's own moments as it is cut (novox/hq ADR 0282 decision 6): when its batch's window
|
|
||||||
// closed — no merge for the window's length, or its maximum, whichever came first — when it was cut, and its
|
|
||||||
// class. A merge walked alone had no window.
|
|
||||||
func walkTimesAtCut(batch, walk inventory.Plan, entries []inventory.Entry, now time.Time) *inventory.PlanTimes {
|
|
||||||
cut := now
|
|
||||||
t := &inventory.PlanTimes{Cut: &cut, Class: classOf(walk, entries)}
|
|
||||||
if batch.Delivery != nil && batch.Delivery.Batch != nil {
|
|
||||||
w := batch.Delivery.Batch
|
|
||||||
closed := w.ClosesAt
|
|
||||||
if !w.AtMost.IsZero() && w.AtMost.Before(closed) {
|
|
||||||
closed = w.AtMost
|
|
||||||
}
|
|
||||||
if !closed.IsZero() {
|
|
||||||
if closed.After(now) {
|
|
||||||
closed = now
|
|
||||||
}
|
|
||||||
t.WindowClosed = &closed
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return t
|
|
||||||
}
|
|
||||||
|
|
||||||
// resolverSeat is the seat of the mesh's resolver: a module claiming it is a core module (ADR 0282 decision 1).
|
|
||||||
const resolverSeat = "mesh-dns-resolver"
|
|
||||||
|
|
||||||
// classOf is a walk's class (novox/hq ADR 0282 decision 1): core when it walks a module on the controller's own
|
|
||||||
// path or one holding the mesh's resolver, leaf otherwise.
|
|
||||||
func classOf(walk inventory.Plan, entries []inventory.Entry) string {
|
|
||||||
resolvers := map[string]bool{}
|
|
||||||
for _, e := range entries {
|
|
||||||
for _, c := range e.Manifest.Claims {
|
|
||||||
if c.Name == resolverSeat {
|
|
||||||
resolvers[e.Manifest.Module] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for m := range walk.Modules {
|
|
||||||
if _, own := onTheControllersPath[m]; own || resolvers[m] {
|
|
||||||
return inventory.ClassCore
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return inventory.ClassLeaf
|
|
||||||
}
|
|
||||||
|
|
||||||
// combinedMerges is a batch's merges as one merge per repository's branch: the latest, with every file the
|
// combinedMerges is a batch's merges as one merge per repository's branch: the latest, with every file the
|
||||||
// merges of it changed and said to be a module's — on a linear trunk the latest contains the others. A file is
|
// merges of it changed and said to be a module's — on a linear trunk the latest contains the others. A file is
|
||||||
// removed when the last merge of the batch that changed it removed it. merges are in the order they were made.
|
// removed when the last merge of the batch that changed it removed it. merges are in the order they were made.
|
||||||
|
|||||||
@@ -1,207 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **A filling bucket or log is said before it is full** (novox/hq ADR 0297 §6, issue 501).
|
|
||||||
//
|
|
||||||
// A module's bucket and its log each have a cap on the bus, and when either is full the bus refuses
|
|
||||||
// the next write: the module stops doing what it writes for, and nothing said so before. On
|
|
||||||
// 2026-10-11 the issue tracker's bucket held a sixth of its cap after two days, growing toward a stop
|
|
||||||
// nobody would have heard coming. So the self-check reads every module's bucket and log, and one at
|
|
||||||
// fillRaiseAt of its cap or more raises a condition naming the module, the bucket or log, and how full
|
|
||||||
// it is.
|
|
||||||
//
|
|
||||||
// **Cleared by observation below fillClearBelow, not below fillRaiseAt**: a bucket or log hovering at
|
|
||||||
// its threshold would otherwise be raised and cleared on every run. Between the two, an open condition
|
|
||||||
// is kept and none is raised.
|
|
||||||
//
|
|
||||||
// **One that cannot be read is said, and the rest are still judged.** An unanswered question about one
|
|
||||||
// stream is no reason to know nothing of the others: it is a finding of its own, naming the bucket or
|
|
||||||
// log and why, and a fill condition already open for it is kept, because not knowing is not a pass.
|
|
||||||
|
|
||||||
// The fill thresholds, in percent of a bucket's or log's cap.
|
|
||||||
const (
|
|
||||||
fillRaiseAt = 75
|
|
||||||
fillClearBelow = 70
|
|
||||||
)
|
|
||||||
|
|
||||||
// The conditions the fill probe raises: one filling, and one that could not be read.
|
|
||||||
const (
|
|
||||||
kindBucketOrLogFilling = "bucket-or-log-filling"
|
|
||||||
kindBucketOrLogUnread = "bucket-or-log-unread"
|
|
||||||
)
|
|
||||||
|
|
||||||
// probeFillID is the probe's id in the registry.
|
|
||||||
const probeFillID = "D-fill"
|
|
||||||
|
|
||||||
// bucketOrLog is one module's bucket or log as the fill probe reads it.
|
|
||||||
type bucketOrLog struct {
|
|
||||||
Module string
|
|
||||||
// Kind is `bucket` or `log`; Name its local name; Stream the stream it is on the bus.
|
|
||||||
Kind, Name, Stream string
|
|
||||||
}
|
|
||||||
|
|
||||||
// filled is how full one bucket or log is, read from the bus.
|
|
||||||
type filled struct {
|
|
||||||
Bytes, Max uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
// percent is how full, in whole percent, rounded down.
|
|
||||||
func (f filled) percent() uint64 {
|
|
||||||
if f.Max == 0 {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
return f.Bytes * 100 / f.Max
|
|
||||||
}
|
|
||||||
|
|
||||||
// fillKey is where a bucket's or log's fill condition is kept.
|
|
||||||
func fillKey(s bucketOrLog) string { return conditions.Key(conditions.ScopeBus, s.Stream, "filling") }
|
|
||||||
|
|
||||||
// fillObservation is what one bucket's or log's fill says: a finding at fillRaiseAt or above, and,
|
|
||||||
// while its condition is open, at fillClearBelow or above too; nothing otherwise, which is what clears
|
|
||||||
// it. The operator's words are the kind's (plain_words.go); the summary and the evidence name the
|
|
||||||
// module, the bucket or log, and the fill.
|
|
||||||
func fillObservation(s bucketOrLog, f filled, open bool) (conditions.Observation, bool) {
|
|
||||||
if f.Max == 0 {
|
|
||||||
return conditions.Observation{}, false // one with no cap cannot fill
|
|
||||||
}
|
|
||||||
// Compared in bytes, not rounded percent: 74.9% is not 75%.
|
|
||||||
atRaise := f.Bytes*100 >= f.Max*fillRaiseAt
|
|
||||||
aboveClear := f.Bytes*100 >= f.Max*fillClearBelow
|
|
||||||
if !atRaise && !(open && aboveClear) {
|
|
||||||
return conditions.Observation{}, false
|
|
||||||
}
|
|
||||||
pct := f.percent()
|
|
||||||
return conditions.Observation{
|
|
||||||
Scope: conditions.ScopeBus, ID: s.Stream, Token: "filling", Kind: kindBucketOrLogFilling,
|
|
||||||
Severity: conditions.Warning,
|
|
||||||
Summary: fmt.Sprintf("%s's %s %s holds %s of %s, %d%%: at its cap the bus refuses the module's next write",
|
|
||||||
s.Module, s.Kind, s.Name, mibWords(f.Bytes), mibWords(f.Max), pct),
|
|
||||||
Said: fmt.Sprintf("module %s, %s %s (stream %s): %d of %d bytes, %d%%", s.Module, s.Kind, s.Name,
|
|
||||||
s.Stream, f.Bytes, f.Max, pct),
|
|
||||||
}, true
|
|
||||||
}
|
|
||||||
|
|
||||||
// unreadObservation says one bucket or log could not be read, and why. Asked over the network, so one
|
|
||||||
// look can be wrong: raised on the second look in a row (confirm.go).
|
|
||||||
func unreadObservation(s bucketOrLog, why error) conditions.Observation {
|
|
||||||
return conditions.Observation{
|
|
||||||
Scope: conditions.ScopeBus, ID: s.Stream, Token: "unread", Kind: kindBucketOrLogUnread,
|
|
||||||
Severity: conditions.Warning, Confirm: true,
|
|
||||||
Summary: fmt.Sprintf("%s's %s %s could not be read, so how full it is is not known", s.Module, s.Kind, s.Name),
|
|
||||||
Said: fmt.Sprintf("module %s, %s %s (stream %s): %v", s.Module, s.Kind, s.Name, s.Stream, why),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// keptFilling is an open fill condition said again for a bucket or log that could not be read this
|
|
||||||
// time: not knowing how full it is now is no reason to say it has room. Only ever made from a condition
|
|
||||||
// read back as open — its own summary and severity, never words made up for it.
|
|
||||||
func keptFilling(s bucketOrLog, open conditions.Condition, why error) conditions.Observation {
|
|
||||||
return conditions.Observation{
|
|
||||||
Scope: conditions.ScopeBus, ID: s.Stream, Token: "filling", Kind: kindBucketOrLogFilling,
|
|
||||||
Severity: open.Severity, Summary: open.Summary,
|
|
||||||
Said: fmt.Sprintf("module %s, %s %s (stream %s): not read this time (%v); kept open as it was",
|
|
||||||
s.Module, s.Kind, s.Name, s.Stream, why),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// mibWords is a size as a person reads it, in MiB with one decimal.
|
|
||||||
func mibWords(b uint64) string {
|
|
||||||
return fmt.Sprintf("%.1f MiB", float64(b)/(1024*1024))
|
|
||||||
}
|
|
||||||
|
|
||||||
// readFill is how full one bucket or log is on the bus; found false when it is not on the bus.
|
|
||||||
type readFill func(ctx context.Context, s bucketOrLog) (f filled, found bool, err error)
|
|
||||||
|
|
||||||
// openFill is the fill condition open under a key, if one is, or why it could not be read.
|
|
||||||
type openFill func(ctx context.Context, key string) (conditions.Condition, bool, error)
|
|
||||||
|
|
||||||
// judgeFills judges every bucket and log on its own: one that cannot be read is said as unread, with
|
|
||||||
// its fill condition kept only when that condition was read back and is open — when it cannot be read
|
|
||||||
// either, nothing is said of its fill, which the unread finding already covers. Every other is judged
|
|
||||||
// by its fill; for one of those, a condition that cannot be read is taken as open, so an unknown never
|
|
||||||
// clears a fill measured between fillClearBelow and fillRaiseAt.
|
|
||||||
func judgeFills(ctx context.Context, all []bucketOrLog, read readFill, open openFill) []conditions.Observation {
|
|
||||||
var out []conditions.Observation
|
|
||||||
for _, s := range all {
|
|
||||||
f, found, err := read(ctx, s)
|
|
||||||
if err != nil {
|
|
||||||
out = append(out, unreadObservation(s, err))
|
|
||||||
if c, isOpen, cerr := open(ctx, fillKey(s)); cerr == nil && isOpen {
|
|
||||||
out = append(out, keptFilling(s, c, err))
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if !found {
|
|
||||||
continue // not on the bus: created on the controller's next raise, and nothing there can fill
|
|
||||||
}
|
|
||||||
_, isOpen, cerr := open(ctx, fillKey(s))
|
|
||||||
isOpen = isOpen || cerr != nil
|
|
||||||
if o, said := fillObservation(s, f, isOpen); said {
|
|
||||||
out = append(out, o)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.SliceStable(out, func(i, j int) bool { return out[i].Key() < out[j].Key() })
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// declaredBucketsAndLogs is every module's bucket and log the catalogue declares, by its stream.
|
|
||||||
func declaredBucketsAndLogs(ctx context.Context, d *doctor) ([]bucketOrLog, error) {
|
|
||||||
buckets, err := d.open.inventory.DeclaredBuckets(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
logs, err := d.open.inventory.DeclaredLogs(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var out []bucketOrLog
|
|
||||||
for _, b := range buckets {
|
|
||||||
out = append(out, bucketOrLog{Module: b.Module, Kind: "bucket", Name: b.Name, Stream: "KV_" + b.Bucket()})
|
|
||||||
}
|
|
||||||
for _, l := range logs {
|
|
||||||
out = append(out, bucketOrLog{Module: l.Module, Kind: "log", Name: l.Name, Stream: l.Stream()})
|
|
||||||
}
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// probeFill reads every module's bucket and log on the bus and says each one filling toward its cap,
|
|
||||||
// and each one that could not be read.
|
|
||||||
func probeFill(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
|
||||||
all, err := declaredBucketsAndLogs(ctx, d)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
js := d.js.Context()
|
|
||||||
read := func(ctx context.Context, s bucketOrLog) (filled, bool, error) {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return filled{}, false, err
|
|
||||||
}
|
|
||||||
info, err := js.StreamInfo(s.Stream, nats.Context(ctx))
|
|
||||||
switch {
|
|
||||||
case errors.Is(err, nats.ErrStreamNotFound):
|
|
||||||
return filled{}, false, nil
|
|
||||||
case err != nil:
|
|
||||||
return filled{}, false, err
|
|
||||||
case info.Config.MaxBytes <= 0:
|
|
||||||
return filled{}, true, nil
|
|
||||||
}
|
|
||||||
return filled{Bytes: info.State.Bytes, Max: uint64(info.Config.MaxBytes)}, true, nil
|
|
||||||
}
|
|
||||||
open := func(ctx context.Context, key string) (conditions.Condition, bool, error) {
|
|
||||||
if d.keeper == nil {
|
|
||||||
return conditions.Condition{}, false, nil
|
|
||||||
}
|
|
||||||
return d.keeper.Get(ctx, key)
|
|
||||||
}
|
|
||||||
return judgeFills(ctx, all, read, open), nil
|
|
||||||
}
|
|
||||||
@@ -1,165 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **A filling log or bucket is said at three quarters of its cap and cleared below seven tenths**
|
|
||||||
// (novox/hq ADR 0297 §6): raised at 75% naming the module, the bucket or log and its fill; not at 74.9%;
|
|
||||||
// kept between 70% and 75% while it is open, and not raised there when it is not; and cleared — said no
|
|
||||||
// more — once it reads below 70%, open or not.
|
|
||||||
func TestAFillingBucketOrLogIsRaisedAtThreeQuartersAndClearedBelowSevenTenths(t *testing.T) {
|
|
||||||
const mib = 1024 * 1024
|
|
||||||
log := bucketOrLog{Module: "mesh-issues", Kind: "log", Name: "changes", Stream: "LOG_mesh-issues_changes"}
|
|
||||||
bucket := bucketOrLog{Module: "mesh-issues", Kind: "bucket", Name: "issues", Stream: "KV_mesh-issues_issues"}
|
|
||||||
for _, c := range []struct {
|
|
||||||
name string
|
|
||||||
of bucketOrLog
|
|
||||||
bytes uint64
|
|
||||||
max uint64
|
|
||||||
open bool
|
|
||||||
said bool
|
|
||||||
}{
|
|
||||||
{"a log at exactly 75%", log, 768 * mib, 1024 * mib, false, true},
|
|
||||||
{"a bucket at exactly 75%", bucket, 48 * mib, 64 * mib, false, true},
|
|
||||||
{"a bucket full", bucket, 64 * mib, 64 * mib, false, true},
|
|
||||||
{"a log just under 75%", log, 768*mib - 1, 1024 * mib, false, false},
|
|
||||||
{"a bucket at 72%, not open", bucket, 64 * mib * 72 / 100, 64 * mib, false, false},
|
|
||||||
{"a bucket at 72%, open: kept", bucket, 64 * mib * 72 / 100, 64 * mib, true, true},
|
|
||||||
{"a log at exactly 70%, open: kept", log, 700 * mib, 1000 * mib, true, true},
|
|
||||||
{"a log just under 70%, open: cleared", log, 700*mib - 1, 1000 * mib, true, false},
|
|
||||||
{"a bucket at 10%, open: cleared", bucket, 64 * mib / 10, 64 * mib, true, false},
|
|
||||||
{"a bucket with no cap", bucket, 100, 0, true, false},
|
|
||||||
} {
|
|
||||||
o, said := fillObservation(c.of, filled{Bytes: c.bytes, Max: c.max}, c.open)
|
|
||||||
if said != c.said {
|
|
||||||
t.Errorf("%s: said %v, want %v", c.name, said, c.said)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if !said {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if o.Key() != fillKey(c.of) || o.Kind != kindBucketOrLogFilling || o.Severity != conditions.Warning {
|
|
||||||
t.Errorf("%s: raised as %s (%s, %s)", c.name, o.Key(), o.Kind, o.Severity)
|
|
||||||
}
|
|
||||||
for _, part := range []string{c.of.Module, c.of.Kind + " " + c.of.Name, "%", " of "} {
|
|
||||||
if !strings.Contains(o.Summary, part) {
|
|
||||||
t.Errorf("%s: does not name %q: %q", c.name, part, o.Summary)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !strings.Contains(o.Said, "bytes") {
|
|
||||||
t.Errorf("%s: the evidence does not say the fill in bytes: %q", c.name, o.Said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
o, _ := fillObservation(log, filled{Bytes: 800 * mib, Max: 1024 * mib}, false)
|
|
||||||
if !strings.Contains(o.Summary, "800.0 MiB of 1024.0 MiB, 78%") {
|
|
||||||
t.Errorf("the fill is said as %q", o.Summary)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **One bucket or log that cannot be read does not stop the others being judged** (review of #231): it is
|
|
||||||
// said as unread with its reason, a fill condition open for it is kept, and every other bucket and log
|
|
||||||
// is judged by its own fill.
|
|
||||||
func TestAnUnreadableBucketOrLogIsSaidAndTheRestAreStillJudged(t *testing.T) {
|
|
||||||
const mib = 1024 * 1024
|
|
||||||
broken := bucketOrLog{Module: "a", Kind: "bucket", Name: "broken", Stream: "KV_a_broken"}
|
|
||||||
brokenOpen := bucketOrLog{Module: "a", Kind: "log", Name: "was-filling", Stream: "LOG_a_was-filling"}
|
|
||||||
full := bucketOrLog{Module: "b", Kind: "log", Name: "changes", Stream: "LOG_b_changes"}
|
|
||||||
empty := bucketOrLog{Module: "c", Kind: "bucket", Name: "quiet", Stream: "KV_c_quiet"}
|
|
||||||
absent := bucketOrLog{Module: "d", Kind: "bucket", Name: "not-yet", Stream: "KV_d_not-yet"}
|
|
||||||
refusal := errors.New("the bus did not answer")
|
|
||||||
read := func(_ context.Context, s bucketOrLog) (filled, bool, error) {
|
|
||||||
switch s {
|
|
||||||
case broken, brokenOpen:
|
|
||||||
return filled{}, false, refusal
|
|
||||||
case full:
|
|
||||||
return filled{Bytes: 900 * mib, Max: 1000 * mib}, true, nil
|
|
||||||
case empty:
|
|
||||||
return filled{Bytes: 1, Max: 64 * mib}, true, nil
|
|
||||||
}
|
|
||||||
return filled{}, false, nil
|
|
||||||
}
|
|
||||||
open := func(_ context.Context, key string) (conditions.Condition, bool, error) {
|
|
||||||
if key == fillKey(brokenOpen) {
|
|
||||||
return conditions.Condition{Key: key, Severity: conditions.Warning,
|
|
||||||
Summary: "a's log was-filling holds 800.0 MiB of 1024.0 MiB, 78%"}, true, nil
|
|
||||||
}
|
|
||||||
return conditions.Condition{}, false, nil
|
|
||||||
}
|
|
||||||
got := map[string]conditions.Observation{}
|
|
||||||
for _, o := range judgeFills(context.Background(), []bucketOrLog{broken, brokenOpen, full, empty, absent}, read, open) {
|
|
||||||
got[o.Key()] = o
|
|
||||||
}
|
|
||||||
for _, s := range []bucketOrLog{broken, brokenOpen} {
|
|
||||||
o, said := got[conditions.Key(conditions.ScopeBus, s.Stream, "unread")]
|
|
||||||
if !said || o.Kind != kindBucketOrLogUnread || !o.Confirm || !strings.Contains(o.Said, refusal.Error()) {
|
|
||||||
t.Errorf("%s could not be read and was said as %+v", s.Stream, o)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if o, kept := got[fillKey(brokenOpen)]; !kept || !strings.Contains(o.Said, "kept open") {
|
|
||||||
t.Errorf("an open fill condition of a log not read this time was not kept: %+v", o)
|
|
||||||
}
|
|
||||||
if _, said := got[fillKey(broken)]; said {
|
|
||||||
t.Error("a bucket not read and not filling was said filling")
|
|
||||||
}
|
|
||||||
if o, said := got[fillKey(full)]; !said || o.Kind != kindBucketOrLogFilling {
|
|
||||||
t.Errorf("a log at 90%% beside an unreadable one was not judged: %+v", got)
|
|
||||||
}
|
|
||||||
if len(got) != 4 {
|
|
||||||
t.Errorf("said %d findings, want 4 (two unread, one kept, one filling): %v", len(got), got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Both kinds have plain words of their own, which hold to the plain rule (novox/hq ADR 0253).
|
|
||||||
func TestAFillingOrUnreadBucketOrLogIsSaidInPlainWords(t *testing.T) {
|
|
||||||
for _, kind := range []string{kindBucketOrLogFilling, kindBucketOrLogUnread} {
|
|
||||||
wording, has := plainWordings[kind]
|
|
||||||
if !has {
|
|
||||||
t.Errorf("%s has no plain words", kind)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if why, ok := conditions.PlainWords(wording(conditions.Observation{Kind: kind})); !ok {
|
|
||||||
t.Errorf("%s: %s", kind, why)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **When neither the bucket or log nor its condition can be read, nothing is said of its fill** (second
|
|
||||||
// review of #231): the unread finding covers it, and no fill is invented for it. For one that is read and
|
|
||||||
// measured between seven tenths and three quarters, a condition that cannot be read is taken as open, so
|
|
||||||
// an unknown never clears it.
|
|
||||||
func TestNothingIsSaidOfAFillWhenNeitherItNorItsConditionCanBeRead(t *testing.T) {
|
|
||||||
const mib = 1024 * 1024
|
|
||||||
broken := bucketOrLog{Module: "a", Kind: "log", Name: "changes", Stream: "LOG_a_changes"}
|
|
||||||
between := bucketOrLog{Module: "b", Kind: "bucket", Name: "issues", Stream: "KV_b_issues"}
|
|
||||||
read := func(_ context.Context, s bucketOrLog) (filled, bool, error) {
|
|
||||||
if s == broken {
|
|
||||||
return filled{}, false, errors.New("the bus did not answer")
|
|
||||||
}
|
|
||||||
return filled{Bytes: 72 * mib, Max: 100 * mib}, true, nil
|
|
||||||
}
|
|
||||||
open := func(context.Context, string) (conditions.Condition, bool, error) {
|
|
||||||
return conditions.Condition{}, false, errors.New("the conditions bucket did not answer")
|
|
||||||
}
|
|
||||||
got := map[string]conditions.Observation{}
|
|
||||||
for _, o := range judgeFills(context.Background(), []bucketOrLog{broken, between}, read, open) {
|
|
||||||
got[o.Key()] = o
|
|
||||||
}
|
|
||||||
if o, said := got[fillKey(broken)]; said {
|
|
||||||
t.Fatalf("a fill was said for a log whose fill and condition could not be read: %+v", o)
|
|
||||||
}
|
|
||||||
if _, said := got[conditions.Key(conditions.ScopeBus, broken.Stream, "unread")]; !said {
|
|
||||||
t.Error("the log that could not be read was not said as unread")
|
|
||||||
}
|
|
||||||
if _, kept := got[fillKey(between)]; !kept {
|
|
||||||
t.Error("a bucket at 72% whose condition could not be read was cleared")
|
|
||||||
}
|
|
||||||
if len(got) != 2 {
|
|
||||||
t.Errorf("said %d findings, want 2: %v", len(got), got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -835,9 +835,6 @@ type answers struct {
|
|||||||
// public name on the machine went dark. The holds were correct; they were recorded only in the
|
// public name on the machine went dark. The holds were correct; they were recorded only in the
|
||||||
// machine's own state file, and the one visible symptom was a count that did not add up.
|
// machine's own state file, and the one visible symptom was a count that did not add up.
|
||||||
untaken map[string]map[string]int
|
untaken map[string]map[string]int
|
||||||
// leftOut is, per machine, every module of its set its composition leaves out, and why (novox/hq issue
|
|
||||||
// 380): assigned and not applied, which every push said only in passing. Not well while there is any.
|
|
||||||
leftOut map[string][]leftOutModule
|
|
||||||
// unheld is every module on a machine whose resources are applied through a seat nothing on
|
// unheld is every module on a machine whose resources are applied through a seat nothing on
|
||||||
// that machine holds (novox/hq ADR 0207), with the modules that could hold it. Reported, not
|
// that machine holds (novox/hq ADR 0207), with the modules that could hold it. Reported, not
|
||||||
// refused, until the switch — and while there is any, the mesh is not all well: the order the
|
// refused, until the switch — and while there is any, the mesh is not all well: the order the
|
||||||
@@ -866,10 +863,6 @@ type answers struct {
|
|||||||
// applied, is not well: an assignment somebody made is not made yet, or will not be.
|
// applied, is not well: an assignment somebody made is not made yet, or will not be.
|
||||||
pending []inventory.PendingAssignment
|
pending []inventory.PendingAssignment
|
||||||
pendingUnread string
|
pendingUnread string
|
||||||
// lastSent is every machine's last recorded send, by name, with what the machine answered of it (novox/hq
|
|
||||||
// issue 485); lastSentUnread why it could not be read.
|
|
||||||
lastSent map[string]inventory.Send
|
|
||||||
lastSentUnread string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
|
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
|
||||||
|
|||||||
@@ -151,6 +151,13 @@ func busCommand(ctx context.Context, args []string) error {
|
|||||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||||
sub, args = args[0], args[1:]
|
sub, args = args[0], args[1:]
|
||||||
}
|
}
|
||||||
|
// The view's credential, a terminal line like a person's (bus_view.go).
|
||||||
|
switch sub {
|
||||||
|
case "view-credential":
|
||||||
|
return busViewCredential(ctx, args)
|
||||||
|
case "view-revoke":
|
||||||
|
return busViewRevoke(ctx, args)
|
||||||
|
}
|
||||||
set := flag.NewFlagSet("bus", flag.ContinueOnError)
|
set := flag.NewFlagSet("bus", flag.ContinueOnError)
|
||||||
snapshot := set.String("snapshot-taken", "", "where the streams' snapshot a person took is, while the mesh takes none itself")
|
snapshot := set.String("snapshot-taken", "", "where the streams' snapshot a person took is, while the mesh takes none itself")
|
||||||
reversible := set.Bool("reversible", false, "the new version can be undone by putting the old one back")
|
reversible := set.Bool("reversible", false, "the new version can be undone by putting the old one back")
|
||||||
@@ -160,14 +167,14 @@ func busCommand(ctx context.Context, args []string) error {
|
|||||||
if rest, err := parseAround(set, args); err != nil {
|
if rest, err := parseAround(set, args); err != nil {
|
||||||
return err
|
return err
|
||||||
} else if len(rest) > 0 {
|
} else if len(rest) > 0 {
|
||||||
return errors.New("bus [upgrade --why … --reversible|--irreversible [--snapshot-taken <where>]]")
|
return errors.New(busUsage)
|
||||||
}
|
}
|
||||||
switch sub {
|
switch sub {
|
||||||
case "":
|
case "":
|
||||||
return busStatus(ctx)
|
return busStatus(ctx)
|
||||||
case "upgrade":
|
case "upgrade":
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("bus says what a bus upgrade would do, or `bus upgrade` — not %q", sub)
|
return fmt.Errorf("bus says what a bus upgrade would do, or `bus upgrade`, `bus view-credential`, `bus view-revoke` — not %q", sub)
|
||||||
}
|
}
|
||||||
// Everything refused before anything is done.
|
// Everything refused before anything is done.
|
||||||
if err := why.require("bus upgrade"); err != nil {
|
if err := why.require("bus upgrade"); err != nil {
|
||||||
|
|||||||
@@ -0,0 +1,117 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The view's credential: the one read-only user a page in a browser connects to the bus as, over the
|
||||||
|
// bus module's WebSocket listener (novox/hq research 036, gap G1; broker.KindView).
|
||||||
|
//
|
||||||
|
// **A terminal line, like a person's credential** (operator.go): printed once, never stored — the mesh
|
||||||
|
// keeps a hash — and revoked by forgetting the row, which the next composition of the user list makes
|
||||||
|
// real. There is one view; issuing it again rotates its password.
|
||||||
|
const busUsage = "bus [upgrade --why … --reversible|--irreversible [--snapshot-taken <where>] | view-credential | view-revoke]"
|
||||||
|
|
||||||
|
// busWebSocketPort is the port the bus module's WebSocket listener is published on, mirrored from the
|
||||||
|
// nats module's manifest (its `bus-websocket` opening), because the credential names where to connect
|
||||||
|
// and the controller does not read the module's configuration. Reached across the overlay only: the
|
||||||
|
// opening is from the mesh, and the mesh's filter admits nothing else.
|
||||||
|
const busWebSocketPort = 4223
|
||||||
|
|
||||||
|
func busViewCredential(ctx context.Context, args []string) error {
|
||||||
|
if len(args) != 0 {
|
||||||
|
return errors.New("bus view-credential takes nothing: there is one view, and this prints its credential once")
|
||||||
|
}
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
// Refused here rather than at the next composition, where it would stop the whole file.
|
||||||
|
if _, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindView, PasswordHash: "x"}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: broker.ViewUser, Kind: inventory.BusView})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
where, err := broker.FromEnvironment()
|
||||||
|
if err != nil && !errors.Is(err, broker.ErrNotConfigured) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
host := where.Address
|
||||||
|
if h, _, err := net.SplitHostPort(where.Address); err == nil {
|
||||||
|
host = h
|
||||||
|
}
|
||||||
|
websocket := ""
|
||||||
|
if host != "" {
|
||||||
|
websocket = "ws://" + net.JoinHostPort(host, strconv.Itoa(busWebSocketPort))
|
||||||
|
}
|
||||||
|
held, err := json.Marshal(struct {
|
||||||
|
WebSocket string `json:"websocket,omitempty"`
|
||||||
|
URL string `json:"url,omitempty"`
|
||||||
|
Fingerprint string `json:"fingerprint,omitempty"`
|
||||||
|
User string `json:"user"`
|
||||||
|
Password string `json:"password"`
|
||||||
|
InboxPrefix string `json:"inbox_prefix"`
|
||||||
|
Hears []string `json:"hears"`
|
||||||
|
Reads string `json:"reads"`
|
||||||
|
HowToRead string `json:"how_to_read"`
|
||||||
|
}{
|
||||||
|
WebSocket: websocket, URL: "nats://" + where.Address, Fingerprint: where.Fingerprint,
|
||||||
|
User: broker.ViewUser, Password: password,
|
||||||
|
// The client must make its inboxes under the view's own prefix: its subscribe grant is
|
||||||
|
// `_INBOX.view.>` and no wider (design 25 §4), and a client's default inbox is not under it.
|
||||||
|
InboxPrefix: "_INBOX." + broker.ViewUser,
|
||||||
|
Hears: broker.ViewHears, Reads: broker.ViewBucket,
|
||||||
|
HowToRead: "direct reads only, no watch (a consumer is refused): list with a request to $JS.API.DIRECT.GET.KV_" +
|
||||||
|
broker.ViewBucket + ` carrying {"multi_last":["$KV.` + broker.ViewBucket + `.>"]}, answered until a 204 status; ` +
|
||||||
|
"read one key with $JS.API.DIRECT.GET.KV_" + broker.ViewBucket + ".$KV." + broker.ViewBucket + ".<number> " +
|
||||||
|
"(nats.js: kvm.open(bucket, {allow_direct: true}), never create); re-read the key an event's number names",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("issued the view, which hears %s and reads the bucket %s, and nothing else\n",
|
||||||
|
strings.Join(broker.ViewHears, ", "), broker.ViewBucket)
|
||||||
|
fmt.Println(" this is the only time the credential is printed; the mesh keeps a hash")
|
||||||
|
fmt.Println(" it works once the bus has been told, which is the next push to the machine holding mesh-broker —")
|
||||||
|
fmt.Println(" and while a new build of the bus module waits for that machine, the next `bus upgrade` a person starts,")
|
||||||
|
fmt.Println(" which is also what brings the WebSocket listener it connects through")
|
||||||
|
fmt.Println()
|
||||||
|
fmt.Println(string(held))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func busViewRevoke(ctx context.Context, args []string) error {
|
||||||
|
if len(args) != 0 {
|
||||||
|
return errors.New("bus view-revoke takes nothing: there is one view")
|
||||||
|
}
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
|
||||||
|
if err := open.inventory.ForgetBusUser(ctx, broker.ViewUser); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// **Revoked at the next composition, not now** — as a person is (operator revoke): the bus's users
|
||||||
|
// are a file, and the credential stops working when the file no longer names it.
|
||||||
|
fmt.Println("the view is forgotten, and its credential stops working at the next composition — " +
|
||||||
|
"push the machine holding mesh-broker to make it so")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -2,7 +2,6 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"slices"
|
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
@@ -29,14 +28,6 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
|
|||||||
for _, e := range entries {
|
for _, e := range entries {
|
||||||
byName[e.Manifest.Module] = e
|
byName[e.Manifest.Module] = e
|
||||||
}
|
}
|
||||||
// The machine holding the bus, whose declaration carries every module's grants (novox/hq issue 490).
|
|
||||||
holder := ""
|
|
||||||
for _, e := range entries {
|
|
||||||
if e.Manifest.BusUsers != "" && e.Manifest.ClaimsSeat(catalogue.BrokerSeat) && len(e.On) > 0 {
|
|
||||||
holder = e.On[0]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var granting []string
|
|
||||||
machines := map[string]*link.MachinePlan{}
|
machines := map[string]*link.MachinePlan{}
|
||||||
machine := func(name string) *link.MachinePlan {
|
machine := func(name string) *link.MachinePlan {
|
||||||
if machines[name] == nil {
|
if machines[name] == nil {
|
||||||
@@ -59,10 +50,6 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
|
|||||||
machine(on).Receives = append(machine(on).Receives, name)
|
machine(on).Receives = append(machine(on).Receives, name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !waits && holder != "" && !(len(e.On) == 1 && e.On[0] == holder) && len(e.On) > 0 &&
|
|
||||||
!slices.Contains(granting, name) {
|
|
||||||
granting = append(granting, name)
|
|
||||||
}
|
|
||||||
switch {
|
switch {
|
||||||
case (name == "nats" || catalogue.ProvidesBus(e.Manifest)) && len(e.On) > 0:
|
case (name == "nats" || catalogue.ProvidesBus(e.Manifest)) && len(e.On) > 0:
|
||||||
// Said before the merge (novox/hq issue 336): a new bus build holds every send to the bus's machine
|
// Said before the merge (novox/hq issue 336): a new bus build holds every send to the bus's machine
|
||||||
@@ -89,13 +76,6 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(granting) > 0 {
|
|
||||||
// Said before the merge (novox/hq issue 490): what the walk does when the change alters the bus's
|
|
||||||
// user list. Whether it does is known only once the builds are registered, so it is said as a rule.
|
|
||||||
p.Steps = append(p.Steps, fmt.Sprintf("%s: if this changes what the bus's user list says (a new tool, "+
|
|
||||||
"subject or user), %s is sent that list alone, every build there kept, before %s is judged on its first "+
|
|
||||||
"machine", grantsStepWord, holder, strings.Join(granting, ", ")))
|
|
||||||
}
|
|
||||||
var names []string
|
var names []string
|
||||||
for name := range machines {
|
for name := range machines {
|
||||||
names = append(names, name)
|
names = append(names, name)
|
||||||
@@ -108,10 +88,6 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
|
|||||||
return p
|
return p
|
||||||
}
|
}
|
||||||
|
|
||||||
// grantsStepWord names the grants step (novox/hq issue 490): the bus's user list sent alone to the machine
|
|
||||||
// holding the bus ahead of a walk's gate. Not the bus step, which replaces the bus itself.
|
|
||||||
const grantsStepWord = "grants step"
|
|
||||||
|
|
||||||
// busUpgradeNeeded is how a change plan says that merging it holds the bus's machine for a person's step.
|
// busUpgradeNeeded is how a change plan says that merging it holds the bus's machine for a person's step.
|
||||||
const busUpgradeNeeded = "merging this needs a person's bus upgrade"
|
const busUpgradeNeeded = "merging this needs a person's bus upgrade"
|
||||||
|
|
||||||
|
|||||||
@@ -179,14 +179,6 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
|||||||
}
|
}
|
||||||
fmt.Fprintf(out, ", keeps state %s", strings.Join(kept, ", "))
|
fmt.Fprintf(out, ", keeps state %s", strings.Join(kept, ", "))
|
||||||
}
|
}
|
||||||
// And the logs it keeps, with their caps (novox/hq ADR 0297).
|
|
||||||
if len(m.Logs) > 0 {
|
|
||||||
kept := make([]string, 0, len(m.Logs))
|
|
||||||
for _, l := range m.Logs {
|
|
||||||
kept = append(kept, fmt.Sprintf("%s (%d MiB)", l.Name, l.Cap()))
|
|
||||||
}
|
|
||||||
fmt.Fprintf(out, ", keeps log %s", strings.Join(kept, ", "))
|
|
||||||
}
|
|
||||||
if len(m.Reads) > 0 {
|
if len(m.Reads) > 0 {
|
||||||
fmt.Fprintf(out, ", reads %s", strings.Join(m.Reads, ", "))
|
fmt.Fprintf(out, ", reads %s", strings.Join(m.Reads, ", "))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,10 +6,8 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/beside"
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The check anybody can run is the check registration runs (novox/hq issue 148, ADR 0037): a manifest
|
// The check anybody can run is the check registration runs (novox/hq issue 148, ADR 0037): a manifest
|
||||||
@@ -58,7 +56,10 @@ func TestModuleCheckJudgesBetweenTheManifestsGiven(t *testing.T) {
|
|||||||
|
|
||||||
// The real catalogue passes the command, the way it passes the test that used to be the only check.
|
// The real catalogue passes the command, the way it passes the test that used to be the only check.
|
||||||
func TestModuleCheckPassesTheCatalogue(t *testing.T) {
|
func TestModuleCheckPassesTheCatalogue(t *testing.T) {
|
||||||
root := beside.Catalogue(t)
|
root := filepath.Join("..", "..", "..", "mesh-catalog", "modules")
|
||||||
|
if _, err := os.Stat(root); err != nil {
|
||||||
|
t.Skipf("catalogue sibling not present: %v", err)
|
||||||
|
}
|
||||||
paths, err := manifestsUnder(root)
|
paths, err := manifestsUnder(root)
|
||||||
if err != nil || len(paths) == 0 {
|
if err != nil || len(paths) == 0 {
|
||||||
t.Fatalf("no manifests under %s: %v", root, err)
|
t.Fatalf("no manifests under %s: %v", root, err)
|
||||||
|
|||||||
@@ -5,8 +5,6 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"path"
|
"path"
|
||||||
"regexp"
|
|
||||||
"runtime/debug"
|
|
||||||
"slices"
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -248,11 +246,9 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
|
|||||||
if dir == "mesh-controller" {
|
if dir == "mesh-controller" {
|
||||||
beside["mesh-controller-main"] = link.CheckedOut{Repository: url, Ref: refs["mesh-controller-main"]}
|
beside["mesh-controller-main"] = link.CheckedOut{Repository: url, Ref: refs["mesh-controller-main"]}
|
||||||
if e.Source.Seat != "" {
|
if e.Source.Seat != "" {
|
||||||
for _, sibling := range []string{"mesh-lab", "mesh-sdk"} {
|
if lab, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository,
|
||||||
if url, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository,
|
"mesh-lab")}); err == nil {
|
||||||
sibling)}); err == nil {
|
beside["mesh-lab"] = link.CheckedOut{Repository: lab, Ref: refs["mesh-lab"]}
|
||||||
beside[sibling] = link.CheckedOut{Repository: url, Ref: refs[sibling]}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -275,54 +271,17 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
|
|||||||
// catalogue, whose checkout beside is what tests read its files from, at its main, what the next merge
|
// catalogue, whose checkout beside is what tests read its files from, at its main, what the next merge
|
||||||
// builds from; and beside the controller its main, for a judge the running controller predates, and the
|
// builds from; and beside the controller its main, for a judge the running controller predates, and the
|
||||||
// lab's main, whose replays every check runs. **One rule, read by the check the controller asks for and by
|
// lab's main, whose replays every check runs. **One rule, read by the check the controller asks for and by
|
||||||
// the facts snapshot** (Facts.Beside), so a check run by hand clones what the build seat clones. Beside the
|
// the facts snapshot** (Facts.Beside), so a check run by hand clones what the build seat clones.
|
||||||
// controller also the SDK, checked out at the commit the running controller's go.mod pins (sdkPinned), not
|
|
||||||
// one a desktop holds (novox/hq issue 449). The checkout only places the clone: the conformance test reads the
|
|
||||||
// fixtures at the pin of the tree under check, from the clone's history, so a pull request moving the SDK is
|
|
||||||
// judged against the SDK it moves to (internal/link/conformance_test.go).
|
|
||||||
func besideRefs(dir, running string) map[string]string {
|
func besideRefs(dir, running string) map[string]string {
|
||||||
switch dir {
|
switch dir {
|
||||||
case "mesh-catalog":
|
case "mesh-catalog":
|
||||||
return map[string]string{dir: "main"}
|
return map[string]string{dir: "main"}
|
||||||
case "mesh-controller":
|
case "mesh-controller":
|
||||||
return map[string]string{dir: running, "mesh-controller-main": "main", "mesh-lab": "main",
|
return map[string]string{dir: running, "mesh-controller-main": "main", "mesh-lab": "main"}
|
||||||
"mesh-sdk": sdkPinned()}
|
|
||||||
}
|
}
|
||||||
return map[string]string{dir: running}
|
return map[string]string{dir: running}
|
||||||
}
|
}
|
||||||
|
|
||||||
// sdkModule is the Go module of the SDK the controller is built against.
|
|
||||||
const sdkModule = "git.novox.be/novox/mesh-sdk/go"
|
|
||||||
|
|
||||||
// pseudoCommit is the commit a Go pseudo-version names: v0.1.11-0.20261009143344-f047d0a4a970 → f047d0a4a970.
|
|
||||||
var pseudoCommit = regexp.MustCompile(`-([0-9a-f]{12})$`)
|
|
||||||
|
|
||||||
// sdkPinned is the ref of the SDK repository this controller was built from, as its go.mod pins it and its
|
|
||||||
// build records it: a pseudo-version's commit, or a release's tag (the SDK tags its Go module under go/).
|
|
||||||
// "main" only for a binary that records no SDK version — a local replace — which a running controller is
|
|
||||||
// not (novox/hq issue 449).
|
|
||||||
func sdkPinned() string {
|
|
||||||
info, ok := debug.ReadBuildInfo()
|
|
||||||
if !ok {
|
|
||||||
return "main"
|
|
||||||
}
|
|
||||||
for _, dep := range info.Deps {
|
|
||||||
if dep.Path != sdkModule {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if dep.Replace != nil {
|
|
||||||
dep = dep.Replace
|
|
||||||
}
|
|
||||||
if m := pseudoCommit.FindStringSubmatch(dep.Version); m != nil {
|
|
||||||
return m[1]
|
|
||||||
}
|
|
||||||
if strings.HasPrefix(dep.Version, "v") {
|
|
||||||
return "go/" + dep.Version
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return "main"
|
|
||||||
}
|
|
||||||
|
|
||||||
// siblingOf is another repository of the same owner: novox/mesh-controller → novox/mesh-lab.
|
// siblingOf is another repository of the same owner: novox/mesh-controller → novox/mesh-lab.
|
||||||
func siblingOf(repository, name string) string {
|
func siblingOf(repository, name string) string {
|
||||||
if cut := strings.LastIndex(repository, "/"); cut >= 0 {
|
if cut := strings.LastIndex(repository, "/"); cut >= 0 {
|
||||||
|
|||||||
@@ -169,44 +169,6 @@ func TestAShrinkOfMoreThanHalfIsUrgent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// THE FALSE ALARM (issue 368), replayed through the store D13 reads: an agent's home moved from the
|
|
||||||
// operator's own home (94.7 MB) to the agent account's fresh one (490 B), and `data-shrank` was raised
|
|
||||||
// for data that was never lost. A moved item is read against its new path only, so nothing is raised —
|
|
||||||
// and a genuine shrink at the new path, a week of history later, still is.
|
|
||||||
func TestAMovedPathIsNoShrinkAndAShrinkThereStillIs(t *testing.T) {
|
|
||||||
inv := inventory.ForTest(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
shelf := shelfFor(t, houseManifest)
|
|
||||||
declared := []inventory.DeclaredData{{Module: "house", Item: "config", Class: "irreplaceable", Owned: true}}
|
|
||||||
start := time.Now().Add(-6 * time.Hour)
|
|
||||||
measure := func(at time.Time, path string, size int64) []conditions.Observation {
|
|
||||||
t.Helper()
|
|
||||||
if _, err := inv.RecordData(ctx, "home", declared, map[string]map[string]inventory.Measurement{"house": {
|
|
||||||
"config": {Path: path, Size: bytesOf(size), MeasuredAt: when(at), LastWrite: when(at),
|
|
||||||
LastBackup: when(at)}}}, "", at); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
records, err := inv.Data(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
peaks, err := inv.DataPeaks(ctx, at.Add(-shrinkWindow))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return dataFindings(records, peaks, shelf, nil, nil, at)
|
|
||||||
}
|
|
||||||
measure(start, "/home/operator/.claude", 94_700_000)
|
|
||||||
if got := findingsByKind(measure(start.Add(10*time.Minute), "/home/agent/.claude", 490)); got[kindDataShrank].Kind != "" {
|
|
||||||
t.Fatalf("a moved path raised a shrink: %+v", got[kindDataShrank])
|
|
||||||
}
|
|
||||||
measure(start.Add(2*time.Hour), "/home/agent/.claude", 300<<20)
|
|
||||||
got := findingsByKind(measure(start.Add(4*time.Hour), "/home/agent/.claude", 1<<20))[kindDataShrank]
|
|
||||||
if got.Severity != conditions.Urgent || !strings.Contains(got.Summary, "shrank") {
|
|
||||||
t.Fatalf("a genuine shrink at the new path was not raised: %+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Data said to be written all the time and not written; data with no backup or an old one — urgent when
|
// Data said to be written all the time and not written; data with no backup or an old one — urgent when
|
||||||
// irreplaceable, a warning when valuable; and a new item given its bound before it is said.
|
// irreplaceable, a warning when valuable; and a new item given its bound before it is said.
|
||||||
func TestQuietDataAndMissingBackupsAreSaidByClass(t *testing.T) {
|
func TestQuietDataAndMissingBackupsAreSaidByClass(t *testing.T) {
|
||||||
|
|||||||
@@ -144,15 +144,11 @@ func actOnDeadLetter(ctx context.Context, on *busHandles, act string, id uint64,
|
|||||||
}
|
}
|
||||||
switch act {
|
switch act {
|
||||||
case "deliver":
|
case "deliver":
|
||||||
delivered, to, err := link.DeliverAgain(on.js, id)
|
_, to, err := link.DeliverAgain(on.js, id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
answer["delivered_on"] = to
|
answer["delivered_on"] = to
|
||||||
if delivered.Original != "" {
|
|
||||||
// What became of the ask in its seat's queue (novox/hq issue 334): removed, or left, and why.
|
|
||||||
answer["original"] = delivered.Original
|
|
||||||
}
|
|
||||||
answer["done"] = fmt.Sprintf("dead letter %d was delivered again to %s, and nobody else; it is no longer kept",
|
answer["done"] = fmt.Sprintf("dead letter %d was delivered again to %s, and nobody else; it is no longer kept",
|
||||||
id, consumerWho(d.Stream, d.Consumer))
|
id, consumerWho(d.Stream, d.Consumer))
|
||||||
case "drop":
|
case "drop":
|
||||||
|
|||||||
+13
-476
@@ -117,411 +117,28 @@ func letGo(ctx context.Context, inv *inventory.Inventory, id, by, why string) (i
|
|||||||
// stopWalk ends a walk on its delivery's word: failed, said as stopped by whom, why. What it asked still
|
// stopWalk ends a walk on its delivery's word: failed, said as stopped by whom, why. What it asked still
|
||||||
// builds and registers; nothing further is asked or sent.
|
// builds and registers; nothing further is asked or sent.
|
||||||
func stopWalk(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, error) {
|
func stopWalk(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, error) {
|
||||||
p, _, err := stopWalkWithout(ctx, inv, id, by, why, nil, time.Now().UTC())
|
|
||||||
return p, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// stopAnswer is what a stop that leaves merges out says, as data (novox/hq ADR 0299): the record stopped, the
|
|
||||||
// batch that walks its other merges again with those merges, every merge left out, the modules the stopped walk
|
|
||||||
// already sent that stay on the machines named, the modules the batch's walk withholds, and every merge elsewhere
|
|
||||||
// that will deliver a stopped change when it is walked — a later merge of its repository, or one moving its modules.
|
|
||||||
type stopAnswer struct {
|
|
||||||
Stopped string `json:"stopped"`
|
|
||||||
WalkedAgainBy string `json:"walked_again_by,omitempty"`
|
|
||||||
WalkedAgain []string `json:"walked_again,omitempty"`
|
|
||||||
LeftOut []inventory.PlanLeftOut `json:"left_out,omitempty"`
|
|
||||||
Kept []inventory.PlanKept `json:"kept,omitempty"`
|
|
||||||
Withheld []string `json:"withheld,omitempty"`
|
|
||||||
StillCarriedBy []string `json:"still_carried_by,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// stopWalkWithout ends a walk on its delivery's word and, given merges to leave out, walks every other merge it
|
|
||||||
// answered again without them (novox/hq issue 459, ADR 0299): one walk carries every merge of its batch, so ending
|
|
||||||
// it for one group member ended every other delivery on it.
|
|
||||||
//
|
|
||||||
// - Left out are the merges named, each later merge of their repository's branch, which contains one, and each
|
|
||||||
// merge moving a module a left-out merge moves: a module is built at its branch, which holds the stopped merge.
|
|
||||||
// - The others join the open batch of their kind — merges heard while the walk ran wait there, and an older
|
|
||||||
// commit never walks after a newer one — or a new batch when none is open; its window is closed, and its walk
|
|
||||||
// withholds the left-out merges' modules where it would build them only as dependents.
|
|
||||||
// - Given a batch not yet cut, the merges left out leave it for a stopped record of their own, and the batch
|
|
||||||
// goes on without them; with nothing left in it, the batch itself is stopped.
|
|
||||||
// - The stopped record, the batch and every merge moved are written in one transaction (decision 7).
|
|
||||||
//
|
|
||||||
// With nothing left to walk, or no merge named, the walk is only ended. A merge named that the record does not
|
|
||||||
// answer refuses the stop whole.
|
|
||||||
func stopWalkWithout(ctx context.Context, inv *inventory.Inventory, id, by, why string, without []string,
|
|
||||||
now time.Time) (inventory.Plan, stopAnswer, error) {
|
|
||||||
said := stopAnswer{Stopped: id}
|
|
||||||
release, err := inv.HoldPlans(ctx, true)
|
release, err := inv.HoldPlans(ctx, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return inventory.Plan{}, said, err
|
return inventory.Plan{}, err
|
||||||
}
|
}
|
||||||
defer release()
|
defer release()
|
||||||
p, err := inv.PlanByID(ctx, id)
|
p, err := inv.PlanByID(ctx, id)
|
||||||
if err != nil {
|
|
||||||
return inventory.Plan{}, said, err
|
|
||||||
}
|
|
||||||
batch := p.Batch()
|
|
||||||
if !p.Open() && !(batch && len(without) > 0) {
|
|
||||||
return p, said, fmt.Errorf("%s is already %s", p.ID, p.State)
|
|
||||||
}
|
|
||||||
var merges, again []inventory.BatchedMerge
|
|
||||||
if len(without) > 0 {
|
|
||||||
if merges, err = inv.MergesOf(ctx, p.ID); err != nil {
|
|
||||||
return p, said, err
|
|
||||||
}
|
|
||||||
if again, said.LeftOut, err = leaveOut(merges, without); err != nil {
|
|
||||||
return p, said, fmt.Errorf("%s is not stopped: %w", p.ID, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
stopped := func(r *inventory.Plan) {
|
|
||||||
if r.Delivery == nil {
|
|
||||||
r.Delivery = &inventory.PlanDelivery{}
|
|
||||||
}
|
|
||||||
r.Delivery.Stopped, r.Delivery.StoppedWhy, r.Delivery.LeftOut = by, why, said.LeftOut
|
|
||||||
r.State = inventory.PlanFailed
|
|
||||||
r.Note = fmt.Sprintf("stopped by %s at tier %d: %s", by, r.Tier, why)
|
|
||||||
}
|
|
||||||
withheld := movesOf(leftOutMerges(merges, said.LeftOut))
|
|
||||||
said.Withheld = withheld
|
|
||||||
if said.StillCarriedBy, err = stillCarried(ctx, inv, p.ID, merges, said.LeftOut, withheld); err != nil {
|
|
||||||
return p, said, err
|
|
||||||
}
|
|
||||||
if batch {
|
|
||||||
return p, said, stopInBatch(ctx, inv, &p, merges, again, &said, stopped, now)
|
|
||||||
}
|
|
||||||
entries, err := inv.Catalogued(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return p, said, err
|
|
||||||
}
|
|
||||||
running := map[string][]string{}
|
|
||||||
for _, e := range entries {
|
|
||||||
running[e.Manifest.Module] = e.On
|
|
||||||
}
|
|
||||||
said.Kept = keptOf(p, movesOf(again), running)
|
|
||||||
stopped(&p)
|
|
||||||
p.Delivery.Kept = said.Kept
|
|
||||||
if len(again) == 0 {
|
|
||||||
if len(said.LeftOut) > 0 {
|
|
||||||
p.Note += fmt.Sprintf("; left out %s, nothing else to walk", mergesLeftOutWords(said.LeftOut))
|
|
||||||
}
|
|
||||||
return p, said, inv.SavePlan(ctx, &p)
|
|
||||||
}
|
|
||||||
target, err := batchToWalkAgain(ctx, inv, again, now)
|
|
||||||
if err != nil {
|
|
||||||
return p, said, fmt.Errorf("%s is not stopped: %w", p.ID, err)
|
|
||||||
}
|
|
||||||
target.Delivery.Batch.Closed = true
|
|
||||||
target.Delivery.Batch.Withheld = union(target.Delivery.Batch.Withheld, withheld)
|
|
||||||
sort.Strings(target.Delivery.Batch.Withheld)
|
|
||||||
said.WalkedAgainBy = target.ID
|
|
||||||
var moves []inventory.MergeMove
|
|
||||||
for _, m := range again {
|
|
||||||
said.WalkedAgain = append(said.WalkedAgain, m.Repository+"@"+m.Commit)
|
|
||||||
moves = append(moves, inventory.MergeMove{Repository: m.Repository, Commit: m.Commit, Plan: target.ID, Alone: m.Alone})
|
|
||||||
}
|
|
||||||
p.Delivery.WalkedAgainBy = target.ID
|
|
||||||
p.Note += fmt.Sprintf("; its other merges are walked again by %s: %s", target.ID, strings.Join(said.WalkedAgain, ", "))
|
|
||||||
if len(said.LeftOut) > 0 {
|
|
||||||
p.Note += "; left out " + mergesLeftOutWords(said.LeftOut)
|
|
||||||
}
|
|
||||||
if len(said.Kept) > 0 {
|
|
||||||
p.Note += "; " + keptWords(said.Kept)
|
|
||||||
}
|
|
||||||
// The batch first, so no merge names a plan the store does not hold; all of it, or nothing (decision 7).
|
|
||||||
if err := inv.SavePlansMoving(ctx, []*inventory.Plan{&target, &p}, moves); err != nil {
|
|
||||||
return p, said, fmt.Errorf("%s is not stopped, and none of its merges moved: %w", p.ID, err)
|
|
||||||
}
|
|
||||||
// What the batch holds and names now: written again by the cutter's next look should this fail.
|
|
||||||
if err := keepBatch(ctx, inv, &target, now, ""); err != nil {
|
|
||||||
fmt.Printf(" %s holds its merges; its record is written again at the next look: %v\n", target.ID, err)
|
|
||||||
}
|
|
||||||
fmt.Printf("%s stopped by %s; %s walks its other merges again: %s\n", p.ID, by, target.ID,
|
|
||||||
strings.Join(said.WalkedAgain, ", "))
|
|
||||||
return p, said, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// stopInBatch leaves merges out of a batch not yet cut (ADR 0299): they go to a stopped record of their own, which
|
|
||||||
// names the batch as walking the others, and the batch withholds their modules; a batch left with none is stopped
|
|
||||||
// itself.
|
|
||||||
func stopInBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan, merges, again []inventory.BatchedMerge,
|
|
||||||
said *stopAnswer, stopped func(*inventory.Plan), now time.Time) error {
|
|
||||||
if len(again) == 0 {
|
|
||||||
stopped(b)
|
|
||||||
b.Delivery.Batch = nil
|
|
||||||
b.Note = "a batch whose every merge was left out by a stop: " + b.Note
|
|
||||||
return inv.SavePlan(ctx, b)
|
|
||||||
}
|
|
||||||
left := leftOutMerges(merges, said.LeftOut)
|
|
||||||
first := left[0]
|
|
||||||
r := inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch,
|
|
||||||
Commit: first.Commit, Merged: first.Merged, Created: now, Tiers: [][]string{},
|
|
||||||
Modules: map[string]*inventory.PlanModule{}}
|
|
||||||
if r.ID == b.ID {
|
|
||||||
r.ID += "-left"
|
|
||||||
}
|
|
||||||
stopped(&r)
|
|
||||||
r.Delivery.WalkedAgainBy = b.ID
|
|
||||||
r.Note += fmt.Sprintf("; left out of %s before it was walked: %s; %s walks the others", b.ID,
|
|
||||||
mergesLeftOutWords(said.LeftOut), b.ID)
|
|
||||||
b.Delivery.Batch.Withheld = union(b.Delivery.Batch.Withheld, said.Withheld)
|
|
||||||
sort.Strings(b.Delivery.Batch.Withheld)
|
|
||||||
var moves []inventory.MergeMove
|
|
||||||
for _, m := range left {
|
|
||||||
moves = append(moves, inventory.MergeMove{Repository: m.Repository, Commit: m.Commit, Plan: r.ID, Alone: m.Alone})
|
|
||||||
}
|
|
||||||
for _, m := range again {
|
|
||||||
said.WalkedAgain = append(said.WalkedAgain, m.Repository+"@"+m.Commit)
|
|
||||||
}
|
|
||||||
said.Stopped, said.WalkedAgainBy = r.ID, b.ID
|
|
||||||
if err := inv.SavePlansMoving(ctx, []*inventory.Plan{&r, b}, moves); err != nil {
|
|
||||||
return fmt.Errorf("%s is not changed, and none of its merges moved: %w", b.ID, err)
|
|
||||||
}
|
|
||||||
if err := keepBatch(ctx, inv, b, now, ""); err != nil {
|
|
||||||
fmt.Printf(" %s holds its merges; its record is written again at the next look: %v\n", b.ID, err)
|
|
||||||
}
|
|
||||||
fmt.Printf("%s: %s left out by a stop, kept by %s\n", b.ID, mergesLeftOutWords(said.LeftOut), r.ID)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// batchToWalkAgain is the batch a stopped walk's merges join: the open batch of their kind, or a new one, not yet
|
|
||||||
// written. The kind is read from what the merges moved.
|
|
||||||
func batchToWalkAgain(ctx context.Context, inv *inventory.Inventory, again []inventory.BatchedMerge,
|
|
||||||
now time.Time) (inventory.Plan, error) {
|
|
||||||
own := false
|
|
||||||
for _, n := range movesOf(again) {
|
|
||||||
if _, on := onTheControllersPath[n]; on {
|
|
||||||
own = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
batches, err := inv.Batches(ctx)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return inventory.Plan{}, err
|
return inventory.Plan{}, err
|
||||||
}
|
}
|
||||||
for _, b := range batches {
|
if !p.Open() {
|
||||||
if b.OwnPath() == own {
|
return p, fmt.Errorf("%s is already %s", p.ID, p.State)
|
||||||
if b.Delivery == nil {
|
|
||||||
b.Delivery = &inventory.PlanDelivery{}
|
|
||||||
}
|
}
|
||||||
if b.Delivery.Batch == nil {
|
if p.Delivery == nil {
|
||||||
b.Delivery.Batch = &inventory.PlanBatch{Own: own}
|
p.Delivery = &inventory.PlanDelivery{}
|
||||||
}
|
}
|
||||||
return b, nil
|
p.Delivery.Stopped, p.Delivery.StoppedWhy = by, why
|
||||||
|
p.State = inventory.PlanFailed
|
||||||
|
p.Note = fmt.Sprintf("stopped by %s at tier %d: %s", by, p.Tier, why)
|
||||||
|
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||||
|
return p, err
|
||||||
}
|
}
|
||||||
}
|
return p, nil
|
||||||
first := again[0]
|
|
||||||
return inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch,
|
|
||||||
Commit: first.Commit, Merged: first.Merged, Created: now, State: inventory.PlanAssembling,
|
|
||||||
Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{},
|
|
||||||
Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: own}}}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// leaveOut splits a walk's merges by the merges named to leave out, each `<repository>@<commit>` (a commit's
|
|
||||||
// first characters will do): the merges walked again, and those left out — each named one; each later merge of
|
|
||||||
// its repository's branch, which contains it; and each merge moving a module a left-out merge moves, which is
|
|
||||||
// built at its branch and so holds the stopped merge — until none more is. A name the walk does not answer is
|
|
||||||
// refused.
|
|
||||||
func leaveOut(merges []inventory.BatchedMerge, without []string) ([]inventory.BatchedMerge, []inventory.PlanLeftOut, error) {
|
|
||||||
var stopped []inventory.BatchedMerge
|
|
||||||
for _, w := range without {
|
|
||||||
repository, commit, ok := strings.Cut(strings.TrimSpace(w), "@")
|
|
||||||
if !ok || repository == "" || len(commit) < 7 {
|
|
||||||
return nil, nil, fmt.Errorf("%q names no merge: <repository>@<commit>", w)
|
|
||||||
}
|
|
||||||
i := slices.IndexFunc(merges, func(m inventory.BatchedMerge) bool {
|
|
||||||
return strings.EqualFold(m.Repository, repository) && strings.HasPrefix(m.Commit, commit)
|
|
||||||
})
|
|
||||||
if i < 0 {
|
|
||||||
return nil, nil, fmt.Errorf("it answers no merge %s@%s", repository, commit)
|
|
||||||
}
|
|
||||||
stopped = append(stopped, merges[i])
|
|
||||||
}
|
|
||||||
same := func(a, b inventory.BatchedMerge) bool {
|
|
||||||
return strings.EqualFold(a.Repository, b.Repository) && a.Commit == b.Commit
|
|
||||||
}
|
|
||||||
out := map[int]inventory.PlanLeftOut{}
|
|
||||||
for i, m := range merges {
|
|
||||||
if slices.ContainsFunc(stopped, func(s inventory.BatchedMerge) bool { return same(s, m) }) {
|
|
||||||
out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for grew := true; grew; {
|
|
||||||
grew = false
|
|
||||||
for i, m := range merges {
|
|
||||||
if _, left := out[i]; left {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
for j, o := range out {
|
|
||||||
l := merges[j]
|
|
||||||
contains := o.Contains
|
|
||||||
if contains == "" {
|
|
||||||
contains = l.Commit
|
|
||||||
}
|
|
||||||
if strings.EqualFold(l.Repository, m.Repository) && l.Branch == m.Branch && laterMerge(m, l) {
|
|
||||||
out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit, Contains: contains}
|
|
||||||
} else if shared := sharedMoves(m, l); len(shared) > 0 {
|
|
||||||
out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit, Contains: contains,
|
|
||||||
Shares: shared}
|
|
||||||
} else {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
grew = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var again []inventory.BatchedMerge
|
|
||||||
var left []inventory.PlanLeftOut
|
|
||||||
for i, m := range merges {
|
|
||||||
if o, is := out[i]; is {
|
|
||||||
left = append(left, o)
|
|
||||||
} else {
|
|
||||||
again = append(again, m)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.SliceStable(left, func(i, j int) bool { return left[i].Contains == "" && left[j].Contains != "" })
|
|
||||||
return again, left, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// sharedMoves are the modules two kept merges both moved when they were heard.
|
|
||||||
func sharedMoves(a, b inventory.BatchedMerge) []string {
|
|
||||||
var out []string
|
|
||||||
theirs := announcedOf(b).Moves
|
|
||||||
for _, n := range announcedOf(a).Moves {
|
|
||||||
if slices.Contains(theirs, n) && !slices.Contains(out, n) {
|
|
||||||
out = append(out, n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.Strings(out)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// leftOutMerges are the kept merges a stop left out.
|
|
||||||
func leftOutMerges(merges []inventory.BatchedMerge, out []inventory.PlanLeftOut) []inventory.BatchedMerge {
|
|
||||||
var left []inventory.BatchedMerge
|
|
||||||
for _, m := range merges {
|
|
||||||
if slices.ContainsFunc(out, func(o inventory.PlanLeftOut) bool {
|
|
||||||
return strings.EqualFold(o.Repository, m.Repository) && o.Commit == m.Commit
|
|
||||||
}) {
|
|
||||||
left = append(left, m)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return left
|
|
||||||
}
|
|
||||||
|
|
||||||
// movesOf is every module kept merges moved when they were heard, sorted.
|
|
||||||
func movesOf(merges []inventory.BatchedMerge) []string {
|
|
||||||
var out []string
|
|
||||||
for _, m := range merges {
|
|
||||||
out = union(out, announcedOf(m).Moves)
|
|
||||||
}
|
|
||||||
sort.Strings(out)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// keptOf are the modules a stopped walk sent that no merge walked again moves, and the machines each reached:
|
|
||||||
// those keep running the stopped walk's build. A module sent to its first machines reached those, and the rest it
|
|
||||||
// was sent to after them; one sent with no first machine (its policy sends it to all together, or a later tier is
|
|
||||||
// built by it) reached every machine running it.
|
|
||||||
func keptOf(p inventory.Plan, walkedAgain []string, running map[string][]string) []inventory.PlanKept {
|
|
||||||
var out []inventory.PlanKept
|
|
||||||
for name, m := range p.Modules {
|
|
||||||
if m == nil || slices.Contains(walkedAgain, name) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
var machines []string
|
|
||||||
if m.FirstAt != nil || m.SentAt != nil {
|
|
||||||
machines = union(machines, m.First)
|
|
||||||
}
|
|
||||||
if m.SentAt != nil && len(m.First) == 0 {
|
|
||||||
machines = union(machines, running[name])
|
|
||||||
}
|
|
||||||
if m.Gate != nil && m.Gate.Since != nil {
|
|
||||||
machines = union(machines, m.Gate.Machines)
|
|
||||||
}
|
|
||||||
for machine := range m.Rest {
|
|
||||||
machines = union(machines, []string{machine})
|
|
||||||
}
|
|
||||||
if len(machines) == 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
sort.Strings(machines)
|
|
||||||
out = append(out, inventory.PlanKept{Module: name, Machines: machines})
|
|
||||||
}
|
|
||||||
sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module })
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// keptWords is what a stopped walk leaves on the machines, as a person reads it.
|
|
||||||
func keptWords(kept []inventory.PlanKept) string {
|
|
||||||
var words []string
|
|
||||||
for _, k := range kept {
|
|
||||||
words = append(words, k.Module+" on "+strings.Join(k.Machines, ", "))
|
|
||||||
}
|
|
||||||
return "left running the stopped walk's build, which nothing walks again: " + strings.Join(words, "; ")
|
|
||||||
}
|
|
||||||
|
|
||||||
// stillCarried are the merges outside the stopped record, in a batch or an open walk, that will deliver a stopped
|
|
||||||
// change when they are walked: a later merge of its repository's branch, which contains it, or one that moves a
|
|
||||||
// module it moves. Holding a merged change back for good is a revert's (ADR 0299).
|
|
||||||
func stillCarried(ctx context.Context, inv *inventory.Inventory, id string, merges []inventory.BatchedMerge,
|
|
||||||
out []inventory.PlanLeftOut, withheld []string) ([]string, error) {
|
|
||||||
var named []inventory.BatchedMerge
|
|
||||||
for _, m := range leftOutMerges(merges, out) {
|
|
||||||
if slices.ContainsFunc(out, func(o inventory.PlanLeftOut) bool { return o.Commit == m.Commit && o.Contains == "" }) {
|
|
||||||
named = append(named, m)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(named) == 0 {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
open, err := inv.OpenPlans(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
batches, err := inv.Batches(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var carried []string
|
|
||||||
for _, p := range append(open, batches...) {
|
|
||||||
if p.ID == id {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
ms, err := inv.MergesOf(ctx, p.ID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
for _, m := range ms {
|
|
||||||
later := slices.ContainsFunc(named, func(s inventory.BatchedMerge) bool {
|
|
||||||
return strings.EqualFold(s.Repository, m.Repository) && s.Branch == m.Branch && laterMerge(m, s)
|
|
||||||
})
|
|
||||||
shares := slices.ContainsFunc(announcedOf(m).Moves, func(n string) bool { return slices.Contains(withheld, n) })
|
|
||||||
if later || shares {
|
|
||||||
carried = append(carried, fmt.Sprintf("%s@%s (in %s)", m.Repository, m.Commit, p.ID))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.Strings(carried)
|
|
||||||
return carried, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// mergesLeftOutWords is the merges a stop left out, as a person reads them: each that contains a stopped one, or
|
|
||||||
// shares its modules, says so.
|
|
||||||
func mergesLeftOutWords(out []inventory.PlanLeftOut) string {
|
|
||||||
var words []string
|
|
||||||
for _, o := range out {
|
|
||||||
s := o.Repository + "@" + short(o.Commit)
|
|
||||||
switch {
|
|
||||||
case len(o.Shares) > 0:
|
|
||||||
s += " (which moves " + strings.Join(o.Shares, ", ") + " as " + short(o.Contains) + " does)"
|
|
||||||
case o.Contains != "":
|
|
||||||
s += " (which contains " + short(o.Contains) + ")"
|
|
||||||
}
|
|
||||||
words = append(words, s)
|
|
||||||
}
|
|
||||||
return strings.Join(words, ", ")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// orderMember is one member of a group, as mesh-delivery says it.
|
// orderMember is one member of a group, as mesh-delivery says it.
|
||||||
@@ -849,8 +466,6 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
|||||||
group := set.String("group", "", "a delivery group's id")
|
group := set.String("group", "", "a delivery group's id")
|
||||||
by := set.String("by", catalogue.DeliverySeat, "who says it")
|
by := set.String("by", catalogue.DeliverySeat, "who says it")
|
||||||
why := set.String("why", "", "why")
|
why := set.String("why", "", "why")
|
||||||
without := set.String("without", "", "merges a stop leaves out, <repository>@<commit>, comma-separated: the walk's "+
|
|
||||||
"other merges are walked again without them (novox/hq ADR 0299)")
|
|
||||||
limit := set.Int("n", 50, "how many ended walks to answer beside the open ones")
|
limit := set.Int("n", 50, "how many ended walks to answer beside the open ones")
|
||||||
planID := set.String("plan", "", "one walk")
|
planID := set.String("plan", "", "one walk")
|
||||||
positionals, err := parseAround(set, rest)
|
positionals, err := parseAround(set, rest)
|
||||||
@@ -937,21 +552,15 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
|||||||
return nil
|
return nil
|
||||||
case "stop":
|
case "stop":
|
||||||
if len(positionals) != 1 || strings.TrimSpace(*why) == "" {
|
if len(positionals) != 1 || strings.TrimSpace(*why) == "" {
|
||||||
return errors.New("delivery stop <plan> --why <text> [--by <who>] [--without <repository>@<commit>,…]")
|
return errors.New("delivery stop <plan> --why <text> [--by <who>]")
|
||||||
}
|
}
|
||||||
var p inventory.Plan
|
var p inventory.Plan
|
||||||
var said stopAnswer
|
|
||||||
if err := sayingOnTheBus(ctx, func() (err error) {
|
if err := sayingOnTheBus(ctx, func() (err error) {
|
||||||
p, said, err = stopWalkWithout(ctx, inv, positionals[0], *by, strings.TrimSpace(*why), splitList(*without),
|
p, err = stopWalk(ctx, inv, positionals[0], *by, strings.TrimSpace(*why))
|
||||||
time.Now().UTC())
|
|
||||||
return err
|
return err
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if *without != "" {
|
|
||||||
// What it left out and what it walks again, as data: the delivery's owner reads it (ADR 0299).
|
|
||||||
return answer(said)
|
|
||||||
}
|
|
||||||
fmt.Printf("%s stopped by %s at tier %d of %d; what was asked still builds and registers, nothing further "+
|
fmt.Printf("%s stopped by %s at tier %d of %d; what was asked still builds and registers, nothing further "+
|
||||||
"is asked or sent\n", p.ID, *by, p.Tier, len(p.Tiers))
|
"is asked or sent\n", p.ID, *by, p.Tier, len(p.Tiers))
|
||||||
return nil
|
return nil
|
||||||
@@ -987,81 +596,12 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// Each walk's phases, with the rest's reports (novox/hq ADR 0282 decision 6).
|
|
||||||
now := time.Now()
|
|
||||||
for i := range walks {
|
|
||||||
walks[i].Phases = walks[i].WalkPhases(now, appliedFrom(ctx, inv))
|
|
||||||
}
|
|
||||||
return answer(map[string]any{"held": deliverySeatHeld(entries), "walks": walks,
|
return answer(map[string]any{"held": deliverySeatHeld(entries), "walks": walks,
|
||||||
"own-path": sortedKeysOf(ownPathWords())})
|
"own-path": sortedKeysOf(ownPathWords())})
|
||||||
}
|
}
|
||||||
return fmt.Errorf("delivery %s: plan, order, check, go, stop or walks", sub)
|
return fmt.Errorf("delivery %s: plan, order, check, go, stop or walks", sub)
|
||||||
}
|
}
|
||||||
|
|
||||||
// appliedFrom answers a machine's first report after a send from the controller's `apply` durations; a lookup
|
|
||||||
// that fails is a report not read, which leaves the walk's end unknown rather than wrong.
|
|
||||||
func appliedFrom(ctx context.Context, inv *inventory.Inventory) inventory.AppliedLookup {
|
|
||||||
return func(node string, sent time.Time) (inventory.AppliedReport, bool) {
|
|
||||||
r, ok, err := inv.FirstAppliedAfter(ctx, node, sent)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "the report of %s after %s could not be read: %v\n", node, sent.Format(time.RFC3339), err)
|
|
||||||
return inventory.AppliedReport{}, false
|
|
||||||
}
|
|
||||||
return r, ok
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// recordWalkPhases keeps, once, each phase of every walk ended lately whose end is known, as a duration of kind
|
|
||||||
// walk-phase per class (novox/hq ADR 0282 decision 6): what `durations` summarises. A walk whose end is unknown
|
|
||||||
// past ApplySilentAfter keeps its measured phases without its total.
|
|
||||||
func recordWalkPhases(ctx context.Context, inv *inventory.Inventory, now time.Time) error {
|
|
||||||
recent, err := inv.RecentPlans(ctx, 30)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
for _, p := range recent {
|
|
||||||
if p.State != inventory.PlanDone || p.Release != nil || now.Sub(p.Updated) > 2*time.Hour {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
anyKept, totalKept, err := inv.WalkPhasesKept(ctx, p.ID)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if totalKept {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
ph := p.WalkPhases(now, appliedFrom(ctx, inv))
|
|
||||||
if ph != nil && ph.End == nil && anyKept {
|
|
||||||
continue // kept without its end; kept again only once its end is known
|
|
||||||
}
|
|
||||||
if ph == nil || (ph.End == nil && now.Sub(p.Updated) < inventory.ApplySilentAfter+time.Minute) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
class := ph.Class
|
|
||||||
if class == "" {
|
|
||||||
class = "unclassed"
|
|
||||||
}
|
|
||||||
for _, x := range ph.Phases {
|
|
||||||
if x.State != inventory.PhaseMeasured || x.Start == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if err := inv.RecordDuration(ctx, inventory.Duration{Kind: inventory.DurationWalkPhase,
|
|
||||||
Subject: class + "/" + x.Name, Ref: fmt.Sprintf("%s/%s/%d", p.ID, x.Name, x.Tier), Started: *x.Start,
|
|
||||||
Took: time.Duration(x.TookMS) * time.Millisecond, Detail: p.Named()}); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if ph.End != nil && ph.From != nil {
|
|
||||||
if err := inv.RecordDuration(ctx, inventory.Duration{Kind: inventory.DurationWalkPhase,
|
|
||||||
Subject: class + "/total", Ref: p.ID + "/total", Started: *ph.From,
|
|
||||||
Took: time.Duration(ph.TotalMS) * time.Millisecond, Detail: ph.Said}); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ownPathWords is the controller's own path as words, for an answer.
|
// ownPathWords is the controller's own path as words, for an answer.
|
||||||
func ownPathWords() map[string]string { return onTheControllersPath }
|
func ownPathWords() map[string]string { return onTheControllersPath }
|
||||||
|
|
||||||
@@ -1188,9 +728,6 @@ func sayPlanMoved(ctx context.Context, bus link.Bus, p inventory.Plan) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func publishPlanMoved(ctx context.Context, bus link.Bus, p inventory.Plan) {
|
func publishPlanMoved(ctx context.Context, bus link.Bus, p inventory.Plan) {
|
||||||
// Its phases so far (novox/hq ADR 0282 decision 6): the rest's reports come after the walk ends, and are
|
|
||||||
// read by whoever asks for the walk (`delivery walks`).
|
|
||||||
p.Phases = p.WalkPhases(time.Now(), nil)
|
|
||||||
body, err := json.Marshal(p)
|
body, err := json.Marshal(p)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -37,34 +37,12 @@ type stalledLine struct {
|
|||||||
// Number is the pull request's, for a line of a head the forge never announced (novox/hq issue 347).
|
// Number is the pull request's, for a line of a head the forge never announced (novox/hq issue 347).
|
||||||
Number int `json:"number,omitempty"`
|
Number int `json:"number,omitempty"`
|
||||||
State string `json:"state"`
|
State string `json:"state"`
|
||||||
// Waiting are the merge checks a line of the state `unanswered` waits on, as mesh-delivery says each:
|
|
||||||
// "mesh/merge-gate pending since <UTC time>", "mesh/repo-check never set" (novox/hq issue 438).
|
|
||||||
Waiting []string `json:"waiting,omitempty"`
|
|
||||||
// Checks are the same merge checks as data, which the controller words in the operator's own time (novox/hq
|
|
||||||
// issue 443): a time inside a finished sentence cannot be said again in another zone. A mesh-delivery from before
|
|
||||||
// says none, and Waiting is said as it reads.
|
|
||||||
Checks []waitingCheck `json:"checks,omitempty"`
|
|
||||||
For string `json:"for"`
|
For string `json:"for"`
|
||||||
Bound string `json:"bound"`
|
Bound string `json:"bound"`
|
||||||
H2 string `json:"h2"`
|
H2 string `json:"h2"`
|
||||||
Says string `json:"says"`
|
Says string `json:"says"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// waitingCheck is one merge check a stalled line waits on, as mesh-delivery gives it: its context, its state —
|
|
||||||
// "pending", or "never-set" — and, for a pending one, since when in RFC 3339, empty when the forge did not say.
|
|
||||||
type waitingCheck struct {
|
|
||||||
Context string `json:"context"`
|
|
||||||
State string `json:"state"`
|
|
||||||
Since string `json:"since,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// wordsNow is the time the words are said at, which says whether a time needs its day; a seam a test replaces.
|
|
||||||
var wordsNow = time.Now
|
|
||||||
|
|
||||||
// wordsZone is the zone a stalled line's times are said in: the controller's local zone, as every other time in its
|
|
||||||
// messages. A seam a test replaces, so that no test writes time.Local, which every goroutine of the package reads.
|
|
||||||
var wordsZone = func() *time.Location { return time.Local }
|
|
||||||
|
|
||||||
// operatorsOnly is whether the table leaves H2 nothing to do for the line: the state is the operator's.
|
// operatorsOnly is whether the table leaves H2 nothing to do for the line: the state is the operator's.
|
||||||
func (l stalledLine) operatorsOnly() bool { return l.H2 == "" || strings.HasPrefix(l.H2, "none") }
|
func (l stalledLine) operatorsOnly() bool { return l.H2 == "" || strings.HasPrefix(l.H2, "none") }
|
||||||
|
|
||||||
|
|||||||
@@ -74,12 +74,8 @@ type probe struct {
|
|||||||
// probeRegistry is the registry, in to-be 45's order. **The registry is the design's live form**: a
|
// probeRegistry is the registry, in to-be 45's order. **The registry is the design's live form**: a
|
||||||
// probe added to a design is a row added here.
|
// probe added to a design is a row added here.
|
||||||
var probeRegistry = []probe{
|
var probeRegistry = []probe{
|
||||||
// D1 also says every module assigned and left out of a machine's declaration (novox/hq issue 380), from the
|
{ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation",
|
||||||
// resolution it already makes: needs-operator for a setting nobody gave, left-out for any other cause.
|
From: "issues 236, 263, 275", Kind: "declaration-refused", Raises: []string{kindAwaitingPush}, Phase: 1,
|
||||||
{ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation; no module " +
|
|
||||||
"assigned to a machine is left out of its declaration unsaid",
|
|
||||||
From: "issues 236, 263, 275, 380", Kind: "declaration-refused",
|
|
||||||
Raises: []string{kindAwaitingPush, kindLeftOut, kindNeedsOperator}, Phase: 1,
|
|
||||||
run: probeDeclarations},
|
run: probeDeclarations},
|
||||||
{ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6",
|
{ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6",
|
||||||
From: "issue 262", Kind: "resolver-wrong", Phase: 1, run: probeResolvers},
|
From: "issue 262", Kind: "resolver-wrong", Phase: 1, run: probeResolvers},
|
||||||
@@ -120,11 +116,6 @@ var probeRegistry = []probe{
|
|||||||
{ID: probeDeliveriesID, Asserts: "no delivery is held past its state's bound unsaid: mesh-delivery's " +
|
{ID: probeDeliveriesID, Asserts: "no delivery is held past its state's bound unsaid: mesh-delivery's " +
|
||||||
"`stalled`, each with the transition its table lets healer H2 take", From: "ADR 0239",
|
"`stalled`, each with the transition its table lets healer H2 take", From: "ADR 0239",
|
||||||
Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries},
|
Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries},
|
||||||
// The delivery budgets (novox/hq ADR 0282 decision 7): the newest delivery of each class within its budget,
|
|
||||||
// read from mesh-delivery's `times`; a measurement said, never a delivery held.
|
|
||||||
{ID: probeBudgetsID, Asserts: "the newest delivery of a leaf module ran on every machine within three minutes of " +
|
|
||||||
"its merge, and of a core module within ten: mesh-delivery's `times`", From: "ADR 0282, ADR 0304",
|
|
||||||
Kind: kindOverBudget, Phase: 3, run: probeBudgets},
|
|
||||||
// A client of the bus reconnecting in a loop (novox/hq issue 327), from the server's record of closed
|
// A client of the bus reconnecting in a loop (novox/hq issue 327), from the server's record of closed
|
||||||
// connections, which the bus's own module reads.
|
// connections, which the bus's own module reads.
|
||||||
{ID: probeReconnectsID, Asserts: "no user of the bus had its connection dropped more than twelve times in the " +
|
{ID: probeReconnectsID, Asserts: "no user of the bus had its connection dropped more than twelve times in the " +
|
||||||
@@ -140,12 +131,6 @@ var probeRegistry = []probe{
|
|||||||
{ID: "D-root", Asserts: "no agent can become root without a person on a machine where the router or a channel " +
|
{ID: "D-root", Asserts: "no agent can become root without a person on a machine where the router or a channel " +
|
||||||
"proving its sender runs: not by its own account, and not through a tool that runs its command as an account " +
|
"proving its sender runs: not by its own account, and not through a tool that runs its command as an account " +
|
||||||
"that can", From: "ADR 0259 §8", Kind: kindRootNotFree, Phase: 2, run: probeAgentRoot},
|
"that can", From: "ADR 0259 §8", Kind: kindRootNotFree, Phase: 2, run: probeAgentRoot},
|
||||||
// A module's bucket or log filling toward its cap (novox/hq ADR 0297 §6): said at three quarters, cleared
|
|
||||||
// below seven tenths, so one at its threshold is not raised and cleared on every run. One that cannot be
|
|
||||||
// read is said on its own, and every other is still judged.
|
|
||||||
{ID: probeFillID, Asserts: "every module's bucket and log holds less than three quarters of its cap; one " +
|
|
||||||
"said filling is cleared once it holds less than seven tenths", From: "ADR 0297, issue 501",
|
|
||||||
Kind: kindBucketOrLogFilling, Raises: []string{kindBucketOrLogUnread}, Phase: 1, run: probeFill},
|
|
||||||
{ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals",
|
{ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals",
|
||||||
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs},
|
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs},
|
||||||
// The core's health definitions (novox/hq to-be 45 §8, ADR 0236): what a core component's new build is
|
// The core's health definitions (novox/hq to-be 45 §8, ADR 0236): what a core component's new build is
|
||||||
|
|||||||
@@ -514,10 +514,6 @@ func composedAndValidated(ctx context.Context, open *stores, node string, gens m
|
|||||||
if declared.Epoch, err = open.inventory.SentEpoch(ctx, record.ID); err != nil {
|
if declared.Epoch, err = open.inventory.SentEpoch(ctx, record.ID); err != nil {
|
||||||
return sendable{}, nil, err
|
return sendable{}, nil, err
|
||||||
}
|
}
|
||||||
// And the generation it was last sent, as the would-send is (novox/hq issue 234).
|
|
||||||
if declared.Generation, err = open.inventory.SentGeneration(ctx, record.ID); err != nil {
|
|
||||||
return sendable{}, nil, err
|
|
||||||
}
|
|
||||||
body, err := declared.Body()
|
body, err := declared.Body()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return sendable{}, nil, err
|
return sendable{}, nil, err
|
||||||
|
|||||||
@@ -1,66 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A provider whose wait fails the controller's check lists who waits on it (novox/hq issue 450).
|
|
||||||
//
|
|
||||||
// A wait that does not check out is judged unhealthy (ADR 0283 decision 3), so the provider raises its unhealthy
|
|
||||||
// condition and its consumers are held under it (ADR 0240 rule 5). What is stored is what the machine said, the
|
|
||||||
// wait unchecked: read as said, the provider seems to wait for the operator, and the held consumers were never
|
|
||||||
// listed on the condition that is open.
|
|
||||||
|
|
||||||
// refusedWait is a wait for a secret the database's manifest does not declare: it fails the check.
|
|
||||||
var refusedWait = inventory.Wait{Part: "postgres-database", Secret: "certificate", What: "the database's certificate"}
|
|
||||||
|
|
||||||
// judgedRefused is the provider's resource as the controller judges it: unhealthy, saying why.
|
|
||||||
func judgedRefused() inventory.ResourceHealth {
|
|
||||||
r := waitingDatabaseFor(refusedWait)
|
|
||||||
r.State, r.Waits = link.StateUnhealthy, nil
|
|
||||||
r.Reason = "says it waits for the secret certificate, which db does not declare"
|
|
||||||
return r
|
|
||||||
}
|
|
||||||
|
|
||||||
// The consumer's statement arrives after the provider's, so it is the consumer's judging (sayWaiters) that must list
|
|
||||||
// it at the provider's unhealthy condition, made urgent by who waits on it.
|
|
||||||
func TestAProviderWhoseWaitFailedItsCheckListsWhoWaitsOnIt(t *testing.T) {
|
|
||||||
k, _ := withConditionsInMemory(t)
|
|
||||||
stored := waitingDatabaseFor(refusedWait)
|
|
||||||
open := judgeBoth(t, k, []inventory.ResourceHealth{judgedRefused()},
|
|
||||||
map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Resources: []inventory.ResourceHealth{stored}}},
|
|
||||||
shopFailingBeside(stored))
|
|
||||||
provider := conditionOf(open, moduleUnhealthyKey("db", "anchor"))
|
|
||||||
if len(open) != 1 || provider == nil {
|
|
||||||
t.Fatalf("a provider whose wait failed its check and a consumer of it raised %v; want the provider's "+
|
|
||||||
"unhealthy alone", openKeysOf(open))
|
|
||||||
}
|
|
||||||
if said := provider.Evidence[0].Said; !strings.Contains(said, "shop on laptop") {
|
|
||||||
t.Fatalf("the provider's unhealthy condition does not list shop on laptop as waiting on it: %s", said)
|
|
||||||
}
|
|
||||||
if provider.Severity != conditions.Urgent {
|
|
||||||
t.Fatalf("the provider's unhealthy condition is %s with a consumer waiting on it; want urgent", provider.Severity)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A provider that waits for a secret already given is unhealthy to its consumers too, before its own condition opens:
|
|
||||||
// they are held under it as under any unhealthy provider, never as waiting for the operator, and its condition, when
|
|
||||||
// open, is said as unhealthy with who waits on it.
|
|
||||||
func TestAProviderWaitingForASecretAlreadyGivenIsUnhealthyToItsConsumers(t *testing.T) {
|
|
||||||
given := holdingOf(nil, shopFailingBeside(waitingDatabase()), shopOnTheDatabase)
|
|
||||||
given.waits.given["db@anchor"] = map[string]time.Time{"licence": time.Now().Add(-time.Hour)}
|
|
||||||
by, held := given.heldWith("laptop", "shop", []inventory.ResourceHealth{failingConsumer("shop")})
|
|
||||||
if !held || by.provider != theDatabase || len(by.waits) > 0 {
|
|
||||||
t.Fatalf("shop under a database waiting for a licence already given: held %v under %v with waits %v; want "+
|
|
||||||
"held under db on anchor as unhealthy, no waits", held, by.provider, by.waits)
|
|
||||||
}
|
|
||||||
if _, uncovered := given.waitingUncovered("laptop", "shop", []inventory.ResourceHealth{failingConsumer("shop")}); uncovered {
|
|
||||||
t.Fatalf("a provider whose wait failed its check is said as waiting for another part")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+18
-143
@@ -27,10 +27,8 @@ import (
|
|||||||
// **"Reported applied" is not enough.** ADR 0218 sent a module to one machine first and the rest once
|
// **"Reported applied" is not enough.** ADR 0218 sent a module to one machine first and the rest once
|
||||||
// that machine reported it applied. A build that applies and then does nothing, crashes, serves no
|
// that machine reported it applied. A build that applies and then does nothing, crashes, serves no
|
||||||
// tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by
|
// tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by
|
||||||
// the component's health — the core's health definitions, or a module's own — within ten minutes of the
|
// the component's health — the core's health definitions, or a module's own — passing three times over
|
||||||
// apply: a core module passing three times over at least two minutes, a leaf module once, as soon as the
|
// at least two minutes, within ten minutes of the apply. Only then are the rest sent.
|
||||||
// node-engine states it healthy (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it). Only then
|
|
||||||
// are the rest sent.
|
|
||||||
//
|
//
|
||||||
// **A failing gate stops the plan and puts the previous build back there.** The build is marked failed
|
// **A failing gate stops the plan and puts the previous build back there.** The build is marked failed
|
||||||
// at its gate — registration refuses it and nothing sends it again on its own — the module's registered
|
// at its gate — registration refuses it and nothing sends it again on its own — the module's registered
|
||||||
@@ -45,38 +43,11 @@ var (
|
|||||||
gateSettle = 2 * time.Minute
|
gateSettle = 2 * time.Minute
|
||||||
// gateBound is how long after the apply the build may take to become healthy.
|
// gateBound is how long after the apply the build may take to become healthy.
|
||||||
gateBound = 10 * time.Minute
|
gateBound = 10 * time.Minute
|
||||||
// gatePasses is how many consecutive judgings must find a core module healthy, gateEvery apart at the
|
// gatePasses is how many consecutive judgings must find it healthy, gateEvery apart at the least.
|
||||||
// least, after gateSettle. gateLeafPasses is a leaf module's: one, taken once the node-engine states every
|
|
||||||
// resource of it healthy by its own check, with no settle (novox/hq ADR 0282 decision 4 as ADR 0304
|
|
||||||
// decision 3 changed it, issue 524).
|
|
||||||
gatePasses = 3
|
gatePasses = 3
|
||||||
gateLeafPasses = 1
|
|
||||||
gateLeafSettles = false
|
|
||||||
gateEvery = 40 * time.Second
|
gateEvery = 40 * time.Second
|
||||||
)
|
)
|
||||||
|
|
||||||
// coreModule is whether a module is judged as the core (novox/hq ADR 0282 decision 1): one on the controller's
|
|
||||||
// own path, or one holding the mesh's resolver. Every other module is a leaf.
|
|
||||||
func coreModule(module string, m catalogue.Manifest) bool {
|
|
||||||
if _, own := onTheControllersPath[module]; own {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
for _, c := range m.Claims {
|
|
||||||
if c.Name == resolverSeat {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return coreComponent(module) != ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// gateWants is how many passes a module needs, and whether it must stay healthy gateSettle first.
|
|
||||||
func gateWants(module string, m catalogue.Manifest) (passes int, settles bool) {
|
|
||||||
if coreModule(module, m) {
|
|
||||||
return gatePasses, true
|
|
||||||
}
|
|
||||||
return gateLeafPasses, gateLeafSettles
|
|
||||||
}
|
|
||||||
|
|
||||||
// gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions
|
// gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions
|
||||||
// are raised by a plan as it judges, and kept or cleared by the probe on every run.
|
// are raised by a plan as it judges, and kept or cleared by the probe on every run.
|
||||||
const gateProbe = "DG"
|
const gateProbe = "DG"
|
||||||
@@ -150,13 +121,10 @@ type gateFacts struct {
|
|||||||
health map[string]inventory.NodeHealth
|
health map[string]inventory.NodeHealth
|
||||||
healthErr error
|
healthErr error
|
||||||
// heldOn is, per "<module>@<machine>", the provider its findings are held under (ADR 0240 rule 5).
|
// heldOn is, per "<module>@<machine>", the provider its findings are held under (ADR 0240 rule 5).
|
||||||
heldOn map[string]heldReading
|
heldOn map[string]string
|
||||||
// groupsAdded is, per module, whether the move judged puts an account in a group its previous build did
|
// groupsAdded is, per module, whether the move judged puts an account in a group its previous build did
|
||||||
// not (issue 318 review): the only move whose wait for a new login is excused.
|
// not (issue 318 review): the only move whose wait for a new login is excused.
|
||||||
groupsAdded map[string]bool
|
groupsAdded map[string]bool
|
||||||
// waits is what the controller holds to check a module's wait for the operator (novox/hq ADR 0283): the
|
|
||||||
// manifest of each module's build judged, and the secrets given on each machine.
|
|
||||||
waits operatorWaitFacts
|
|
||||||
// sent is, per machine, the declaration the gate's own send carried there (novox/hq issue 352): a
|
// sent is, per machine, the declaration the gate's own send carried there (novox/hq issue 352): a
|
||||||
// report is held against it, never against the send made last. sentBuilds is what each machine was
|
// report is held against it, never against the send made last. sentBuilds is what each machine was
|
||||||
// last sent of every module, and judged the commit of each module this gate judges: a machine last
|
// last sent of every module, and judged the commit of each module this gate judges: a machine last
|
||||||
@@ -166,29 +134,6 @@ type gateFacts struct {
|
|||||||
commits map[string]string
|
commits map[string]string
|
||||||
}
|
}
|
||||||
|
|
||||||
// heldReading is the provider a module's findings are held under, as "<module> on <machine>", and, when that
|
|
||||||
// provider only waits for the operator for the part the module needs, its wait in one sentence (novox/hq issue
|
|
||||||
// 405): the module's gate then reads as a wait for a person (ADR 0254), a pass carrying the wait, as ADR 0283
|
|
||||||
// decision 4 reads the waiting provider itself. A walk never waits on the operator's secret, there or here.
|
|
||||||
type heldReading struct {
|
|
||||||
on, waits string
|
|
||||||
}
|
|
||||||
|
|
||||||
// heldReadings is, per "<module>@<machine>" in every machine's newest statement, what its findings are held under.
|
|
||||||
func heldReadings(hold *holding) map[string]heldReading {
|
|
||||||
out := map[string]heldReading{}
|
|
||||||
for machine := range hold.healths {
|
|
||||||
for module, by := range hold.heldModules(machine) {
|
|
||||||
reading := heldReading{on: by.provider.Module + " on " + by.provider.Node}
|
|
||||||
if len(by.waits) > 0 {
|
|
||||||
reading.waits = operatorWaitSaid(by.provider.Module, by.provider.Node, by.waits)
|
|
||||||
}
|
|
||||||
out[module+"@"+machine] = reading
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// reportedOn says a machine's last report is on what the gate sent it (novox/hq issue 352): on that
|
// reportedOn says a machine's last report is on what the gate sent it (novox/hq issue 352): on that
|
||||||
// declaration, or one it was sent after it — or, for a gate kept before sends were kept on it, on the
|
// declaration, or one it was sent after it — or, for a gate kept before sends were kept on it, on the
|
||||||
// declaration last sent. On 2026-10-09 a release's gate read the control node's report against a newer
|
// declaration last sent. On 2026-10-09 a release's gate read the control node's report against a newer
|
||||||
@@ -254,7 +199,12 @@ var gatherGateFacts = func(ctx context.Context, open *stores, component string)
|
|||||||
// Whose findings wait on an unhealthy provider (ADR 0240 rule 5): their gates wait, not fail.
|
// Whose findings wait on an unhealthy provider (ADR 0240 rule 5): their gates wait, not fail.
|
||||||
if f.healthErr == nil && f.openErr == nil {
|
if f.healthErr == nil && f.openErr == nil {
|
||||||
if hold, err := readHolding(ctx, inv, f.open); err == nil {
|
if hold, err := readHolding(ctx, inv, f.open); err == nil {
|
||||||
f.heldOn = heldReadings(hold)
|
f.heldOn = map[string]string{}
|
||||||
|
for machine := range f.health {
|
||||||
|
for module, p := range hold.heldModules(machine) {
|
||||||
|
f.heldOn[module+"@"+machine] = p.Module + " on " + p.Node
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if theLease != nil {
|
if theLease != nil {
|
||||||
@@ -307,11 +257,10 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string,
|
|||||||
firstLine(f.openErr.Error())
|
firstLine(f.openErr.Error())
|
||||||
}
|
}
|
||||||
for _, c := range f.open {
|
for _, c := range f.open {
|
||||||
// A wait for a person's new login, for a directory used as found to be handed over, or for the
|
// A wait for a person's new login, or for a directory used as found to be handed over, is the module's
|
||||||
// operator's secret or setting, is the module's reading, not a fault raised since the send: the gate
|
// reading, not a fault raised since the send: the gate reads it from the statement below (ADR 0254,
|
||||||
// reads it from the statement below (ADR 0254, novox/hq issue 339, ADR 0283).
|
// novox/hq issue 339).
|
||||||
if c.Source == gateProbe || c.OpenAt(since) || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound ||
|
if c.Source == gateProbe || c.OpenAt(since) || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound {
|
||||||
c.Kind == kindNeedsOperator {
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
onIt := c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) ||
|
onIt := c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) ||
|
||||||
@@ -528,7 +477,6 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
|||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
facts.groupsAdded = movesAddingGroups(ctx, open.inventory, g, pairs, shelf)
|
facts.groupsAdded = movesAddingGroups(ctx, open.inventory, g, pairs, shelf)
|
||||||
facts.waits = gateWaitFacts(ctx, open.inventory, g, pairs, shelf, facts.health)
|
|
||||||
facts.sent = g.Sent
|
facts.sent = g.Sent
|
||||||
facts.commits, facts.sentBuilds = judgedCommits(g, pairs), map[string]map[string]string{}
|
facts.commits, facts.sentBuilds = judgedCommits(g, pairs), map[string]map[string]string{}
|
||||||
// A module this gate put back at once (putBackBroken) was sent its earlier build by the gate itself:
|
// A module this gate put back at once (putBackBroken) was sent its earlier build by the gate itself:
|
||||||
@@ -637,24 +585,8 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
settled := now.Sub(*g.Since) >= gateSettle
|
settled := now.Sub(*g.Since) >= gateSettle
|
||||||
// What the send needs is what its most demanding module needs: a send carrying a core module is judged
|
|
||||||
// as the core.
|
|
||||||
wants, settles := 0, false
|
|
||||||
for _, m := range modules {
|
|
||||||
p, s := gateWants(m, shelf[m])
|
|
||||||
wants, settles = max(wants, p), settles || s
|
|
||||||
}
|
|
||||||
if len(modules) == 0 {
|
|
||||||
// A send judged on no machine of its modules (its first machine not running them) has no class to
|
|
||||||
// be judged by: it is judged as the core, never passed at once without a reading.
|
|
||||||
wants, settles = gatePasses, true
|
|
||||||
}
|
|
||||||
g.Wants = wants
|
|
||||||
// A module's own pass is its own class's: a leaf beside a core module that fails keeps its build after
|
|
||||||
// its one reading, and only what did not pass is put back (issue 318's rule, by class since issue 524).
|
|
||||||
passedAlone := func(m string) bool {
|
passedAlone := func(m string) bool {
|
||||||
p, s := gateWants(m, shelf[m])
|
return settled && reading[m] <= healthPerson && g.Healthy[m] >= gatePasses
|
||||||
return (settled || !s) && reading[m] <= healthPerson && g.Healthy[m] >= p
|
|
||||||
}
|
}
|
||||||
// fail decides the send failed: what passed on its own keeps its pass, everything else is put back.
|
// fail decides the send failed: what passed on its own keeps its pass, everything else is put back.
|
||||||
fail := func(why string) {
|
fail := func(why string) {
|
||||||
@@ -671,7 +603,6 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
|||||||
pastBound := now.Sub(*g.Since) > gateBound
|
pastBound := now.Sub(*g.Since) > gateBound
|
||||||
switch {
|
switch {
|
||||||
case worst == healthBroken:
|
case worst == healthBroken:
|
||||||
g.Read(now, false, g.BrokenWhy)
|
|
||||||
var judging []string
|
var judging []string
|
||||||
for _, m := range modules {
|
for _, m := range modules {
|
||||||
if reading[m] != healthBroken && !passedAlone(m) {
|
if reading[m] != healthBroken && !passedAlone(m) {
|
||||||
@@ -690,10 +621,8 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
|||||||
// Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either —
|
// Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either —
|
||||||
// the provider's own condition says what is wrong (ADR 0240 rule 5).
|
// the provider's own condition says what is wrong (ADR 0240 rule 5).
|
||||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
||||||
g.Read(now, false, why)
|
|
||||||
case worst == healthNotYet:
|
case worst == healthNotYet:
|
||||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
||||||
g.Read(now, false, why)
|
|
||||||
if pastBound {
|
if pastBound {
|
||||||
fail(fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why))
|
fail(fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why))
|
||||||
}
|
}
|
||||||
@@ -701,8 +630,7 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
|||||||
// Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict.
|
// Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict.
|
||||||
g.Passes++
|
g.Passes++
|
||||||
g.LastPass, g.Last, g.Failing = &now, "", nil
|
g.LastPass, g.Last, g.Failing = &now, "", nil
|
||||||
g.Read(now, true, "")
|
if g.Passes >= gatePasses && settled {
|
||||||
if g.Passes >= wants && (settled || !settles) {
|
|
||||||
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
|
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
|
||||||
now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now)
|
now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now)
|
||||||
}
|
}
|
||||||
@@ -819,47 +747,6 @@ func movesAddingGroups(ctx context.Context, inv *inventory.Inventory, g *invento
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// gateWaitFacts reads what checks the waits for the operator of the modules a gate judges (novox/hq ADR 0283): the
|
|
||||||
// manifest of the build judged — the one it moves to, else the catalogue's — and the secrets given on each machine
|
|
||||||
// that says a module of them waits.
|
|
||||||
func gateWaitFacts(ctx context.Context, inv *inventory.Inventory, g *inventory.PlanGate, pairs []judged,
|
|
||||||
shelf map[string]catalogue.Manifest, health map[string]inventory.NodeHealth) operatorWaitFacts {
|
|
||||||
var f operatorWaitFacts
|
|
||||||
judgedManifests := map[string]catalogue.Manifest{}
|
|
||||||
byMachine := map[string][]string{}
|
|
||||||
for _, j := range pairs {
|
|
||||||
waiting := false
|
|
||||||
for _, r := range health[j.node].Resources {
|
|
||||||
if r.Module == j.module && r.State == link.StateWaiting {
|
|
||||||
waiting = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !waiting {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
byMachine[j.node] = append(byMachine[j.node], j.module)
|
|
||||||
if _, done := judgedManifests[j.module]; done {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
to := g.To
|
|
||||||
for _, c := range g.Carried {
|
|
||||||
if c.Module == j.module {
|
|
||||||
to = c.To
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if m, found, err := inv.ManifestAt(ctx, j.module, to); err == nil && found {
|
|
||||||
judgedManifests[j.module] = m
|
|
||||||
} else if m, known := shelf[j.module]; known {
|
|
||||||
judgedManifests[j.module] = m
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for machine, modules := range byMachine {
|
|
||||||
readWaitFacts(ctx, inv, machine, modules, judgedManifests, &f)
|
|
||||||
}
|
|
||||||
return f
|
|
||||||
}
|
|
||||||
|
|
||||||
// decide sets a gate's verdict.
|
// decide sets a gate's verdict.
|
||||||
func decide(g *inventory.PlanGate, verdict, why string, now time.Time) {
|
func decide(g *inventory.PlanGate, verdict, why string, now time.Time) {
|
||||||
g.Verdict, g.Why, g.JudgedAt = verdict, why, &now
|
g.Verdict, g.Why, g.JudgedAt = verdict, why, &now
|
||||||
@@ -1385,9 +1272,9 @@ func gateLine(g *inventory.PlanGate) string {
|
|||||||
if g.Why != "" {
|
if g.Why != "" {
|
||||||
line += ": " + g.Why
|
line += ": " + g.Why
|
||||||
} else if g.Last != "" {
|
} else if g.Last != "" {
|
||||||
line += fmt.Sprintf(" (%d of %d passes; wanting: %s)", g.Passes, gateWanted(g), g.Last)
|
line += fmt.Sprintf(" (%d of %d passes; wanting: %s)", g.Passes, gatePasses, g.Last)
|
||||||
} else if g.Verdict == "" {
|
} else if g.Verdict == "" {
|
||||||
line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gateWanted(g))
|
line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gatePasses)
|
||||||
}
|
}
|
||||||
if g.Verdict == "" && len(g.Waits) > 0 {
|
if g.Verdict == "" && len(g.Waits) > 0 {
|
||||||
line += waitsSaid(g.Waits)
|
line += waitsSaid(g.Waits)
|
||||||
@@ -1398,10 +1285,6 @@ func gateLine(g *inventory.PlanGate) string {
|
|||||||
if g.Rollback != "" {
|
if g.Rollback != "" {
|
||||||
line += "; " + g.Rollback
|
line += "; " + g.Rollback
|
||||||
}
|
}
|
||||||
// Before the send it judges (novox/hq issue 490).
|
|
||||||
if said := grantsSaid(g.Grants); said != "" {
|
|
||||||
line += "; " + said
|
|
||||||
}
|
|
||||||
return line
|
return line
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1430,11 +1313,3 @@ func witnessObservation(node string, r lease.Rollback) conditions.Observation {
|
|||||||
Summary: fmt.Sprintf("the witness on %s judged the %s %s not healthy %s at %s: %s", node, r.Component,
|
Summary: fmt.Sprintf("the witness on %s judged the %s %s not healthy %s at %s: %s", node, r.Component,
|
||||||
short(r.From), what, r.At.UTC().Format(time.RFC3339), r.Why)}
|
short(r.From), what, r.At.UTC().Format(time.RFC3339), r.Why)}
|
||||||
}
|
}
|
||||||
|
|
||||||
// gateWanted is the passes a gate's send needs, three where the gate was judged before it was said.
|
|
||||||
func gateWanted(g *inventory.PlanGate) int {
|
|
||||||
if g.Wants > 0 {
|
|
||||||
return g.Wants
|
|
||||||
}
|
|
||||||
return gatePasses
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -53,7 +53,6 @@ func healthyContainer(module string) inventory.ResourceHealth {
|
|||||||
// tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass,
|
// tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass,
|
||||||
// counted over the gate's spacing — before it says its own. The broken one alone is put back.
|
// counted over the gate's spacing — before it says its own. The broken one alone is put back.
|
||||||
func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) {
|
func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) {
|
||||||
judgeLeavesAsTheCore(t)
|
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
withGateBounds(t, 150*time.Millisecond, 20*time.Millisecond, 10*time.Second)
|
withGateBounds(t, 150*time.Millisecond, 20*time.Millisecond, 10*time.Second)
|
||||||
@@ -285,7 +284,6 @@ func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) {
|
|||||||
// tools are marked and put back in the very judging that found them broken — their registered build is the
|
// tools are marked and put back in the very judging that found them broken — their registered build is the
|
||||||
// one before, and the laptop is sent it — while `app` goes on being judged, recovers, and keeps its own pass.
|
// one before, and the laptop is sent it — while `app` goes on being judged, recovers, and keeps its own pass.
|
||||||
func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testing.T) {
|
func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testing.T) {
|
||||||
judgeLeavesAsTheCore(t)
|
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
|
|||||||
@@ -153,7 +153,6 @@ func (g *gateMesh) plan(t *testing.T) inventory.Plan {
|
|||||||
// registered and sent to it again — and never reaches the second machine; it is marked, said as a
|
// registered and sent to it again — and never reaches the second machine; it is marked, said as a
|
||||||
// condition and an event, never registered or rolled back again.
|
// condition and an event, never registered or rolled back again.
|
||||||
func TestABuildThatFailsItsGateIsRolledBackOnItsFirstMachineAndGoesNoFurther(t *testing.T) {
|
func TestABuildThatFailsItsGateIsRolledBackOnItsFirstMachineAndGoesNoFurther(t *testing.T) {
|
||||||
judgeLeavesAsTheCore(t)
|
|
||||||
g := aGateMesh(t)
|
g := aGateMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
inv := g.open.inventory
|
inv := g.open.inventory
|
||||||
@@ -256,7 +255,7 @@ func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
|
|||||||
t.Fatalf("sent %v", g.sent)
|
t.Fatalf("sent %v", g.sent)
|
||||||
}
|
}
|
||||||
gate := p.Modules["app"].Gate
|
gate := p.Modules["app"].Gate
|
||||||
if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gateWanted(gate) {
|
if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gatePasses {
|
||||||
t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate)
|
t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate)
|
||||||
}
|
}
|
||||||
if v, found, err := g.open.inventory.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
if v, found, err := g.open.inventory.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||||
@@ -267,52 +266,8 @@ func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A leaf module passes its gate on one reading, as soon as its first machine is healthy, without waiting out the
|
|
||||||
// core's settle time (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it, issue 524).
|
|
||||||
func TestALeafModulePassesOnOneHealthyReading(t *testing.T) {
|
|
||||||
g := aGateMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
gateSettle = time.Hour // a core module could not pass for an hour
|
|
||||||
advancePlans(ctx, g.open) // the first machine is sent the new build
|
|
||||||
advancePlans(ctx, g.open) // one healthy judging
|
|
||||||
p := g.plan(t)
|
|
||||||
gate := p.Modules["app"].Gate
|
|
||||||
if gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes != 1 || gate.Wants != 1 {
|
|
||||||
t.Fatalf("a leaf module's gate after one healthy judging: %+v (plan %s: %s)", gate, p.State, p.Note)
|
|
||||||
}
|
|
||||||
if !strings.Contains(gate.Why, "healthy 1 times") {
|
|
||||||
t.Errorf("the verdict says %q", gate.Why)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Which modules are judged as the core: those on the controller's own path and the resolver's holder, three
|
|
||||||
// readings after the settle time; every other module one (novox/hq ADR 0282 decision 1, ADR 0304 decision 3).
|
|
||||||
func TestAModuleIsJudgedByItsClass(t *testing.T) {
|
|
||||||
for _, c := range []struct {
|
|
||||||
module string
|
|
||||||
claims []catalogue.Claim
|
|
||||||
passes int
|
|
||||||
settles bool
|
|
||||||
}{
|
|
||||||
{"app", nil, 1, false},
|
|
||||||
{"nzbget", nil, 1, false},
|
|
||||||
{"mesh-controller", nil, 3, true},
|
|
||||||
{"mesh-host", nil, 3, true},
|
|
||||||
{"node-tools", nil, 3, true},
|
|
||||||
{"nats", nil, 3, true},
|
|
||||||
{catalogue.DeliverySeat, nil, 3, true},
|
|
||||||
{"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, true},
|
|
||||||
} {
|
|
||||||
p, s := gateWants(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims})
|
|
||||||
if p != c.passes || s != c.settles {
|
|
||||||
t.Errorf("%s: %d passes, settles %v; want %d, %v", c.module, p, s, c.passes, c.settles)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A first machine that never becomes healthy fails its gate at the bound, and is put back.
|
// A first machine that never becomes healthy fails its gate at the bound, and is put back.
|
||||||
func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) {
|
func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) {
|
||||||
judgeLeavesAsTheCore(t)
|
|
||||||
g := aGateMesh(t)
|
g := aGateMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
advancePlans(ctx, g.open)
|
advancePlans(ctx, g.open)
|
||||||
@@ -638,13 +593,3 @@ func TestAWitnessesVerdictIsAConditionWhileItsReportsSayIt(t *testing.T) {
|
|||||||
t.Fatalf("a verdict the reports no longer carry is still said: %+v %v", obs, err)
|
t.Fatalf("a verdict the reports no longer carry is still said: %+v %v", obs, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// judgeLeavesAsTheCore makes a test's leaf modules judged as a core module is, three readings after the settle
|
|
||||||
// time, for a test of the gate's other rules (a fault, the bound, a supersession) that needs a judging still
|
|
||||||
// open after a healthy reading.
|
|
||||||
func judgeLeavesAsTheCore(t *testing.T) {
|
|
||||||
t.Helper()
|
|
||||||
wasPasses, wasSettles := gateLeafPasses, gateLeafSettles
|
|
||||||
gateLeafPasses, gateLeafSettles = gatePasses, true
|
|
||||||
t.Cleanup(func() { gateLeafPasses, gateLeafSettles = wasPasses, wasSettles })
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,300 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The assignment generation a declaration was composed from, and the record of every send (novox/hq
|
|
||||||
// issue 234).
|
|
||||||
//
|
|
||||||
// On 2026-10-04 a declaration newer in sequence than every other named four fewer modules than the
|
|
||||||
// assignments held, and a machine applied it and undeclared all four. The sequence orders arrival and
|
|
||||||
// cannot tell a later send that carries an older view of the assignments. So a declaration now carries
|
|
||||||
// the generation of the assignments it was composed from — a counter the store raises in the same
|
|
||||||
// transaction as every assignment change — and a machine refuses one older than it applied, unless it is
|
|
||||||
// a gate's put-back. And every send is written down with who sent it, from which generation and naming
|
|
||||||
// which modules: the controller logged no send then, and which process sent the stale declaration could
|
|
||||||
// not be read back from anything the mesh kept.
|
|
||||||
|
|
||||||
// kindOlderGeneration is S20's kind: a machine refused a send for the generation it was composed from.
|
|
||||||
const kindOlderGeneration = "older-generation"
|
|
||||||
|
|
||||||
// generationRefusalsSaid is how long a refused send is said after its refusal: an hour, as an advisory is.
|
|
||||||
const generationRefusalsSaid = advisoryQuiet
|
|
||||||
|
|
||||||
// stamp gives a composed declaration the order allotted to it before it was composed: its sequence, and
|
|
||||||
// the epoch and generation when the machine reads them — and keeps the generation and acting epoch for the
|
|
||||||
// record of the send whether or not the machine is sent them.
|
|
||||||
//
|
|
||||||
// **No put-back mark is sent.** A gate puts a machine back by composing it again (sendRollout), so its
|
|
||||||
// declaration is allotted here like any other and carries the generation as it stands — never older than
|
|
||||||
// what the machine applied. The node-engine reads a `put_back` key (mesh-host#82); this controller never
|
|
||||||
// needs to send one.
|
|
||||||
func (o order) stamp(d *sendable) {
|
|
||||||
d.Sequence, d.Epoch = o.sequence, o.epoch
|
|
||||||
d.composedFrom, d.actingEpoch = o.generation, o.acting
|
|
||||||
d.Generation = 0
|
|
||||||
if o.readsGeneration && o.generation > 0 {
|
|
||||||
d.Generation = o.generation
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sentRecord is what the record of a send keeps beside its digest.
|
|
||||||
type sentRecord struct {
|
|
||||||
sequence int64
|
|
||||||
epoch uint64
|
|
||||||
generation int64
|
|
||||||
// toldGeneration is the generation on the wire, which the would-send is stamped with: zero for a machine
|
|
||||||
// whose node-engine has not said it reads one.
|
|
||||||
toldGeneration int64
|
|
||||||
sender string
|
|
||||||
modules []string
|
|
||||||
}
|
|
||||||
|
|
||||||
// sentRecordOf is a composed send's record: its sender is the caller this process acts for.
|
|
||||||
func sentRecordOf(ctx context.Context, d sendable) sentRecord {
|
|
||||||
return sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom,
|
|
||||||
toldGeneration: d.Generation, sender: senderOf(callerOf(ctx)), modules: d.modules}
|
|
||||||
}
|
|
||||||
|
|
||||||
// callerOf is who asked for what this process does: the seat call's caller when ctx belongs to one, the
|
|
||||||
// caller the controller ran this command for, or the account at the shell.
|
|
||||||
func callerOf(ctx context.Context) string {
|
|
||||||
if c := link.CallerIn(ctx); c != "" {
|
|
||||||
return c
|
|
||||||
}
|
|
||||||
return link.Caller()
|
|
||||||
}
|
|
||||||
|
|
||||||
// senderOf is a send's sender in words: who asked, and the process and build that composed it — the
|
|
||||||
// answer issue 234 could not find, because two controllers and a one-shot push were all sending then.
|
|
||||||
func senderOf(caller string) string {
|
|
||||||
host, _ := os.Hostname()
|
|
||||||
return fmt.Sprintf("%s (pid %d on %s, build %s)", caller, os.Getpid(), host, version)
|
|
||||||
}
|
|
||||||
|
|
||||||
// namedModules are the modules a declaration names: its machine's set, less what was left out of it.
|
|
||||||
func namedModules(plan catalogue.Resolution, leftOut map[string]string) []string {
|
|
||||||
out := make([]string, 0, len(plan.Modules))
|
|
||||||
for _, m := range plan.Modules {
|
|
||||||
if _, left := leftOut[m.Module]; !left {
|
|
||||||
out = append(out, m.Module)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.Strings(out)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// heardGenerationRefusal keeps a machine's refusal of a send for its generation, so S20 names its sender
|
|
||||||
// and says it (novox/hq issue 234): on the send it refused or — when the mesh has no record of sending that
|
|
||||||
// sequence — as a refusal of its own, naming the sender as unknown. Either way S20 is raised: a refusal the
|
|
||||||
// mesh cannot attribute is the louder fact, not a quieter one.
|
|
||||||
//
|
|
||||||
// **And raises the mesh's counter past what the machine applied, when the refused send was composed from
|
|
||||||
// the counter as it stands** (counterBehind): then the sender's view was not stale — the counter is behind
|
|
||||||
// the machine, which is a store put back from a backup — and every send after would be refused for ever. A
|
|
||||||
// stale sender's generation is below the counter, and the counter is left alone for it.
|
|
||||||
//
|
|
||||||
// Nothing is sent from here. This runs in the controller's receive loop, and a push from it would hold that
|
|
||||||
// loop, and the machine's hold, for as long as the push takes — the deaf controller of issues 184 and 185.
|
|
||||||
// S20 names `push <node>` for that case instead.
|
|
||||||
func heardGenerationRefusal(ctx context.Context, inv *inventory.Inventory, report link.Report) {
|
|
||||||
r := report.OlderGeneration
|
|
||||||
if r == nil || inv == nil || report.Node == "" {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
node, err := inv.NodeByName(ctx, report.Node)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the machine cannot be "+
|
|
||||||
"read, so it is not raised: %v\n", report.Node, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var raised int64
|
|
||||||
if now, err := inv.AssignmentGeneration(ctx); err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the mesh's own cannot be "+
|
|
||||||
"read: %v\n", report.Node, err)
|
|
||||||
} else if counterBehind(*r, now) {
|
|
||||||
if raised, err = inv.RaiseAssignmentGeneration(ctx, r.Applied); err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation (%d) is behind what %s applied (%d), "+
|
|
||||||
"and could not be raised: %v\n", now, report.Node, r.Applied, err)
|
|
||||||
raised = 0
|
|
||||||
} else {
|
|
||||||
fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation was %d, behind what %s applied (%d) — "+
|
|
||||||
"a store put back from a backup — and is raised to %d; `push %s` sends it what the mesh holds now\n",
|
|
||||||
now, report.Node, r.Applied, raised, report.Node)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
send, found, err := inv.RefusedSend(ctx, node.ID, report.Sequence, r.Applied, raised)
|
|
||||||
if err == nil && !found {
|
|
||||||
send, err = inv.RecordUnrecordedRefusal(ctx, inventory.Send{Node: node.ID, Sequence: report.Sequence,
|
|
||||||
Epoch: report.Epoch, Generation: r.Generation, Digest: report.Declared, RefusedApplied: r.Applied,
|
|
||||||
CounterRaisedTo: raised})
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d for its generation, and the refusal could not be "+
|
|
||||||
"kept, so it is not raised: %v\n", report.Node, report.Sequence, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d from %s: composed from assignment generation %d, "+
|
|
||||||
"and it applied %d\n", report.Node, report.Sequence, send.Sender, r.Generation, r.Applied)
|
|
||||||
}
|
|
||||||
|
|
||||||
// counterBehind says a refusal shows the mesh's counter behind the machine rather than a stale sender: the
|
|
||||||
// refused send carried the counter as it stands now, and the machine applied more than that.
|
|
||||||
func counterBehind(r link.GenerationRefusal, now int64) bool {
|
|
||||||
return r.Generation >= now && r.Applied > r.Generation
|
|
||||||
}
|
|
||||||
|
|
||||||
// watchGenerationRefusals is S20: every send a machine refused within the hour for the generation it was
|
|
||||||
// composed from, naming who sent it (novox/hq issue 234). One per machine, the newest refusal.
|
|
||||||
func watchGenerationRefusals(f *signalFacts) []conditions.Observation {
|
|
||||||
seen := map[string]bool{}
|
|
||||||
var out []conditions.Observation
|
|
||||||
for _, s := range f.refusedSends {
|
|
||||||
if s.RefusedAt == nil || f.now.Sub(*s.RefusedAt) > generationRefusalsSaid || seen[s.NodeName] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
seen[s.NodeName] = true
|
|
||||||
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: s.NodeName, Kind: kindOlderGeneration,
|
|
||||||
Machine: s.NodeName, Severity: conditions.Warning,
|
|
||||||
Summary: fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, and "+
|
|
||||||
"%s applied generation %d — a sender composing from a view of the assignments the mesh has moved "+
|
|
||||||
"past; it named %s", s.NodeName, s.Sequence, s.Sender, s.Generation, s.NodeName, s.RefusedApplied,
|
|
||||||
modulesWords(s.Modules)),
|
|
||||||
Said: fmt.Sprintf("refused at %s", s.RefusedAt.UTC().Format(time.RFC3339)),
|
|
||||||
Headline: fmt.Sprintf("%s refused an out-of-date update", s.NodeName),
|
|
||||||
Explanation: fmt.Sprintf("Something sent %s an update made from an older list of what runs there. %s "+
|
|
||||||
"refused it, so nothing was removed. Nothing for you to do unless it repeats.", s.NodeName, s.NodeName),
|
|
||||||
Resolved: fmt.Sprintf("%s has had no out-of-date update for an hour", s.NodeName)}
|
|
||||||
if s.CounterRaisedTo > 0 {
|
|
||||||
// Not a stale sender: the mesh's counter was behind the machine (a store put back from a backup) and
|
|
||||||
// was raised; nothing has sent the machine its declaration since, so a person is asked to.
|
|
||||||
o.Summary = fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, "+
|
|
||||||
"the mesh's own, and %s applied generation %d — the mesh's counter was behind the machine (a store "+
|
|
||||||
"put back from a backup?) and is raised to %d; `push %s` sends it what the mesh holds now",
|
|
||||||
s.NodeName, s.Sequence, s.Sender, s.Generation, s.NodeName, s.RefusedApplied, s.CounterRaisedTo,
|
|
||||||
s.NodeName)
|
|
||||||
o.Explanation = fmt.Sprintf("Needs you: push %s. The mesh's record was older than %s, so %s refused its "+
|
|
||||||
"update and kept what it had. The record is repaired; a push sends the update again.",
|
|
||||||
s.NodeName, s.NodeName, s.NodeName)
|
|
||||||
}
|
|
||||||
out = append(out, o)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// modulesWords is a send's modules as a sentence says them.
|
|
||||||
func modulesWords(modules []string) string {
|
|
||||||
if len(modules) == 0 {
|
|
||||||
return "no module the mesh recorded"
|
|
||||||
}
|
|
||||||
return strings.Join(modules, ", ")
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234), and
|
|
||||||
// what it answered (issue 485): nothing when the mesh has not recorded a send to it.
|
|
||||||
func writeLastSend(ctx context.Context, w io.Writer, inv *inventory.Inventory, node string) error {
|
|
||||||
s, found, err := inv.LastSend(ctx, node)
|
|
||||||
if err != nil || !found {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
fmt.Fprintf(w, "%s:\n", node)
|
|
||||||
for _, line := range lastSendLines(s) {
|
|
||||||
fmt.Fprintln(w, line)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// lastSendLines is a machine's last send in full, as `node show` and `plan` say it (novox/hq issue 485): when,
|
|
||||||
// by whom, under which lease epoch, from which assignment generation, naming which modules, and what the
|
|
||||||
// machine answered of it — the refusal in its own words.
|
|
||||||
func lastSendLines(s inventory.Send) []string {
|
|
||||||
const stamp = "2006-01-02 15:04:05"
|
|
||||||
epoch := "under no lease epoch"
|
|
||||||
if s.Epoch > 0 {
|
|
||||||
epoch = fmt.Sprintf("under lease epoch %d", s.Epoch)
|
|
||||||
}
|
|
||||||
generation := "no assignment generation recorded"
|
|
||||||
if s.Generation > 0 {
|
|
||||||
generation = fmt.Sprintf("assignment generation %d", s.Generation)
|
|
||||||
}
|
|
||||||
lines := []string{
|
|
||||||
fmt.Sprintf(" last sent sequence %d at %s", s.Sequence, s.SentAt.Local().Format(stamp)),
|
|
||||||
fmt.Sprintf(" by %s", s.Sender),
|
|
||||||
fmt.Sprintf(" %s, composed from %s", epoch, generation),
|
|
||||||
fmt.Sprintf(" naming %s", modulesWords(s.Modules)),
|
|
||||||
}
|
|
||||||
answer := func(words string) { lines = append(lines, " its answer "+words) }
|
|
||||||
switch a := s.Answer; {
|
|
||||||
case s.RefusedAt != nil:
|
|
||||||
// Before the report: a refusal for the generation is kept on the send itself and is the machine's
|
|
||||||
// answer to this send in particular.
|
|
||||||
answer(fmt.Sprintf("refused it at %s: it had applied assignment generation %d, newer than this one",
|
|
||||||
s.RefusedAt.Local().Format(stamp), s.RefusedApplied))
|
|
||||||
case a.Outcome == "" || a.At == nil:
|
|
||||||
answer("the machine has not reported on it yet")
|
|
||||||
case a.Outcome == inventory.OutcomeApplied:
|
|
||||||
answer("applied it at " + a.At.Local().Format(stamp))
|
|
||||||
case a.Outcome == inventory.OutcomeRefused:
|
|
||||||
answer("refused it at " + a.At.Local().Format(stamp) + ", in its words:")
|
|
||||||
for _, line := range strings.Split(strings.TrimRight(a.Refused, "\n"), "\n") {
|
|
||||||
lines = append(lines, " "+strings.TrimSpace(line))
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
answer(fmt.Sprintf("%s at %s: %d resource(s) failed — `status` says which", a.Outcome,
|
|
||||||
a.At.Local().Format(stamp), a.Failed))
|
|
||||||
}
|
|
||||||
return lines
|
|
||||||
}
|
|
||||||
|
|
||||||
// sendAnswerWord is what a machine answered of a send, in a word or few, as status says it in one line.
|
|
||||||
func sendAnswerWord(s inventory.Send) string {
|
|
||||||
switch a := s.Answer; {
|
|
||||||
case s.RefusedAt != nil:
|
|
||||||
return fmt.Sprintf("refused: it had applied generation %d", s.RefusedApplied)
|
|
||||||
case a.Outcome == "" || a.At == nil:
|
|
||||||
return "not reported on yet"
|
|
||||||
case a.Outcome == inventory.OutcomeFailed:
|
|
||||||
return fmt.Sprintf("failed (%d resource(s))", a.Failed)
|
|
||||||
default:
|
|
||||||
return a.Outcome
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// printLastSends says, one line per machine, when it was last sent a declaration, by whom, from which assignment
|
|
||||||
// generation and what it answered (novox/hq issue 485). `node show <node>` says the send in full.
|
|
||||||
func printLastSends(nodes []inventory.Node, sends map[string]inventory.Send, unread string) {
|
|
||||||
if unread != "" {
|
|
||||||
fmt.Printf("what each machine was last sent could NOT be read: %s\n\n", unread)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if len(nodes) == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
fmt.Println("last sent, per machine:")
|
|
||||||
for _, n := range nodes {
|
|
||||||
s, found := sends[n.Name]
|
|
||||||
if !found {
|
|
||||||
fmt.Printf(" %-12s no send recorded\n", n.Name)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
generation := "no generation"
|
|
||||||
if s.Generation > 0 {
|
|
||||||
generation = fmt.Sprintf("generation %d", s.Generation)
|
|
||||||
}
|
|
||||||
fmt.Printf(" %-12s %s by %s, %s — %s\n", n.Name, s.SentAt.Local().Format("2006-01-02 15:04"), s.Sender,
|
|
||||||
generation, sendAnswerWord(s))
|
|
||||||
}
|
|
||||||
fmt.Printf("\n `node show <node>` says a machine's last send in full\n\n")
|
|
||||||
}
|
|
||||||
@@ -1,148 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The assignment generation a declaration was composed from (novox/hq issue 234).
|
|
||||||
|
|
||||||
func bodyKeys(t *testing.T, s sendable) map[string]any {
|
|
||||||
t.Helper()
|
|
||||||
raw, err := s.Body()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var keys map[string]any
|
|
||||||
if err := json.Unmarshal(raw, &keys); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return keys
|
|
||||||
}
|
|
||||||
|
|
||||||
// **The generation goes on the wire only to a machine whose node-engine said it reads one**: an older
|
|
||||||
// node-engine decodes strictly and refuses an unknown key, whole. No put-back mark is ever sent: a gate
|
|
||||||
// composes its put-back afresh, with the generation as it stands.
|
|
||||||
func TestTheGenerationIsSentOnlyToAMachineThatReadsOne(t *testing.T) {
|
|
||||||
resources := []map[string]any{{"id": "a", "type": "file", "path": "/etc/a", "content": "x\n"}}
|
|
||||||
reads := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: true, acting: 57}
|
|
||||||
var told sendable
|
|
||||||
told.Resources = resources
|
|
||||||
reads.stamp(&told)
|
|
||||||
keys := bodyKeys(t, told)
|
|
||||||
if keys["generation"] != float64(40) || keys["sequence"] != float64(12) {
|
|
||||||
t.Fatalf("a machine that reads a generation was sent %v", keys)
|
|
||||||
}
|
|
||||||
if _, there := keys["put_back"]; there {
|
|
||||||
t.Fatalf("a put-back mark was sent: %v", keys)
|
|
||||||
}
|
|
||||||
if told.composedFrom != 40 || told.actingEpoch != 57 {
|
|
||||||
t.Errorf("the send does not keep what it was composed from for its record: %+v", told)
|
|
||||||
}
|
|
||||||
|
|
||||||
older := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: false, acting: 57}
|
|
||||||
var untold sendable
|
|
||||||
untold.Resources = resources
|
|
||||||
older.stamp(&untold)
|
|
||||||
if _, there := bodyKeys(t, untold)["generation"]; there {
|
|
||||||
t.Fatal("a machine whose node-engine never said it reads a generation was sent one")
|
|
||||||
}
|
|
||||||
if untold.composedFrom != 40 {
|
|
||||||
t.Errorf("the generation it was composed from is recorded whether or not it was sent: %+v", untold)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **The sender is named**: the caller of the seat call when there is one, else the shell's account, and the
|
|
||||||
// process and build that composed it.
|
|
||||||
func TestASendNamesItsSender(t *testing.T) {
|
|
||||||
said := senderOf("g14.node-tools, through the mesh-controller seat")
|
|
||||||
if !strings.Contains(said, "g14.node-tools") || !strings.Contains(said, "pid ") || !strings.Contains(said, "build ") {
|
|
||||||
t.Fatalf("the sender reads %q", said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// refusedSend is a send a machine refused for its generation at a moment.
|
|
||||||
func refusedSend(at time.Time) inventory.Send {
|
|
||||||
return inventory.Send{NodeName: "anchor", Sequence: 12, Epoch: 57, Generation: 38, RefusedApplied: 40,
|
|
||||||
Sender: "a one-shot push by jochen at a shell on anchor (pid 4242 on anchor, build 2026.10.11)",
|
|
||||||
Modules: []string{"docker"}, SentAt: at.Add(-time.Second), RefusedAt: &at, Recorded: true}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **A refused send is raised naming its sender** (novox/hq issue 234): who sent it, from which generation,
|
|
||||||
// against which the machine applied, and its sequence — the facts that took a morning to look for.
|
|
||||||
func TestARefusedSendIsRaisedNamingItsSender(t *testing.T) {
|
|
||||||
now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC)
|
|
||||||
f := calm(now)
|
|
||||||
f.refusedSends = []inventory.Send{refusedSend(now.Add(-time.Minute))}
|
|
||||||
got := watchGenerationRefusals(f)
|
|
||||||
if len(got) != 1 {
|
|
||||||
t.Fatalf("%+v", got)
|
|
||||||
}
|
|
||||||
o := got[0]
|
|
||||||
if o.Key() != "machine.anchor.older-generation" || o.Machine != "anchor" {
|
|
||||||
t.Errorf("raised as %s about %q", o.Key(), o.Machine)
|
|
||||||
}
|
|
||||||
for _, want := range []string{"a one-shot push by jochen", "generation 38", "generation 40", "sequence 12"} {
|
|
||||||
if !strings.Contains(o.Summary, want) {
|
|
||||||
t.Errorf("the summary does not say %q: %s", want, o.Summary)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if o.Headline == "" || o.Explanation == "" || o.Resolved == "" {
|
|
||||||
t.Errorf("the condition is not worded for the operator: %+v", o)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **A refusal of a send the mesh has no record of is raised too, its sender said to be unknown** — never
|
|
||||||
// only a line on stderr.
|
|
||||||
func TestARefusalOfAnUnrecordedSendIsRaisedSayingTheSenderIsUnknown(t *testing.T) {
|
|
||||||
now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC)
|
|
||||||
f := calm(now)
|
|
||||||
s := refusedSend(now.Add(-time.Minute))
|
|
||||||
s.Recorded, s.Sender, s.Modules = false, "a sender the mesh has no record of (no send of this sequence was recorded)", nil
|
|
||||||
f.refusedSends = []inventory.Send{s}
|
|
||||||
got := watchGenerationRefusals(f)
|
|
||||||
if len(got) != 1 || !strings.Contains(got[0].Summary, "no record of") {
|
|
||||||
t.Fatalf("an unattributed refusal was not raised as one: %+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **When the refusal showed the mesh's counter behind the machine, the condition asks for a push** — it was
|
|
||||||
// raised, and nothing has sent the machine its declaration since — and does not say there is nothing to do.
|
|
||||||
func TestACounterBehindTheMachineAsksForAPush(t *testing.T) {
|
|
||||||
now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC)
|
|
||||||
f := calm(now)
|
|
||||||
s := refusedSend(now.Add(-time.Minute))
|
|
||||||
s.Generation, s.RefusedApplied, s.CounterRaisedTo = 12, 40, 41
|
|
||||||
f.refusedSends = []inventory.Send{s}
|
|
||||||
got := watchGenerationRefusals(f)
|
|
||||||
if len(got) != 1 {
|
|
||||||
t.Fatalf("%+v", got)
|
|
||||||
}
|
|
||||||
if !strings.Contains(got[0].Summary, "`push anchor`") || !strings.Contains(got[0].Explanation, "push anchor") ||
|
|
||||||
strings.Contains(got[0].Explanation, "Nothing for you to do") {
|
|
||||||
t.Errorf("a counter behind the machine does not ask for a push: %s / %s", got[0].Summary, got[0].Explanation)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **Only a refusal of the counter as it stands is the counter behind**: a stale sender's generation is below
|
|
||||||
// it, and the counter is left alone for that.
|
|
||||||
func TestOnlyARefusalOfTheCounterAsItStandsRaisesIt(t *testing.T) {
|
|
||||||
for _, c := range []struct {
|
|
||||||
refused, applied, now int64
|
|
||||||
behind bool
|
|
||||||
}{
|
|
||||||
{refused: 12, applied: 40, now: 12, behind: true}, // the store was put back: the mesh says 12, the machine had 40
|
|
||||||
{refused: 38, applied: 40, now: 41, behind: false}, // a stale sender: the counter is already past
|
|
||||||
{refused: 38, applied: 40, now: 40, behind: false}, // a stale sender: the counter is where the machine is
|
|
||||||
} {
|
|
||||||
r := link.GenerationRefusal{Generation: c.refused, Applied: c.applied}
|
|
||||||
if got := counterBehind(r, c.now); got != c.behind {
|
|
||||||
t.Errorf("refused %d, applied %d, counter %d: behind %v, want %v", c.refused, c.applied, c.now, got, c.behind)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,198 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/identity"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The grants step on a store (novox/hq issue 490): the machine holding the bus is sent its new user list,
|
|
||||||
// and nothing else — not the new build of a module it runs, not a module newly assigned there.
|
|
||||||
//
|
|
||||||
// The restic shape: anchor holds the bus and runs restic at c1, as does laptop; restic's c2 gives it a
|
|
||||||
// health tool. The grants step's send to anchor composes restic at c1, carries the user list that grants
|
|
||||||
// laptop's node-engine c2's tool, and records that anchor still runs c1. A module newly assigned to anchor
|
|
||||||
// makes the step fail, said, rather than install it unjudged.
|
|
||||||
func TestTheGrantsStepSendsTheUserListAndNothingElse(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
start := time.Now().Add(-time.Hour)
|
|
||||||
build := func(module, commit string, at time.Time, manifest map[string]any) link.BuildResult {
|
|
||||||
manifest["module"], manifest["version"] = module, "1"
|
|
||||||
raw, _ := json.Marshal(manifest)
|
|
||||||
return link.BuildResult{ID: link.NewBuildID(at), Repository: "novox/mesh-catalog", Path: "modules/" + module,
|
|
||||||
On: "anchor", Module: module, Commit: commit, Manifest: raw,
|
|
||||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
|
||||||
}
|
|
||||||
for _, b := range []link.BuildResult{
|
|
||||||
build("nats", "n1", start, natsFixture()),
|
|
||||||
build("restic", "c1", start.Add(time.Second), resticFixture(false)),
|
|
||||||
build("wallpaper", "w1", start.Add(2*time.Second), wallpaperFixture()),
|
|
||||||
} {
|
|
||||||
if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, a := range [][2]string{{"anchor", "nats"}, {"anchor", "restic"}, {"laptop", "restic"}} {
|
|
||||||
if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, n := range []string{"anchor", "laptop"} {
|
|
||||||
if _, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: "node." + n, Kind: inventory.BusNode, Node: n}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
wasEpoch := epochForActs
|
|
||||||
epochForActs = func(context.Context) (uint64, error) { return 7, nil }
|
|
||||||
t.Cleanup(func() { epochForActs = wasEpoch })
|
|
||||||
|
|
||||||
// A send to anchor composed as sendToEach composes it: numbered, planned, declared, stamped.
|
|
||||||
compose := func(under context.Context) (catalogue.Resolution, sendable) {
|
|
||||||
t.Helper()
|
|
||||||
numbered, err := allot(under, inv, "anchor")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
plan, settings, err := planFor(under, open, "anchor")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
gens, err := generators(under, open)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
declared, err := declarationWith(under, open, "anchor", plan, settings, gens, Allocating)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
numbered.stamp(&declared)
|
|
||||||
return plan, declared
|
|
||||||
}
|
|
||||||
record := func(declared sendable) {
|
|
||||||
t.Helper()
|
|
||||||
body, err := declared.Body()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := recordSent(ctx, inv, "anchor", body, declared.Builds, declared.Epoch,
|
|
||||||
sentRecordOf(ctx, declared)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Kept for anchor alone. laptop is never recorded as sent, on purpose: composing anchor resolves laptop
|
|
||||||
// too (who is on the private network), and a step that kept every machine's builds refused anchor's
|
|
||||||
// composition for want of laptop's last send (repo-check of #230 at 0f853e93).
|
|
||||||
kept := keepingEveryBuild(keepingRecorded(ctx), "anchor")
|
|
||||||
|
|
||||||
// What anchor was last sent: everything at the builds of before the merge, as an ordinary send sends it.
|
|
||||||
_, before := compose(keepingRecorded(ctx))
|
|
||||||
record(before)
|
|
||||||
plan, declared := compose(kept)
|
|
||||||
if only, err := grantsOnlyIn(ctx, open, "anchor", plan, declared); err != nil || only != "" {
|
|
||||||
t.Fatalf("with nothing changed, the step reads %q, %v", only, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The merge: restic's c2 gives it a health tool.
|
|
||||||
if _, _, err := takeIn(ctx, inv, build("restic", "c2", start.Add(time.Minute),
|
|
||||||
resticFixture(true))); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The step's send to anchor: restic as anchor runs it, c1, and the user list granting c2's tool.
|
|
||||||
generation, err := inv.AssignmentGeneration(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
plan, declared = compose(kept)
|
|
||||||
for _, m := range plan.Modules {
|
|
||||||
if m.Module == "restic" && len(m.Tools) > 0 {
|
|
||||||
t.Fatalf("the grants step composed restic's new build on anchor: tools %v", m.Tools)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if declared.Builds["restic"] != "c1" {
|
|
||||||
t.Fatalf("the step's send records it carries restic %q; want c1, which anchor runs", declared.Builds["restic"])
|
|
||||||
}
|
|
||||||
if !strings.Contains(declared.BusUsers, "mesh.mod.restic.tool.backup_health.laptop") {
|
|
||||||
t.Errorf("the step's declaration does not grant laptop's node-engine restic's new tool:\n%s", declared.BusUsers)
|
|
||||||
}
|
|
||||||
// The generation it carries is the current one (novox/hq issue 234), and the step does not raise it.
|
|
||||||
if declared.composedFrom != generation {
|
|
||||||
t.Errorf("the step's send was composed from generation %d; the mesh is at %d", declared.composedFrom, generation)
|
|
||||||
}
|
|
||||||
// The guard, on the very declaration the send sends: only the user list differs from the last send.
|
|
||||||
if only, err := grantsOnlyIn(ctx, open, "anchor", plan, declared); err != nil || only != "" {
|
|
||||||
t.Fatalf("the step reads as changing more than the user list: %q, %v", only, err)
|
|
||||||
}
|
|
||||||
// And the gate's refusal still reads the step's send: it moves nothing there.
|
|
||||||
if names, err := ungatedIn(kept, open, []string{"anchor"}, ""); err != nil || strings.Join(names, ",") != "anchor" {
|
|
||||||
t.Fatalf("the step's send is refused as a move: %v, %v", names, err)
|
|
||||||
}
|
|
||||||
// Recorded as the send records it: anchor still runs restic c1, its gate still to come there.
|
|
||||||
record(declared)
|
|
||||||
if sent, _, err := inv.SentBuilds(ctx, "anchor"); err != nil || sent["restic"] != "c1" {
|
|
||||||
t.Fatalf("after the step anchor reads as sent restic %q (%v); want c1", sent["restic"], err)
|
|
||||||
}
|
|
||||||
if after, err := inv.AssignmentGeneration(ctx); err != nil || after != generation {
|
|
||||||
t.Fatalf("the step moved the assignment generation from %d to %d (%v)", generation, after, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A module newly assigned to anchor: the step would install it, unjudged, so it is refused unsent.
|
|
||||||
if _, err := inv.Assign(ctx, "anchor", "wallpaper"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
plan, declared = compose(kept)
|
|
||||||
only, err := grantsOnlyIn(ctx, open, "anchor", plan, declared)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !strings.Contains(only, "wallpaper newly assigned") || !strings.Contains(only, "+wallpaper") {
|
|
||||||
t.Fatalf("a new assignment on the bus's machine reads %q; want the step refused, naming it", only)
|
|
||||||
}
|
|
||||||
// And the send itself refuses it, unsent: judged where it is composed, before the bus is dialled.
|
|
||||||
identity.ForTest(t)
|
|
||||||
if _, err := sendToEach(kept, open, []string{"anchor"}); !errors.Is(err, errNotGrantsOnly) ||
|
|
||||||
!strings.Contains(err.Error(), "wallpaper") {
|
|
||||||
t.Fatalf("the grants step's send of a new assignment was not refused by the send: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// fixtureImage is the image every fixture container of the grants step's tests runs.
|
|
||||||
var fixtureImage = "registry.invalid:5000/restic/backup@sha256:" + strings.Repeat("b", 64)
|
|
||||||
|
|
||||||
// resticFixture is restic's manifest in the shape of mesh-catalog #210: at c2 (withTool) its backup judged by
|
|
||||||
// its new tool `backup_health`, beside a check of another kind it does not run itself — a tool check alone
|
|
||||||
// is refused (ADR 0227 rule 8, ADR 0240 rule 2) — and at c1 neither.
|
|
||||||
func resticFixture(withTool bool) map[string]any {
|
|
||||||
backup := map[string]any{"id": "backup", "type": "container", "name": "restic-backup", "image": fixtureImage}
|
|
||||||
measure := map[string]any{"id": "measure", "type": "container", "name": "restic-measure", "image": fixtureImage}
|
|
||||||
m := map[string]any{"resources": []any{backup, measure}}
|
|
||||||
if withTool {
|
|
||||||
backup["health"] = map[string]any{"kind": catalogue.HealthTool, "tool": "backup_health"}
|
|
||||||
measure["health"] = map[string]any{"kind": "runtime"}
|
|
||||||
m["tools"] = []string{"backup_health"}
|
|
||||||
}
|
|
||||||
return m
|
|
||||||
}
|
|
||||||
|
|
||||||
// natsFixture is the bus's module: it holds mesh-broker and is sent the user list.
|
|
||||||
func natsFixture() map[string]any {
|
|
||||||
return map[string]any{"bus-users": "/var/lib/nats-module/conf/accounts.conf",
|
|
||||||
"claims": []any{map[string]any{"name": catalogue.BrokerSeat, "scope": catalogue.ScopeMesh}}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// wallpaperFixture is a module the bus's machine is newly assigned.
|
|
||||||
func wallpaperFixture() map[string]any {
|
|
||||||
return map[string]any{"resources": []any{
|
|
||||||
map[string]any{"id": "paper", "type": "container", "name": "wallpaper", "image": fixtureImage}}}
|
|
||||||
}
|
|
||||||
@@ -1,226 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The grants reach the bus before the gate (novox/hq issue 490).
|
|
||||||
//
|
|
||||||
// On 2026-10-11 a merge gave restic, assigned to every machine, a new health tool. Its walk sent restic to
|
|
||||||
// its first machine and judged it there; that machine's node-engine asked the new tool and the bus refused
|
|
||||||
// it, because the grant was in the bus's user list, which only the machine holding the bus carries — and
|
|
||||||
// that machine runs restic too, so it was left out of the send while restic's new build waited there for
|
|
||||||
// the very gate that could not pass. A person pushed it by hand, carrying restic's new build there unjudged.
|
|
||||||
|
|
||||||
// aSend is one send the walk made: to which machines, and whether every build there was kept.
|
|
||||||
type aSend struct {
|
|
||||||
names []string
|
|
||||||
keepsAll bool
|
|
||||||
judged []string
|
|
||||||
}
|
|
||||||
|
|
||||||
// sendsRecorded replaces the walk's send and the reading of the bus's user list for one test: the machine
|
|
||||||
// holding the bus is novox, and its list is behind as behind says.
|
|
||||||
func sendsRecorded(t *testing.T, holder string, behind bool, refuse error) *[]aSend {
|
|
||||||
t.Helper()
|
|
||||||
var sends []aSend
|
|
||||||
wasSend, wasBehind := sendRollout, brokerBehindOf
|
|
||||||
t.Cleanup(func() { sendRollout, brokerBehindOf = wasSend, wasBehind })
|
|
||||||
brokerBehindOf = func(_ context.Context, _ *stores, names []string) (string, bool, error) {
|
|
||||||
if slices.Contains(names, holder) {
|
|
||||||
return holder, false, nil
|
|
||||||
}
|
|
||||||
return holder, behind, nil
|
|
||||||
}
|
|
||||||
sendRollout = func(ctx context.Context, _ *stores, names []string) ([]string, error) {
|
|
||||||
s := aSend{names: append([]string(nil), names...), keepsAll: len(names) == 1 && everyBuildKept(ctx, names[0])}
|
|
||||||
for n := range scopeOf(ctx).judged {
|
|
||||||
s.judged = append(s.judged, n)
|
|
||||||
}
|
|
||||||
sends = append(sends, s)
|
|
||||||
if refuse != nil && s.keepsAll {
|
|
||||||
return nil, refuse
|
|
||||||
}
|
|
||||||
return names, nil
|
|
||||||
}
|
|
||||||
return &sends
|
|
||||||
}
|
|
||||||
|
|
||||||
// The restic shape: a new tool on a module on every machine, its first machine ace, the bus on novox.
|
|
||||||
func TestAWalkSendsTheGrantsToTheBusBeforeTheFirstMachineIsJudged(t *testing.T) {
|
|
||||||
sends := sendsRecorded(t, "novox", true, nil)
|
|
||||||
sent, grants, err := sendJudged(t.Context(), nil, "ace")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(*sends) != 2 {
|
|
||||||
t.Fatalf("the walk made %d send(s), want the grants step and then the first machine: %+v", len(*sends), *sends)
|
|
||||||
}
|
|
||||||
first, then := (*sends)[0], (*sends)[1]
|
|
||||||
if strings.Join(first.names, ",") != "novox" || !first.keepsAll || len(first.judged) != 0 {
|
|
||||||
t.Errorf("the first send is %+v, want novox alone, every build there kept, judging nothing", first)
|
|
||||||
}
|
|
||||||
if strings.Join(then.names, ",") != "ace" || then.keepsAll || strings.Join(then.judged, ",") != "ace" {
|
|
||||||
t.Errorf("the second send is %+v, want ace, judged", then)
|
|
||||||
}
|
|
||||||
if strings.Join(sent, ",") != "ace" {
|
|
||||||
t.Errorf("the gate's machines are %v, want ace alone: the bus's machine is not judged", sent)
|
|
||||||
}
|
|
||||||
if grants == nil || grants.Node != "novox" || grants.At == nil || grants.Failed != "" {
|
|
||||||
t.Fatalf("the gate keeps %+v as its grants step", grants)
|
|
||||||
}
|
|
||||||
line := gateLine(&inventory.PlanGate{Machines: sent, Grants: grants})
|
|
||||||
if !strings.Contains(line, "grants step: novox sent the bus's user list first, its builds kept") {
|
|
||||||
t.Errorf("plans says the gate as %q", line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Nothing more when there is nothing to carry: the list unchanged, or the first machine holds the bus.
|
|
||||||
func TestAWalkTakesNoGrantsStepWhenTheListIsAlreadyThere(t *testing.T) {
|
|
||||||
for _, c := range []struct {
|
|
||||||
name string
|
|
||||||
node string
|
|
||||||
behind bool
|
|
||||||
}{{"the list unchanged", "ace", false}, {"the first machine holds the bus", "novox", true}} {
|
|
||||||
t.Run(c.name, func(t *testing.T) {
|
|
||||||
sends := sendsRecorded(t, "novox", c.behind, nil)
|
|
||||||
_, grants, err := sendJudged(t.Context(), nil, c.node)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(*sends) != 1 || (*sends)[0].keepsAll || grants != nil {
|
|
||||||
t.Errorf("the walk made %+v with grants %+v, want the judged send alone", *sends, grants)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A grants step that cannot be sent is said on the gate, and the walk goes on as it did before.
|
|
||||||
func TestAGrantsStepThatFailsIsSaidAndTheSendGoesOn(t *testing.T) {
|
|
||||||
sends := sendsRecorded(t, "novox", true, errors.New("a bus upgrade waits for a person"))
|
|
||||||
sent, grants, err := sendJudged(t.Context(), nil, "ace")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if strings.Join(sent, ",") != "ace" || len(*sends) != 2 {
|
|
||||||
t.Errorf("the walk made %+v", *sends)
|
|
||||||
}
|
|
||||||
if grants == nil || grants.Failed == "" || grants.At != nil {
|
|
||||||
t.Fatalf("the gate keeps %+v", grants)
|
|
||||||
}
|
|
||||||
if line := gateLine(&inventory.PlanGate{Machines: sent, Grants: grants}); !strings.Contains(line,
|
|
||||||
"grants step to novox FAILED, sent without it: a bus upgrade waits for a person") {
|
|
||||||
t.Errorf("plans says the gate as %q", line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A step whose send would change more than the user list is refused unsent by the send (sendToEach): the
|
|
||||||
// gate says it FAILED with what differed, and the walk goes on.
|
|
||||||
func TestAGrantsStepThatWouldCarryMoreIsNotSent(t *testing.T) {
|
|
||||||
refusal := fmt.Errorf("%w: its send would change more than the bus's user list: -postgres.login-n8n", errNotGrantsOnly)
|
|
||||||
sends := sendsRecorded(t, "novox", true, refusal)
|
|
||||||
sent, grants, err := sendJudged(t.Context(), nil, "ace")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if strings.Join(sent, ",") != "ace" || len(*sends) != 2 {
|
|
||||||
t.Errorf("the walk made %+v, want the refused step and then ace's judged send", *sends)
|
|
||||||
}
|
|
||||||
if grants == nil || grants.At != nil || !strings.Contains(grants.Failed, "-postgres.login-n8n") {
|
|
||||||
t.Fatalf("the gate keeps %+v", grants)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The step's exemption is the list's content alone: its path or mode moving is a change like any other.
|
|
||||||
func TestOnlyTheUserListsContentIsExempt(t *testing.T) {
|
|
||||||
list := "nats.bus-users"
|
|
||||||
file := func(fields map[string]any) sentResource {
|
|
||||||
r := map[string]any{"id": list, "type": "file", "path": "/conf/accounts.conf", "mode": "0600", "content": "a"}
|
|
||||||
for k, v := range fields {
|
|
||||||
r[k] = v
|
|
||||||
}
|
|
||||||
body, _ := json.Marshal(map[string]any{"resources": []any{r}})
|
|
||||||
s, err := summarize(body)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return s[0]
|
|
||||||
}
|
|
||||||
was := file(nil)
|
|
||||||
if other := otherThanTheList(diffSent([]sentResource{was}, []sentResource{file(map[string]any{"content": "b"})}), list); len(other) != 0 {
|
|
||||||
t.Errorf("the list's new content reads as more: %v", other)
|
|
||||||
}
|
|
||||||
for _, f := range []map[string]any{{"mode": "0644"}, {"path": "/elsewhere"}, {"content": "b", "owner": "nats"}} {
|
|
||||||
other := otherThanTheList(diffSent([]sentResource{was}, []sentResource{file(f)}), list)
|
|
||||||
if len(other) != 1 || !strings.HasPrefix(other[0], "~"+list) {
|
|
||||||
t.Errorf("the list's resource changed by %v reads %v; want it refused", f, other)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The step keeps the builds of the machine it sends alone: the others its composition resolves are composed
|
|
||||||
// as any send composes them.
|
|
||||||
func TestTheGrantsStepKeepsOneMachinesBuilds(t *testing.T) {
|
|
||||||
ctx := keepingEveryBuild(t.Context(), "novox")
|
|
||||||
if !everyBuildKept(ctx, "novox") || everyBuildKept(ctx, "ace") || everyBuildKept(t.Context(), "novox") {
|
|
||||||
t.Error("the grants step keeps the builds of a machine it does not send")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The store test's manifests are manifests the controller takes in: checked here, where no store is needed,
|
|
||||||
// so a fixture the module rules refuse fails before the store test is run (repo-check of #230 at 325575b2).
|
|
||||||
func TestTheGrantsStepFixturesAreManifests(t *testing.T) {
|
|
||||||
for name, m := range map[string]map[string]any{"nats": natsFixture(), "restic-c1": resticFixture(false),
|
|
||||||
"restic-c2": resticFixture(true), "wallpaper": wallpaperFixture()} {
|
|
||||||
m["module"], m["version"] = strings.SplitN(name, "-", 2)[0], "1"
|
|
||||||
raw, err := json.Marshal(m)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := catalogue.ParseManifest(raw); err != nil {
|
|
||||||
t.Errorf("%s: %v", name, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The delivery plan says the step before the merge.
|
|
||||||
func TestAChangePlanSaysTheGrantsStep(t *testing.T) {
|
|
||||||
const catalogue_ = "http://forge.internal:20000/novox/mesh-catalog.git"
|
|
||||||
entry := func(name string, on ...string) inventory.Entry {
|
|
||||||
e := fromRepo(name, catalogue_, "modules/"+name)
|
|
||||||
e.Source.BuiltFrom = "old"
|
|
||||||
e.On = on
|
|
||||||
return e
|
|
||||||
}
|
|
||||||
nats := entry("nats", "novox")
|
|
||||||
nats.Manifest.BusUsers = "/etc/nats/users.conf"
|
|
||||||
nats.Manifest.Claims = []catalogue.Claim{{Name: catalogue.BrokerSeat, Scope: catalogue.ScopeMesh}}
|
|
||||||
restic := entry("restic", "ace", "novox", "shanks")
|
|
||||||
only := entry("bus-tools", "novox")
|
|
||||||
entries := []inventory.Entry{nats, restic, only}
|
|
||||||
rollsOut := func(string) (inventory.Upgrade, bool) { return inventory.Upgrade{RollOut: true}, true }
|
|
||||||
plan := func(paths ...string) link.ChangePlan {
|
|
||||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: paths}
|
|
||||||
return changePlanOf("novox/mesh-catalog", "main", "head", reachOfMerge(m, entries, nil, nil), entries, rollsOut)
|
|
||||||
}
|
|
||||||
|
|
||||||
text := planText(plan("modules/restic/module.json"))
|
|
||||||
if !strings.Contains(text, "grants step: if this changes what the bus's user list says (a new tool, subject or "+
|
|
||||||
"user), novox is sent that list alone, every build there kept, before restic is judged on its first machine") {
|
|
||||||
t.Errorf("the change plan does not say the grants step:\n%s", text)
|
|
||||||
}
|
|
||||||
// A module only on the bus's machine goes there with the list in its own send: no step of its own.
|
|
||||||
if text := planText(plan("modules/bus-tools/module.json")); strings.Contains(text, "grants step") {
|
|
||||||
t.Errorf("a module on the bus's machine alone reads:\n%s", text)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -84,9 +84,6 @@ var handActVerbs = []handActVerb{
|
|||||||
"deliberate test, never a repair", DecidedFor: []string{causeDrill}},
|
"deliberate test, never a repair", DecidedFor: []string{causeDrill}},
|
||||||
// A drill: something broken on purpose to see the mesh raise and clear it. A person's test, never a
|
// A drill: something broken on purpose to see the mesh raise and clear it. A person's test, never a
|
||||||
// repair, however often it is run.
|
// repair, however often it is run.
|
||||||
// A credential a provider's software issued, taken from the provider and kept for its consumers (novox/hq ADR
|
|
||||||
// 0305): the mesh working as decided, however often the software changes it.
|
|
||||||
{Verb: handActIssued, Decision: "a credential the provider's software issued, kept for its consumers (ADR 0305)"},
|
|
||||||
{Verb: "hand-act drill", Decision: "a drill is a person's deliberate test of the mesh, never a repair"},
|
{Verb: "hand-act drill", Decision: "a drill is a person's deliberate test of the mesh, never a repair"},
|
||||||
// A person's decisions by design.
|
// A person's decisions by design.
|
||||||
{Verb: "retire approve", Decision: "nothing is retired past its bound without a person (ADR 0230)"},
|
{Verb: "retire approve", Decision: "nothing is retired past its bound without a person (ADR 0230)"},
|
||||||
|
|||||||
@@ -128,7 +128,7 @@ func (r *recordedDelivery) grant(context.Context, []readyNode) error { return ni
|
|||||||
|
|
||||||
func (r *recordedDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
|
func (r *recordedDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
|
||||||
r.declared = append(r.declared, s.node)
|
r.declared = append(r.declared, s.node)
|
||||||
return recordSent(ctx, r.inv, s.node, body, s.declared.Builds, s.declared.Epoch, sentRecordOf(ctx, s.declared))
|
return recordSent(ctx, r.inv, s.node, body, s.declared.Builds, s.declared.Epoch)
|
||||||
}
|
}
|
||||||
|
|
||||||
// aResolver is a module built from a repository, at a commit, with something on the machine that
|
// aResolver is a module built from a repository, at a commit, with something on the machine that
|
||||||
|
|||||||
@@ -25,7 +25,6 @@ import (
|
|||||||
// another send reaches anchor, unreported; the gate still passes. And a send that moves a judged module
|
// another send reaches anchor, unreported; the gate still passes. And a send that moves a judged module
|
||||||
// to another build supersedes the judging: no verdict, nothing put back.
|
// to another build supersedes the judging: no verdict, nothing put back.
|
||||||
func TestReplay352(t *testing.T) {
|
func TestReplay352(t *testing.T) {
|
||||||
judgeLeavesAsTheCore(t)
|
|
||||||
t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn)
|
t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn)
|
||||||
t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging)
|
t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging)
|
||||||
}
|
}
|
||||||
@@ -67,11 +66,10 @@ func testANewerSendDoesNotUnreportTheGatesOwn(t *testing.T) {
|
|||||||
|
|
||||||
// A plan's own first send waits while a release judges the same module on that machine with another build.
|
// A plan's own first send waits while a release judges the same module on that machine with another build.
|
||||||
func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
|
func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
|
||||||
judgeLeavesAsTheCore(t)
|
|
||||||
b := aBacklog(t)
|
b := aBacklog(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
advancePlans(ctx, b.open) // the release judges app c2 on anchor
|
advancePlans(ctx, b.open) // the release judges app c2 on anchor
|
||||||
_, _, _, err := gatedSend(ctx, b.open, "anchor", []inventory.CarriedMove{{Module: "app", Node: "anchor", From: "c2", To: "c3", Build: "build-app-c3"}})
|
_, _, err := gatedSend(ctx, b.open, "anchor", []inventory.CarriedMove{{Module: "app", Node: "anchor", From: "c2", To: "c3", Build: "build-app-c3"}})
|
||||||
if !errors.Is(err, errWalkedElsewhere) || !strings.Contains(err.Error(), "release-") {
|
if !errors.Is(err, errWalkedElsewhere) || !strings.Contains(err.Error(), "release-") {
|
||||||
t.Fatalf("a newer build of a judged module was sent under the release's gate: %v", err)
|
t.Fatalf("a newer build of a judged module was sent under the release's gate: %v", err)
|
||||||
}
|
}
|
||||||
@@ -82,7 +80,6 @@ func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
|
|||||||
|
|
||||||
// A merge plan's judging is superseded the same way: another send moved its module on the first machine.
|
// A merge plan's judging is superseded the same way: another send moved its module on the first machine.
|
||||||
func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) {
|
func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) {
|
||||||
judgeLeavesAsTheCore(t)
|
|
||||||
g := aGateMesh(t)
|
g := aGateMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
inv := g.open.inventory
|
inv := g.open.inventory
|
||||||
|
|||||||
@@ -1,427 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"reflect"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// novox/hq issue 459, ADR 0299: since the merge window one walk carries every merge of its batch, so a stop of one
|
|
||||||
// group member through `delivery-stop` ended the walk of every other delivery in it. The stop now names the merges
|
|
||||||
// to leave out: the walk is ended, and every other merge it answered is walked again in a fresh batch without them
|
|
||||||
// — the stopped merge, and each later merge of its repository's branch, which contains it and ends with it.
|
|
||||||
|
|
||||||
// endsItsLine ends, as the test ends, a line the controller's output left open (a walk's ask prints " tier N: " and
|
|
||||||
// the asked build the rest, which the tests' fake ask never prints): `go test -v` would put the test's "--- PASS"
|
|
||||||
// after it, and the store-tests step of merge-check.sh, which reads that marker at a line's start, would say the
|
|
||||||
// test never ran.
|
|
||||||
func endsItsLine(t *testing.T) {
|
|
||||||
t.Cleanup(func() { fmt.Println() })
|
|
||||||
}
|
|
||||||
|
|
||||||
// leaved is a kept merge of a repository, made at a moment.
|
|
||||||
func leaved(repository, commit string, made time.Time) inventory.BatchedMerge {
|
|
||||||
return inventory.BatchedMerge{Repository: repository, Branch: "main", Commit: commit, Merged: made, Heard: made}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The stopped merge is left out, and so is each later merge of its repository's branch, naming the merge it
|
|
||||||
// contains; an earlier merge of that repository and every other repository's merge are walked again.
|
|
||||||
func TestLeftOutAreTheStoppedMergeAndTheMergesThatContainIt(t *testing.T) {
|
|
||||||
c1 := leaved("novox/mesh-catalog", "c1aaaaaaaaaa", t0)
|
|
||||||
c2 := leaved("novox/mesh-catalog", "c2bbbbbbbbbb", t0.Add(10*time.Second))
|
|
||||||
c3 := leaved("novox/mesh-catalog", "c3cccccccccc", t0.Add(20*time.Second))
|
|
||||||
o1 := leaved("novox/one", "o1dddddddddd", t0.Add(5*time.Second))
|
|
||||||
again, out, err := leaveOut([]inventory.BatchedMerge{c1, o1, c2, c3}, []string{"novox/Mesh-Catalog@c2bbbbbbbbbb"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var walked []string
|
|
||||||
for _, m := range again {
|
|
||||||
walked = append(walked, m.Commit)
|
|
||||||
}
|
|
||||||
if !reflect.DeepEqual(walked, []string{c1.Commit, o1.Commit}) {
|
|
||||||
t.Fatalf("walked again: %v; want the earlier catalogue merge and the other repository's", walked)
|
|
||||||
}
|
|
||||||
want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: c2.Commit},
|
|
||||||
{Repository: "novox/mesh-catalog", Commit: c3.Commit, Contains: c2.Commit}}
|
|
||||||
if !reflect.DeepEqual(out, want) {
|
|
||||||
t.Fatalf("left out: %+v; want %+v", out, want)
|
|
||||||
}
|
|
||||||
// A merge the walk does not answer is refused, naming it: nothing ends.
|
|
||||||
if _, _, err := leaveOut([]inventory.BatchedMerge{c1, o1}, []string{"novox/mesh-catalog@ffffffffffff"}); err == nil ||
|
|
||||||
!strings.Contains(err.Error(), "ffffffffffff") {
|
|
||||||
t.Fatalf("a merge the walk does not answer was left out: %v", err)
|
|
||||||
}
|
|
||||||
if _, _, err := leaveOut([]inventory.BatchedMerge{c1}, []string{"no-commit"}); err == nil {
|
|
||||||
t.Fatal("a merge without its commit was taken")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// moving is a kept merge that moved modules when it was heard.
|
|
||||||
func moving(m inventory.BatchedMerge, modules ...string) inventory.BatchedMerge {
|
|
||||||
m.Event, _ = json.Marshal(keptMerge{Moves: modules})
|
|
||||||
return m
|
|
||||||
}
|
|
||||||
|
|
||||||
// A merge of another repository that moves a module the stopped merge moves is left out too, naming the modules
|
|
||||||
// it shares: the module is built at its branch, which holds the stopped merge (review of #232). And so on, until
|
|
||||||
// none more is: a merge sharing a module with that one is left out as well.
|
|
||||||
func TestAMergeSharingAStoppedMergesModuleIsLeftOut(t *testing.T) {
|
|
||||||
stopped := moving(leaved("novox/mesh-catalog", "c1aaaaaaaaaa", t0), "app")
|
|
||||||
reader := moving(leaved("novox/one", "o1bbbbbbbbbb", t0.Add(time.Second)), "app", "one")
|
|
||||||
next := moving(leaved("novox/two", "t1cccccccccc", t0.Add(2*time.Second)), "one")
|
|
||||||
free := moving(leaved("novox/three", "h1dddddddddd", t0.Add(3*time.Second)), "three")
|
|
||||||
again, out, err := leaveOut([]inventory.BatchedMerge{stopped, reader, next, free}, []string{"novox/mesh-catalog@c1aaaaaaa"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(again) != 1 || again[0].Commit != free.Commit {
|
|
||||||
t.Fatalf("walked again: %+v; want the merge that shares nothing alone", again)
|
|
||||||
}
|
|
||||||
want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: stopped.Commit},
|
|
||||||
{Repository: "novox/one", Commit: reader.Commit, Contains: stopped.Commit, Shares: []string{"app"}},
|
|
||||||
{Repository: "novox/two", Commit: next.Commit, Contains: stopped.Commit, Shares: []string{"one"}}}
|
|
||||||
if !reflect.DeepEqual(out, want) {
|
|
||||||
t.Fatalf("left out %+v; want %+v", out, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A walk withholds a module a stopped merge moved where it would build it only as a dependent; one a merge of its
|
|
||||||
// own batch moves is built, and empty tiers go.
|
|
||||||
func TestAWalkWithholdsAStoppedMergesDependents(t *testing.T) {
|
|
||||||
plan := inventory.Plan{Tiers: [][]string{{"one"}, {"app"}, {"notes", "app2"}},
|
|
||||||
Modules: map[string]*inventory.PlanModule{"one": {}, "app": {}, "notes": {}, "app2": {}}}
|
|
||||||
got := withhold(&plan, []string{"app", "notes", "absent"}, []string{"notes"})
|
|
||||||
if !reflect.DeepEqual(got, []string{"app"}) || !reflect.DeepEqual(plan.Tiers, [][]string{{"one"}, {"notes", "app2"}}) {
|
|
||||||
t.Fatalf("withheld %v, tiers %v", got, plan.Tiers)
|
|
||||||
}
|
|
||||||
if _, in := plan.Modules["app"]; in {
|
|
||||||
t.Fatal("a withheld module is still in the plan")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The seat verb passes the merges to leave out to the command.
|
|
||||||
func TestDeliveryStopTakesTheMergesToLeaveOut(t *testing.T) {
|
|
||||||
got, err := argvFor("delivery-stop", map[string]any{"plan": "plan-1", "why": "w", "by": "jochen",
|
|
||||||
"without": "novox/mesh-lab@b1,novox/mesh-lab@b2"})
|
|
||||||
want := []string{"delivery", "stop", "plan-1", "--why", "w", "--by", "mesh-delivery for jochen",
|
|
||||||
"--without", "novox/mesh-lab@b1,novox/mesh-lab@b2"}
|
|
||||||
if err != nil || !reflect.DeepEqual(got, want) {
|
|
||||||
t.Fatalf("delivery-stop with without → %v %v, wanted %v", got, err, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A walk of the catalogue's merge and another repository's is stopped without the catalogue's: it ends stopped,
|
|
||||||
// names the batch that walks the other merge again and the merge it left out, and that batch is cut into a walk
|
|
||||||
// that builds the other repository's module alone.
|
|
||||||
func TestAStopWithoutAMergeWalksTheOthersAgain(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
endsItsLine(t)
|
|
||||||
asksRecorded(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
|
|
||||||
one := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second))
|
|
||||||
hear(t, open, cat, t0.Add(time.Second))
|
|
||||||
hear(t, open, one, t0.Add(6*time.Second))
|
|
||||||
cutAt(t, open, t0.Add(2*time.Minute))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
if len(ws) != 1 || !ws[0].Open() {
|
|
||||||
t.Fatalf("no open walk to stop: %+v", ws)
|
|
||||||
}
|
|
||||||
stopped := ws[0]
|
|
||||||
p, said, err := stopWalkWithout(ctx, open.inventory, stopped.ID, "mesh-delivery for jochen", "its group stopped",
|
|
||||||
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if p.State != inventory.PlanFailed || p.Delivery.Stopped == "" || p.Delivery.WalkedAgainBy == "" ||
|
|
||||||
!reflect.DeepEqual(p.Delivery.LeftOut, []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: cat.Commit}}) {
|
|
||||||
t.Fatalf("the stopped walk reads %s %+v", p.State, p.Delivery)
|
|
||||||
}
|
|
||||||
if said.WalkedAgainBy != p.Delivery.WalkedAgainBy || !reflect.DeepEqual(said.WalkedAgain, []string{"novox/one@" + one.Commit}) {
|
|
||||||
t.Fatalf("the stop answered %+v", said)
|
|
||||||
}
|
|
||||||
again, err := open.inventory.MergesOf(ctx, p.Delivery.WalkedAgainBy)
|
|
||||||
if err != nil || len(again) != 1 || again[0].Commit != one.Commit {
|
|
||||||
t.Fatalf("the fresh batch answers %+v (%v); want the other repository's merge alone", again, err)
|
|
||||||
}
|
|
||||||
if left, _ := open.inventory.MergesOf(ctx, stopped.ID); len(left) != 1 || left[0].Commit != cat.Commit {
|
|
||||||
t.Fatalf("the stopped walk still answers %+v; want the left-out merge alone", left)
|
|
||||||
}
|
|
||||||
cutAt(t, open, t0.Add(4*time.Minute))
|
|
||||||
fresh, err := open.inventory.PlanByID(ctx, p.Delivery.WalkedAgainBy)
|
|
||||||
if err != nil || fresh.Batch() || !fresh.Open() {
|
|
||||||
t.Fatalf("the fresh batch was not cut into a walk: %s %v", fresh.State, err)
|
|
||||||
}
|
|
||||||
if _, in := fresh.Modules["one"]; !in {
|
|
||||||
t.Fatalf("the fresh walk does not build one: %v", fresh.Modules)
|
|
||||||
}
|
|
||||||
if _, in := fresh.Modules["app"]; in {
|
|
||||||
t.Fatalf("the fresh walk builds app, whose merge was left out: %v", fresh.Modules)
|
|
||||||
}
|
|
||||||
if len(fresh.Delivery.Merges) != 1 || fresh.Delivery.Merges[0].Commit != one.Commit {
|
|
||||||
t.Fatalf("the fresh walk answers %+v", fresh.Delivery.Merges)
|
|
||||||
}
|
|
||||||
// A stopped walk starts no search of its own (ADR 0276 decision 3 is for a failed one).
|
|
||||||
cutAt(t, open, t0.Add(5*time.Minute))
|
|
||||||
if alone, _ := open.inventory.AloneMerges(ctx); len(alone) != 0 {
|
|
||||||
t.Fatalf("a merge of the stopped walk waits to be walked alone: %+v", alone)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A later merge of the stopped merge's repository contains it: it is left out too, named with the merge it
|
|
||||||
// contains, and only the other repository's merge is walked again. With nothing left to walk, the walk is only
|
|
||||||
// ended. A merge the walk does not answer is refused, and nothing ends.
|
|
||||||
func TestAStopWithoutAMergeEndsTheMergesThatContainIt(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
endsItsLine(t)
|
|
||||||
asksRecorded(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
first := catalogueMerge("cat1aaaaaaaa", "app", t0)
|
|
||||||
later := catalogueMerge("cat2cccccccc", "notes", t0.Add(10*time.Second))
|
|
||||||
one := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second))
|
|
||||||
hear(t, open, first, t0.Add(time.Second))
|
|
||||||
hear(t, open, one, t0.Add(6*time.Second))
|
|
||||||
hear(t, open, later, t0.Add(11*time.Second))
|
|
||||||
cutAt(t, open, t0.Add(2*time.Minute))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
walk := ws[0]
|
|
||||||
|
|
||||||
if _, _, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "x",
|
|
||||||
[]string{"novox/mesh-catalog@ffffffffffff"}, t0.Add(3*time.Minute)); err == nil {
|
|
||||||
t.Fatal("a stop leaving out a merge the walk does not answer was taken")
|
|
||||||
}
|
|
||||||
if p, _ := open.inventory.PlanByID(ctx, walk.ID); !p.Open() {
|
|
||||||
t.Fatalf("a refused stop ended the walk: %s", p.State)
|
|
||||||
}
|
|
||||||
|
|
||||||
p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped",
|
|
||||||
[]string{"novox/mesh-catalog@" + first.Commit}, t0.Add(3*time.Minute))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: first.Commit},
|
|
||||||
{Repository: "novox/mesh-catalog", Commit: later.Commit, Contains: first.Commit}}
|
|
||||||
if !reflect.DeepEqual(p.Delivery.LeftOut, want) || !reflect.DeepEqual(said.LeftOut, want) {
|
|
||||||
t.Fatalf("left out %+v, said %+v; want %+v", p.Delivery.LeftOut, said.LeftOut, want)
|
|
||||||
}
|
|
||||||
if !strings.Contains(p.Note, later.Commit[:8]) || !strings.Contains(p.Note, p.Delivery.WalkedAgainBy) {
|
|
||||||
t.Fatalf("the stopped walk's note does not name the merge ended with it and the fresh batch: %q", p.Note)
|
|
||||||
}
|
|
||||||
again, _ := open.inventory.MergesOf(ctx, p.Delivery.WalkedAgainBy)
|
|
||||||
if len(again) != 1 || again[0].Commit != one.Commit {
|
|
||||||
t.Fatalf("walked again: %+v; want the other repository's merge alone", again)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Nothing left once the other repository's merge is left out too: the fresh walk is stopped, and nothing more.
|
|
||||||
cutAt(t, open, t0.Add(4*time.Minute))
|
|
||||||
fresh, _ := open.inventory.PlanByID(ctx, p.Delivery.WalkedAgainBy)
|
|
||||||
q, said, err := stopWalkWithout(ctx, open.inventory, fresh.ID, "mesh-delivery for jochen", "that one too",
|
|
||||||
[]string{"novox/one@" + one.Commit}, t0.Add(5*time.Minute))
|
|
||||||
if err != nil || q.State != inventory.PlanFailed || q.Delivery.WalkedAgainBy != "" || said.WalkedAgainBy != "" {
|
|
||||||
t.Fatalf("a stop leaving nothing to walk: %s %+v %+v %v", q.State, q.Delivery, said, err)
|
|
||||||
}
|
|
||||||
if _, bs := walks(t, open); len(bs) != 0 {
|
|
||||||
t.Fatalf("a batch of nothing was made: %+v", bs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A merge heard while the walk ran waits in the open batch, at a later commit of the same repository. The merge
|
|
||||||
// walked again joins that batch, never a batch of its own that would walk the older commit after the newer one
|
|
||||||
// (review of #232): its window closes, and its walk is at the newer commit, answering both.
|
|
||||||
func TestAStopWalksTheOthersAgainInTheOpenBatch(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
endsItsLine(t)
|
|
||||||
asksRecorded(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
|
|
||||||
one1 := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second))
|
|
||||||
hear(t, open, cat, t0.Add(time.Second))
|
|
||||||
hear(t, open, one1, t0.Add(6*time.Second))
|
|
||||||
cutAt(t, open, t0.Add(2*time.Minute))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
walk := ws[0]
|
|
||||||
one2 := repoMerge("one", "one2cccccccc", t0.Add(3*time.Minute))
|
|
||||||
hear(t, open, one2, t0.Add(3*time.Minute))
|
|
||||||
_, bs := walks(t, open)
|
|
||||||
if len(bs) != 1 {
|
|
||||||
t.Fatalf("the merge heard while the walk ran is in %d batches", len(bs))
|
|
||||||
}
|
|
||||||
waiting := bs[0]
|
|
||||||
p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped",
|
|
||||||
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if p.Delivery.WalkedAgainBy != waiting.ID || said.WalkedAgainBy != waiting.ID {
|
|
||||||
t.Fatalf("walked again by %q (said %q); want the open batch %s", p.Delivery.WalkedAgainBy, said.WalkedAgainBy, waiting.ID)
|
|
||||||
}
|
|
||||||
if _, bs := walks(t, open); len(bs) != 1 {
|
|
||||||
t.Fatalf("%d batches; want the open one alone", len(bs))
|
|
||||||
}
|
|
||||||
b, _ := open.inventory.PlanByID(ctx, waiting.ID)
|
|
||||||
if b.Delivery.Batch == nil || !b.Delivery.Batch.Closed || !slices.Contains(b.Delivery.Batch.Withheld, "app") {
|
|
||||||
t.Fatalf("the open batch reads %+v; want its window closed and app withheld", b.Delivery.Batch)
|
|
||||||
}
|
|
||||||
// Its window closed by the stop, not by the clock: cut at once.
|
|
||||||
cutAt(t, open, t0.Add(3*time.Minute+11*time.Second))
|
|
||||||
fresh, _ := open.inventory.PlanByID(ctx, waiting.ID)
|
|
||||||
if fresh.Batch() || fresh.CommitOf("novox/one") != one2.Commit {
|
|
||||||
t.Fatalf("the batch is %s at %q; want it cut, walking the newer commit", fresh.State, fresh.CommitOf("novox/one"))
|
|
||||||
}
|
|
||||||
var answered []string
|
|
||||||
for _, m := range fresh.Delivery.Merges {
|
|
||||||
answered = append(answered, m.Commit+">"+m.Carried)
|
|
||||||
}
|
|
||||||
if !slices.Contains(answered, one1.Commit+">"+one2.Commit) || !slices.Contains(answered, one2.Commit+">") {
|
|
||||||
t.Fatalf("the walk answers %v; want the earlier merge carried by the later", answered)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A stop names what the stopped walk already sent and nothing walks again, on which machines; and a later merge of
|
|
||||||
// the stopped merge's repository that waits in a batch, which will deliver the stopped change. A stopped walk
|
|
||||||
// that walked its merges again is refused a retry, saying why.
|
|
||||||
func TestAStopNamesWhatStaysOnTheMachinesAndWhatStillCarriesIt(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
endsItsLine(t)
|
|
||||||
asksRecorded(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
|
|
||||||
one1 := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second))
|
|
||||||
hear(t, open, cat, t0.Add(time.Second))
|
|
||||||
hear(t, open, one1, t0.Add(6*time.Second))
|
|
||||||
cutAt(t, open, t0.Add(2*time.Minute))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
walk := ws[0]
|
|
||||||
sent := t0.Add(150 * time.Second)
|
|
||||||
for name, m := range walk.Modules {
|
|
||||||
if name == "app" || name == "one" {
|
|
||||||
m.First, m.FirstAt = []string{"anchor"}, &sent
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := open.inventory.SavePlan(ctx, &walk); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
cat2 := catalogueMerge("cat2dddddddd", "notes", t0.Add(3*time.Minute))
|
|
||||||
hear(t, open, cat2, t0.Add(3*time.Minute))
|
|
||||||
_, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped",
|
|
||||||
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !reflect.DeepEqual(said.Kept, []inventory.PlanKept{{Module: "app", Machines: []string{"anchor"}}}) {
|
|
||||||
t.Fatalf("kept %+v; want app on anchor alone (one is walked again)", said.Kept)
|
|
||||||
}
|
|
||||||
p, _ := open.inventory.PlanByID(ctx, walk.ID)
|
|
||||||
if !reflect.DeepEqual(p.Delivery.Kept, said.Kept) || !strings.Contains(p.Note, "app on anchor") {
|
|
||||||
t.Fatalf("the stopped walk's record does not name what stays: %+v %q", p.Delivery.Kept, p.Note)
|
|
||||||
}
|
|
||||||
if len(said.StillCarriedBy) != 1 || !strings.Contains(said.StillCarriedBy[0], cat2.Commit) {
|
|
||||||
t.Fatalf("still carried by %v; want the later catalogue merge waiting in the batch", said.StillCarriedBy)
|
|
||||||
}
|
|
||||||
if _, err := retryPlan(ctx, open, walk.ID); err == nil || !strings.Contains(err.Error(), "was stopped without") {
|
|
||||||
t.Fatalf("a walk stopped without some merges was retried, or refused for another reason: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A batch not yet cut is stopped without a merge: the merge goes to a stopped record of its own, the batch goes on
|
|
||||||
// with the others and withholds the stopped merge's modules; with every merge left out, the batch is stopped.
|
|
||||||
func TestAStopWithoutAMergeLeavesItOutOfABatch(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
endsItsLine(t)
|
|
||||||
asksRecorded(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
|
|
||||||
one := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second))
|
|
||||||
hear(t, open, cat, t0.Add(time.Second))
|
|
||||||
hear(t, open, one, t0.Add(6*time.Second))
|
|
||||||
_, bs := walks(t, open)
|
|
||||||
b := bs[0]
|
|
||||||
r, said, err := stopWalkWithout(ctx, open.inventory, b.ID, "mesh-delivery for jochen", "its group stopped",
|
|
||||||
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(10*time.Second))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if said.WalkedAgainBy != b.ID || said.Stopped == b.ID {
|
|
||||||
t.Fatalf("said %+v; want a stopped record of its own, and the batch walking the others", said)
|
|
||||||
}
|
|
||||||
if left, _ := open.inventory.MergesOf(ctx, said.Stopped); len(left) != 1 || left[0].Commit != cat.Commit {
|
|
||||||
t.Fatalf("the stopped record holds %+v", left)
|
|
||||||
}
|
|
||||||
if stopped, _ := open.inventory.PlanByID(ctx, said.Stopped); stopped.State != inventory.PlanFailed ||
|
|
||||||
stopped.Delivery.Stopped == "" || stopped.Delivery.WalkedAgainBy != b.ID {
|
|
||||||
t.Fatalf("the stopped record reads %s %+v", stopped.State, stopped.Delivery)
|
|
||||||
}
|
|
||||||
_ = r
|
|
||||||
cutAt(t, open, t0.Add(3*time.Minute))
|
|
||||||
w, _ := open.inventory.PlanByID(ctx, b.ID)
|
|
||||||
if _, in := w.Modules["app"]; in || w.Batch() {
|
|
||||||
t.Fatalf("the batch's walk is %s and builds %v; want it cut without app", w.State, w.Modules)
|
|
||||||
}
|
|
||||||
// A batch whose every merge is left out is stopped itself.
|
|
||||||
hear(t, open, catalogueMerge("cat3eeeeeeee", "notes", t0.Add(4*time.Minute)), t0.Add(4*time.Minute))
|
|
||||||
_, bs = walks(t, open)
|
|
||||||
last := bs[0]
|
|
||||||
if _, said, err := stopWalkWithout(ctx, open.inventory, last.ID, "mesh-delivery for jochen", "x",
|
|
||||||
[]string{"novox/mesh-catalog@cat3eeeeeeee"}, t0.Add(4*time.Minute+time.Second)); err != nil || said.Stopped != last.ID {
|
|
||||||
t.Fatalf("a batch left with nothing: %+v %v", said, err)
|
|
||||||
}
|
|
||||||
if p, _ := open.inventory.PlanByID(ctx, last.ID); p.State != inventory.PlanFailed || p.Batch() {
|
|
||||||
t.Fatalf("the emptied batch is %s", p.State)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The stopped walk, the batch and the merges moved are one act (ADR 0299 decision 7): a plan written by somebody
|
|
||||||
// else since it was read refuses all of it, and no merge moves, no plan is written.
|
|
||||||
func TestSavingPlansAndMovingMergesIsOneAct(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
endsItsLine(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
hear(t, open, catalogueMerge("cat1aaaaaaaa", "app", t0), t0.Add(time.Second))
|
|
||||||
_, bs := walks(t, open)
|
|
||||||
stale := bs[0]
|
|
||||||
moved := stale
|
|
||||||
moved.Note = "written by somebody else"
|
|
||||||
if err := open.inventory.SavePlan(ctx, &moved); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
fresh := inventory.Plan{ID: "plan-fresh", Repository: "novox/mesh-catalog", Branch: "main", Commit: "x", Created: t0,
|
|
||||||
State: inventory.PlanAssembling, Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{}}
|
|
||||||
err := open.inventory.SavePlansMoving(ctx, []*inventory.Plan{&fresh, &stale},
|
|
||||||
[]inventory.MergeMove{{Repository: "novox/mesh-catalog", Commit: "cat1aaaaaaaa", Plan: "plan-fresh"}})
|
|
||||||
if !errors.Is(err, inventory.ErrPlanMoved) {
|
|
||||||
t.Fatalf("a stale plan in the act: %v", err)
|
|
||||||
}
|
|
||||||
if _, err := open.inventory.PlanByID(ctx, "plan-fresh"); err == nil {
|
|
||||||
t.Fatal("the new plan was written though the act was refused")
|
|
||||||
}
|
|
||||||
if m, _, _ := open.inventory.MergeOf(ctx, "novox/mesh-catalog", "cat1aaaaaaaa"); m.Plan != stale.ID {
|
|
||||||
t.Fatalf("the merge moved to %q though the act was refused", m.Plan)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// What a stopped walk leaves on the machines names every machine its sends reached (review of #232): a module sent
|
|
||||||
// with no first machine reached every machine running it; one sent first reached those, and the rest it was sent
|
|
||||||
// to; a module a merge walked again moves is sent again, and is not named.
|
|
||||||
func TestKeptNamesEveryMachineASendReached(t *testing.T) {
|
|
||||||
at := t0
|
|
||||||
p := inventory.Plan{Modules: map[string]*inventory.PlanModule{
|
|
||||||
"together": {SentAt: &at},
|
|
||||||
"first": {First: []string{"anchor"}, FirstAt: &at, Rest: map[string]inventory.SentDeclaration{"laptop": {}}},
|
|
||||||
"again": {SentAt: &at},
|
|
||||||
"unsent": {},
|
|
||||||
}}
|
|
||||||
got := keptOf(p, []string{"again"}, map[string][]string{"together": {"laptop", "anchor"}, "again": {"anchor"}})
|
|
||||||
want := []inventory.PlanKept{{Module: "first", Machines: []string{"anchor", "laptop"}},
|
|
||||||
{Module: "together", Machines: []string{"anchor", "laptop"}}}
|
|
||||||
if !reflect.DeepEqual(got, want) {
|
|
||||||
t.Fatalf("kept %+v; want %+v", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -77,7 +77,7 @@ func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) {
|
|||||||
}
|
}
|
||||||
cancel() // the sender is going away: its context is cancelled between the send and the record
|
cancel() // the sender is going away: its context is cancelled between the send and the record
|
||||||
body := []byte(`{"declaration":1,"resources":[]}`)
|
body := []byte(`{"declaration":1,"resources":[]}`)
|
||||||
digest, err := recordSent(ctx, inv, "anchor", body, nil, 0, sentRecord{sender: "a test"})
|
digest, err := recordSent(ctx, inv, "anchor", body, nil, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// NodeByName on the cancelled context may itself refuse; the record must still be possible
|
// NodeByName on the cancelled context may itself refuse; the record must still be possible
|
||||||
// through the detached context, so look the node up again on a live one.
|
// through the detached context, so look the node up again on a live one.
|
||||||
|
|||||||
@@ -1,272 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
"github.com/novox/mesh-controller/internal/secrets"
|
|
||||||
)
|
|
||||||
|
|
||||||
// `secret issued <provision>`: the credential a provider's software issued, taken from the provider and kept as
|
|
||||||
// the provision's one credential for every consumer (novox/hq ADR 0305, issue 526).
|
|
||||||
//
|
|
||||||
// **The value comes from the provider, never from whoever asks.** A module served by its machine's runtime calls
|
|
||||||
// under the runtime's one bus account, which carries that machine's agents' calls too, so a value handed to a
|
|
||||||
// verb could be anybody's. This line carries none: it asks the provider's own tool, on the provider's machine, in
|
|
||||||
// the controller's own name — the subject that names its caller, which the bus lets the controller alone publish
|
|
||||||
// (issue 365) — with a sealing key made for this one call. The tool answers only the controller, the value sealed
|
|
||||||
// to that key; the controller opens it, seals it to every holder and forgets it, as the desk's value is (ADR 0277
|
|
||||||
// §23). Asking again fetches the same value again, and a value every holder already holds is not sealed anew.
|
|
||||||
//
|
|
||||||
// **What this trusts: the provider's machine.** The answer is taken from whoever answers the provider's tool
|
|
||||||
// subject on the provider's machine first. On that machine its runtime serves the subject for every module it
|
|
||||||
// carries, so anything that runtime runs, or anything else granted that subject there, could answer before the
|
|
||||||
// provider does and hand the controller a value of its own. ADR 0305 accepts that by trusting the provider's
|
|
||||||
// machine, as the provider itself is trusted; the bus keeps every other machine from answering it.
|
|
||||||
|
|
||||||
// handActIssued is the verb a change of an issued value is recorded under: the mesh working as decided.
|
|
||||||
const handActIssued = "secret issued"
|
|
||||||
|
|
||||||
// issuedAsk is how long the provider's tool is given to answer.
|
|
||||||
const issuedAsk = 20 * time.Second
|
|
||||||
|
|
||||||
// issuedProvider is one assignment of a module whose offer says its software issues the provision's credential.
|
|
||||||
type issuedProvider struct {
|
|
||||||
Node, Module, Tool string
|
|
||||||
}
|
|
||||||
|
|
||||||
// issuedTaker is the line's work, with what it reaches handed in: the store, the bus and the sends.
|
|
||||||
type issuedTaker struct {
|
|
||||||
// providers are the assignments to ask, and whether any module offers the provision so at all.
|
|
||||||
providers func(provision string) ([]issuedProvider, bool, error)
|
|
||||||
ask func(p issuedProvider, args map[string]any) (link.Answer, error)
|
|
||||||
keep func(provision, provider, value string) (inventory.IssuedKept, error)
|
|
||||||
record func(act link.HandAct) error
|
|
||||||
send func(machines []string) error
|
|
||||||
}
|
|
||||||
|
|
||||||
// take asks every provider of the provision for its software's value and keeps each; it answers what it did, in
|
|
||||||
// words that never carry the value or its fingerprint.
|
|
||||||
func (t issuedTaker) take(provision string) (string, error) {
|
|
||||||
providers, offered, err := t.providers(provision)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if !offered {
|
|
||||||
return "", fmt.Errorf("no module offers %q with a credential its software issues (\"credential\": "+
|
|
||||||
"{\"issued-by\": \"software\", \"tool\": …}, ADR 0305), so there is nothing to take from a provider; "+
|
|
||||||
"nothing was asked", provision)
|
|
||||||
}
|
|
||||||
if len(providers) == 0 {
|
|
||||||
return fmt.Sprintf("no consumer holds %s from a provider whose software issues it yet, so there is nothing "+
|
|
||||||
"to seal; a consumer that binds holds a value the mesh made until the provider next asks", provision), nil
|
|
||||||
}
|
|
||||||
var said []string
|
|
||||||
var failed []string
|
|
||||||
for _, p := range providers {
|
|
||||||
words, err := t.takeFrom(provision, p)
|
|
||||||
if err != nil {
|
|
||||||
failed = append(failed, fmt.Sprintf("%s on %s: %v", p.Module, p.Node, err))
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
said = append(said, words)
|
|
||||||
}
|
|
||||||
out := strings.Join(said, "\n")
|
|
||||||
if len(failed) > 0 {
|
|
||||||
return out, fmt.Errorf("%s was not taken from every provider:\n %s", provision, strings.Join(failed, "\n "))
|
|
||||||
}
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// takeFrom is one provider: asked, opened, kept, recorded and sent when it changed.
|
|
||||||
func (t issuedTaker) takeFrom(provision string, p issuedProvider) (string, error) {
|
|
||||||
public, private, err := secrets.Keypair()
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("no key could be made to take the value: %w", err)
|
|
||||||
}
|
|
||||||
answer, err := t.ask(p, map[string]any{"provision": provision, "seal_to": public})
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("its tool %s could not be asked: %w", p.Tool, err)
|
|
||||||
}
|
|
||||||
if answer.Error != "" {
|
|
||||||
return "", fmt.Errorf("its tool %s refused: %s", p.Tool, answer.Error)
|
|
||||||
}
|
|
||||||
var given struct {
|
|
||||||
Sealed string `json:"sealed"`
|
|
||||||
}
|
|
||||||
if err := unmarshalAnswer(answer, &given); err != nil || given.Sealed == "" {
|
|
||||||
return "", fmt.Errorf("its tool %s answered no sealed value", p.Tool)
|
|
||||||
}
|
|
||||||
opened, err := secrets.Open(private, given.Sealed)
|
|
||||||
if err != nil {
|
|
||||||
// Never the value, never what failed to open: only that it was not sealed to this call.
|
|
||||||
return "", fmt.Errorf("its tool %s answered a value not sealed to this call; nothing was taken", p.Tool)
|
|
||||||
}
|
|
||||||
value := strings.TrimSpace(string(opened))
|
|
||||||
for i := range opened {
|
|
||||||
opened[i] = 0
|
|
||||||
}
|
|
||||||
if value == "" {
|
|
||||||
return "", fmt.Errorf("its tool %s answered an empty value; nothing was taken", p.Tool)
|
|
||||||
}
|
|
||||||
kept, err := t.keep(provision, p.Node, value)
|
|
||||||
value = ""
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
holders := make([]string, 0, len(kept.Holders))
|
|
||||||
for _, h := range kept.Holders {
|
|
||||||
holders = append(holders, fmt.Sprintf("%s on %s%s", h.ConsumerModule, h.Consumer, asLocal(h.Local)))
|
|
||||||
}
|
|
||||||
words := ""
|
|
||||||
if len(kept.Unsealable) > 0 {
|
|
||||||
var names []string
|
|
||||||
for _, h := range kept.Unsealable {
|
|
||||||
names = append(names, fmt.Sprintf("%s on %s", h.ConsumerModule, h.Consumer))
|
|
||||||
}
|
|
||||||
words = fmt.Sprintf("\n not sealed to %s: no sealing key on that machine", strings.Join(names, ", "))
|
|
||||||
}
|
|
||||||
if !kept.Changed {
|
|
||||||
return fmt.Sprintf("%s from %s on %s: every holder already holds the value its software issued (%s); "+
|
|
||||||
"nothing was sent%s", provision, p.Module, p.Node, strings.Join(holders, ", "), words), nil
|
|
||||||
}
|
|
||||||
sealedTo := make([]string, 0, len(kept.Sealed))
|
|
||||||
for _, h := range kept.Sealed {
|
|
||||||
sealedTo = append(sealedTo, fmt.Sprintf("%s on %s%s", h.ConsumerModule, h.Consumer, asLocal(h.Local)))
|
|
||||||
}
|
|
||||||
recorded := ""
|
|
||||||
if err := t.record(link.HandAct{Verb: handActIssued, Args: []string{provision},
|
|
||||||
Why: fmt.Sprintf("%s on %s gave the credential its software issued for %s; sealed to %s", p.Module, p.Node,
|
|
||||||
provision, strings.Join(sealedTo, ", ")),
|
|
||||||
Cause: "issued-by-software"}); err != nil {
|
|
||||||
recorded = fmt.Sprintf("\n this was NOT recorded in the hand-act log, and is done anyway: %v", err)
|
|
||||||
}
|
|
||||||
if err := t.send(kept.Machines); err != nil {
|
|
||||||
return "", fmt.Errorf("%s from %s on %s is sealed to %s and could not be sent: %w; `push --behind` sends it",
|
|
||||||
provision, p.Module, p.Node, strings.Join(sealedTo, ", "), err)
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("%s from %s on %s: the value its software issued is sealed to %s; sent %s, and each of "+
|
|
||||||
"them starts again on it%s%s", provision, p.Module, p.Node, strings.Join(sealedTo, ", "),
|
|
||||||
strings.Join(kept.Machines, ", "), words, recorded), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// issuedProvidersOf are the assignments to ask for a provision: every machine providing it to a pair the mesh
|
|
||||||
// holds, and on it the assigned module whose offer says its software issues the credential (ADR 0305 §1).
|
|
||||||
func issuedProvidersOf(ctx context.Context, inv *inventory.Inventory, provision string) ([]issuedProvider, bool, error) {
|
|
||||||
shelf, err := inv.Catalogue(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, false, err
|
|
||||||
}
|
|
||||||
if !offersIssued(shelf, provision) {
|
|
||||||
return nil, false, nil
|
|
||||||
}
|
|
||||||
holders, err := inv.HoldersOf(ctx, provision, "")
|
|
||||||
if err != nil {
|
|
||||||
return nil, true, err
|
|
||||||
}
|
|
||||||
nodes := map[string]bool{}
|
|
||||||
for _, h := range holders {
|
|
||||||
nodes[h.Provider] = true
|
|
||||||
}
|
|
||||||
names := make([]string, 0, len(nodes))
|
|
||||||
for n := range nodes {
|
|
||||||
names = append(names, n)
|
|
||||||
}
|
|
||||||
sort.Strings(names)
|
|
||||||
var out []issuedProvider
|
|
||||||
for _, node := range names {
|
|
||||||
assigned, err := inv.Assigned(ctx, node)
|
|
||||||
if err != nil {
|
|
||||||
return nil, true, err
|
|
||||||
}
|
|
||||||
out = append(out, issuedOn(shelf, node, assigned, provision)...)
|
|
||||||
}
|
|
||||||
return out, true, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// offersIssued is whether any module offers the provision with a credential its software issues.
|
|
||||||
func offersIssued(shelf map[string]catalogue.Manifest, provision string) bool {
|
|
||||||
for _, m := range shelf {
|
|
||||||
if _, issued := m.IssuedCredentialOf(provision); issued {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// issuedOn are the modules assigned to one machine whose offer says their software issues the provision's
|
|
||||||
// credential, each with the tool that answers it.
|
|
||||||
func issuedOn(shelf map[string]catalogue.Manifest, node string, assigned []string, provision string) []issuedProvider {
|
|
||||||
var out []issuedProvider
|
|
||||||
for _, module := range assigned {
|
|
||||||
if tool, issued := shelf[module].IssuedCredentialOf(provision); issued {
|
|
||||||
out = append(out, issuedProvider{Node: node, Module: module, Tool: tool})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// issuedRefusesRotation refuses `rotate` of a provision its provider's software issues (ADR 0305 §9): a value the
|
|
||||||
// mesh made would replace one the software holds, and every consumer would hold a dead credential until the
|
|
||||||
// provider next asks.
|
|
||||||
func issuedRefusesRotation(shelf map[string]catalogue.Manifest, provision string) error {
|
|
||||||
if !offersIssued(shelf, provision) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return fmt.Errorf("%s's credential is issued by its provider's software, which the mesh cannot make or "+
|
|
||||||
"rotate (ADR 0305): change it in that software, and `secret issued %s` takes the value it holds now. "+
|
|
||||||
"Nothing was rotated", provision, provision)
|
|
||||||
}
|
|
||||||
|
|
||||||
// secretIssued is `secret issued <provision>`, on this controller's stores and bus.
|
|
||||||
func secretIssued(ctx context.Context, args []string) error {
|
|
||||||
if len(args) != 1 || args[0] == "" || strings.HasPrefix(args[0], "-") {
|
|
||||||
return errors.New("secret issued <provision>")
|
|
||||||
}
|
|
||||||
provision := args[0]
|
|
||||||
open, err := openStores(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer open.Close()
|
|
||||||
t := issuedTaker{
|
|
||||||
providers: func(provision string) ([]issuedProvider, bool, error) {
|
|
||||||
return issuedProvidersOf(ctx, open.inventory, provision)
|
|
||||||
},
|
|
||||||
ask: func(p issuedProvider, args map[string]any) (link.Answer, error) {
|
|
||||||
var answer link.Answer
|
|
||||||
err := onTheBus(func(conn *nats.Conn) error {
|
|
||||||
var err error
|
|
||||||
answer, err = link.AskModuleToolOnAs(ctx, conn, broker.ControllerName, p.Module, p.Tool, p.Node,
|
|
||||||
args, issuedAsk)
|
|
||||||
return err
|
|
||||||
})
|
|
||||||
return answer, err
|
|
||||||
},
|
|
||||||
keep: func(provision, provider, value string) (inventory.IssuedKept, error) {
|
|
||||||
return open.inventory.KeepIssuedSecret(ctx, provision, provider, value)
|
|
||||||
},
|
|
||||||
record: func(act link.HandAct) error {
|
|
||||||
return onTheBus(func(conn *nats.Conn) error {
|
|
||||||
_, err := link.RecordHandAct(ctx, conn, act)
|
|
||||||
return err
|
|
||||||
})
|
|
||||||
},
|
|
||||||
send: func(machines []string) error { return sendTo(ctx, open, machines) },
|
|
||||||
}
|
|
||||||
words, err := t.take(provision)
|
|
||||||
if words != "" {
|
|
||||||
fmt.Println(words)
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
@@ -1,218 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
"github.com/novox/mesh-controller/internal/secrets"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The `issued` verb runs `secret issued <provision>` and nothing else, and that line is the one `secret` line
|
|
||||||
// beside rotate and the desk's that a verb may run (novox/hq ADR 0305 §3, §10, issue 526).
|
|
||||||
func TestTheIssuedVerbRunsTheOneLineThatCarriesNoValue(t *testing.T) {
|
|
||||||
argv, err := argvFor("issued", map[string]any{"provision": "plex-api"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if strings.Join(argv, " ") != "secret issued plex-api" {
|
|
||||||
t.Fatalf("the verb composed %q", argv)
|
|
||||||
}
|
|
||||||
if _, err := argvFor("issued", map[string]any{}); err == nil {
|
|
||||||
t.Fatal("the verb ran with no provision")
|
|
||||||
}
|
|
||||||
if _, err := argvFor("issued", map[string]any{"provision": "plex-api", "secret": "a value"}); err == nil {
|
|
||||||
t.Fatal("the verb took a value beside the provision")
|
|
||||||
}
|
|
||||||
for _, line := range [][]string{
|
|
||||||
{"secret", "issued", "plex-api", "--from", "/tmp/x"},
|
|
||||||
{"secret", "issued", "--from"},
|
|
||||||
{"secret", "issued"},
|
|
||||||
{"secret", "accept", "ace", "kometa", "plex-api", "--provider", "ace"},
|
|
||||||
{"secret", "recover", "ace", "kometa", "plex-api"},
|
|
||||||
} {
|
|
||||||
if err := terminalOnly(line); err == nil {
|
|
||||||
t.Errorf("%q ran through a verb", line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := terminalOnly([]string{"secret", "issued", "plex-api"}); err != nil {
|
|
||||||
t.Fatalf("the issued line was refused through its verb: %v", err)
|
|
||||||
}
|
|
||||||
found := false
|
|
||||||
for _, v := range catalogue.ControllerVerbs {
|
|
||||||
if v.Name == "issued" {
|
|
||||||
found = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !found {
|
|
||||||
t.Fatal("the controller's seat does not declare the issued verb")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A provider's tool answering the value sealed to the call's key: what a provider does (ADR 0305 §2).
|
|
||||||
func issuedSealed(t *testing.T, value string) func(issuedProvider, map[string]any) (link.Answer, error) {
|
|
||||||
return func(_ issuedProvider, args map[string]any) (link.Answer, error) {
|
|
||||||
key, _ := args["seal_to"].(string)
|
|
||||||
if key == "" {
|
|
||||||
t.Fatal("the provider was asked without a key to seal to")
|
|
||||||
}
|
|
||||||
if _, carried := args["value"]; carried {
|
|
||||||
t.Fatal("the ask carried a value")
|
|
||||||
}
|
|
||||||
sealed, err := secrets.Seal(key, []byte(value))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
raw, _ := json.Marshal(map[string]string{"sealed": sealed})
|
|
||||||
return link.Answer{Result: raw}, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type issuedRecorder struct {
|
|
||||||
kept []string
|
|
||||||
acts []link.HandAct
|
|
||||||
sent [][]string
|
|
||||||
again bool // the value is held by every holder already
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *issuedRecorder) taker(t *testing.T, ask func(issuedProvider, map[string]any) (link.Answer, error)) issuedTaker {
|
|
||||||
return issuedTaker{
|
|
||||||
providers: func(provision string) ([]issuedProvider, bool, error) {
|
|
||||||
return []issuedProvider{{Node: "ace", Module: "plex", Tool: "plex_issued_credential"}}, true, nil
|
|
||||||
},
|
|
||||||
ask: ask,
|
|
||||||
keep: func(provision, provider, value string) (inventory.IssuedKept, error) {
|
|
||||||
r.kept = append(r.kept, value)
|
|
||||||
holders := []inventory.Holder{{Provision: provision, Consumer: "ace", ConsumerModule: "kometa", Provider: provider},
|
|
||||||
{Provision: provision, Consumer: "ace", ConsumerModule: "tautulli", Provider: provider}}
|
|
||||||
if r.again {
|
|
||||||
return inventory.IssuedKept{Holders: holders}, nil
|
|
||||||
}
|
|
||||||
return inventory.IssuedKept{Changed: true, Holders: holders, Sealed: holders, Machines: []string{"ace"}}, nil
|
|
||||||
},
|
|
||||||
record: func(act link.HandAct) error { r.acts = append(r.acts, act); return nil },
|
|
||||||
send: func(machines []string) error { r.sent = append(r.sent, machines); return nil },
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The value the provider's tool sealed to the call is kept, recorded and sent, and the answer never says it.
|
|
||||||
func TestAnIssuedValueIsTakenSealedFromTheProviderAndSent(t *testing.T) {
|
|
||||||
var r issuedRecorder
|
|
||||||
words, err := r.taker(t, issuedSealed(t, "the-servers-own-token")).take("plex-api")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(r.kept) != 1 || r.kept[0] != "the-servers-own-token" {
|
|
||||||
t.Fatalf("the provider's value was not kept: %d kept", len(r.kept))
|
|
||||||
}
|
|
||||||
if len(r.sent) != 1 || strings.Join(r.sent[0], ",") != "ace" {
|
|
||||||
t.Fatalf("the holders' machines were not sent: %v", r.sent)
|
|
||||||
}
|
|
||||||
if len(r.acts) != 1 || r.acts[0].Verb != handActIssued {
|
|
||||||
t.Fatalf("the change was not recorded in the hand-act log: %+v", r.acts)
|
|
||||||
}
|
|
||||||
for _, said := range []string{words, r.acts[0].Why, strings.Join(r.acts[0].Args, " ")} {
|
|
||||||
if strings.Contains(said, "the-servers-own-token") || strings.Contains(said, inventory.IssuedStamp("the-servers-own-token")) {
|
|
||||||
t.Fatalf("the value or its fingerprint was said: %q", said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !strings.Contains(words, "kometa on ace") || !strings.Contains(words, "tautulli on ace") {
|
|
||||||
t.Fatalf("the answer does not name the holders: %q", words)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The same value again writes nothing, records nothing and sends nobody (ADR 0305 §5).
|
|
||||||
func TestAnIssuedValueEveryHolderHoldsSendsNobody(t *testing.T) {
|
|
||||||
r := issuedRecorder{again: true}
|
|
||||||
words, err := r.taker(t, issuedSealed(t, "the-servers-own-token")).take("plex-api")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(r.sent) != 0 || len(r.acts) != 0 {
|
|
||||||
t.Fatalf("an unchanged value was sent or recorded: %v %v", r.sent, r.acts)
|
|
||||||
}
|
|
||||||
if !strings.Contains(words, "nothing was sent") {
|
|
||||||
t.Fatalf("the answer does not say nothing was sent: %q", words)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A value not sealed to the call's key is not taken, nor is a refusal, nor an empty answer (ADR 0305 §2).
|
|
||||||
func TestOnlyAValueSealedToTheCallIsTaken(t *testing.T) {
|
|
||||||
other, _, _ := secrets.Keypair()
|
|
||||||
elsewhere, _ := secrets.Seal(other, []byte("somebody's value"))
|
|
||||||
raw, _ := json.Marshal(map[string]string{"sealed": elsewhere})
|
|
||||||
plain, _ := json.Marshal(map[string]string{"value": "the-servers-own-token"})
|
|
||||||
for name, ask := range map[string]func(issuedProvider, map[string]any) (link.Answer, error){
|
|
||||||
"sealed to another key": func(issuedProvider, map[string]any) (link.Answer, error) { return link.Answer{Result: raw}, nil },
|
|
||||||
"answered in the clear": func(issuedProvider, map[string]any) (link.Answer, error) { return link.Answer{Result: plain}, nil },
|
|
||||||
"refused": func(issuedProvider, map[string]any) (link.Answer, error) {
|
|
||||||
return link.Answer{Error: "the controller alone"}, nil
|
|
||||||
},
|
|
||||||
"not reached": func(issuedProvider, map[string]any) (link.Answer, error) {
|
|
||||||
return link.Answer{}, errors.New("503 no responders")
|
|
||||||
},
|
|
||||||
"sealed empty": issuedSealed(t, " "),
|
|
||||||
} {
|
|
||||||
var r issuedRecorder
|
|
||||||
if _, err := r.taker(t, ask).take("plex-api"); err == nil {
|
|
||||||
t.Errorf("%s: taken", name)
|
|
||||||
}
|
|
||||||
if len(r.kept) != 0 || len(r.sent) != 0 {
|
|
||||||
t.Errorf("%s: something was kept or sent", name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A provision no offer says its software issues is refused before anything is asked.
|
|
||||||
func TestAProvisionNoOfferSaysIsIssuedIsRefused(t *testing.T) {
|
|
||||||
asked := false
|
|
||||||
taker := issuedTaker{
|
|
||||||
providers: func(string) ([]issuedProvider, bool, error) { return nil, false, nil },
|
|
||||||
ask: func(issuedProvider, map[string]any) (link.Answer, error) {
|
|
||||||
asked = true
|
|
||||||
return link.Answer{}, nil
|
|
||||||
},
|
|
||||||
}
|
|
||||||
if _, err := taker.take("postgres-database"); err == nil || !strings.Contains(err.Error(), "ADR 0305") {
|
|
||||||
t.Fatalf("expected a refusal naming the record, got %v", err)
|
|
||||||
}
|
|
||||||
if asked {
|
|
||||||
t.Fatal("a provider was asked for a provision no offer says its software issues")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Which assignments are asked: the modules on the providing machine whose offer says so, with their tool.
|
|
||||||
func TestTheProvidersAskedAreTheOffersThatSaySo(t *testing.T) {
|
|
||||||
shelf := map[string]catalogue.Manifest{
|
|
||||||
"plex": {Module: "plex", Provides: []catalogue.Offer{{Name: "plex-api", Credential: &catalogue.OfferCredential{
|
|
||||||
IssuedBy: catalogue.IssuedBySoftware, Tool: "plex_issued_credential"}}}},
|
|
||||||
"jellyfin": {Module: "jellyfin", Provides: []catalogue.Offer{{Name: "plex-api"}}},
|
|
||||||
}
|
|
||||||
got := issuedOn(shelf, "ace", []string{"jellyfin", "plex", "kometa"}, "plex-api")
|
|
||||||
if len(got) != 1 || got[0] != (issuedProvider{Node: "ace", Module: "plex", Tool: "plex_issued_credential"}) {
|
|
||||||
t.Fatalf("the providers asked: %+v", got)
|
|
||||||
}
|
|
||||||
if err := issuedRefusesRotation(shelf, "plex-api"); err == nil || !strings.Contains(err.Error(), "secret issued plex-api") {
|
|
||||||
t.Fatalf("rotate of an issued provision was not refused with the way to take it: %v", err)
|
|
||||||
}
|
|
||||||
if err := issuedRefusesRotation(shelf, "postgres-database"); err != nil {
|
|
||||||
t.Fatalf("rotate of an ordinary provision was refused: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A change of an issued value is the mesh working as decided, never a repair (ADR 0305 §8).
|
|
||||||
func TestAnIssuedValueIsADecisionInTheHandActLog(t *testing.T) {
|
|
||||||
for _, v := range handActVerbs {
|
|
||||||
if v.Verb == handActIssued {
|
|
||||||
if v.Decision == "" {
|
|
||||||
t.Fatal("secret issued counts as a repair")
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
t.Fatal("secret issued is not in the hand-act table")
|
|
||||||
}
|
|
||||||
@@ -1,160 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// novox/hq issue 485: the mesh records every send — sender, lease epoch, generation, modules — and only `plan`
|
|
||||||
// said it. `status` says per machine when it was last sent a declaration and by whom, in one line; `node show`
|
|
||||||
// says the last send in full and what the machine answered; and `status --json` carries it as fields, since
|
|
||||||
// the store's reader may not read the controller's tables and these verbs are the only window onto them.
|
|
||||||
|
|
||||||
func aLastSend(answer inventory.SendAnswer) inventory.Send {
|
|
||||||
sent := time.Date(2026, 10, 11, 18, 40, 5, 0, time.Local)
|
|
||||||
return inventory.Send{NodeName: "laptop", Sequence: 12, Epoch: 57,
|
|
||||||
Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12",
|
|
||||||
Modules: []string{"docker", "sudo"}, SentAt: sent, Recorded: true, Answer: answer}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStatusSaysEachMachinesLastSendInOneLine(t *testing.T) {
|
|
||||||
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.Local)
|
|
||||||
asked := answers{
|
|
||||||
nodes: []inventory.Node{{Name: "anchor", LastSeen: time.Now()}, {Name: "laptop", LastSeen: time.Now()}},
|
|
||||||
lastSent: map[string]inventory.Send{"laptop": aLastSend(inventory.SendAnswer{
|
|
||||||
Outcome: inventory.OutcomeApplied, At: &at})},
|
|
||||||
}
|
|
||||||
shown := printed(t, func() error { return printStatus(asked) })
|
|
||||||
var laptop, anchor []string
|
|
||||||
for _, line := range strings.Split(shown, "\n") {
|
|
||||||
fields := strings.Fields(line)
|
|
||||||
if len(fields) > 0 && fields[0] == "laptop" {
|
|
||||||
laptop = append(laptop, line)
|
|
||||||
}
|
|
||||||
if len(fields) > 0 && fields[0] == "anchor" {
|
|
||||||
anchor = append(anchor, line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(laptop) != 1 || !strings.Contains(laptop[0], "2026-10-11 18:40") ||
|
|
||||||
!strings.Contains(laptop[0], "by a one-shot push by jochen at a shell on anchor") ||
|
|
||||||
!strings.Contains(laptop[0], "generation 40") || !strings.Contains(laptop[0], "applied") {
|
|
||||||
t.Fatalf("status does not say laptop's last send in one line:\n%s", shown)
|
|
||||||
}
|
|
||||||
if len(anchor) != 1 || !strings.Contains(anchor[0], "no send recorded") {
|
|
||||||
t.Fatalf("status does not say anchor has no send recorded, in one line:\n%s", shown)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStatusSaysTheLastSendsCouldNotBeRead(t *testing.T) {
|
|
||||||
asked := answers{nodes: []inventory.Node{{Name: "laptop", LastSeen: time.Now()}},
|
|
||||||
lastSentUnread: "the store went away"}
|
|
||||||
if shown := printed(t, func() error { return printStatus(asked) }); !strings.Contains(shown, "the store went away") {
|
|
||||||
t.Fatalf("a record of sends that could not be read is not said:\n%s", shown)
|
|
||||||
}
|
|
||||||
if asked.well() {
|
|
||||||
t.Error("a mesh whose sends could not be read is called well")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNodeShowSaysTheLastSendInFull(t *testing.T) {
|
|
||||||
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.Local)
|
|
||||||
refused := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeRefused,
|
|
||||||
Refused: "unknown field \"generation\"\nsecond line", At: &at})), "\n")
|
|
||||||
for _, want := range []string{"sequence 12", "2026-10-11 18:40:05", "a one-shot push by jochen at a shell on anchor",
|
|
||||||
"lease epoch 57", "generation 40", "docker, sudo", "refused", "unknown field \"generation\""} {
|
|
||||||
if !strings.Contains(refused, want) {
|
|
||||||
t.Errorf("node show's last send does not say %q:\n%s", want, refused)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if applied := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeApplied,
|
|
||||||
At: &at})), "\n"); !strings.Contains(applied, "applied it at 2026-10-11 18:41:00") {
|
|
||||||
t.Errorf("an applied send is not said applied:\n%s", applied)
|
|
||||||
}
|
|
||||||
if waiting := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{})), "\n"); !strings.Contains(waiting,
|
|
||||||
"not reported on it yet") {
|
|
||||||
t.Errorf("a send not reported on is not said so:\n%s", waiting)
|
|
||||||
}
|
|
||||||
stale := aLastSend(inventory.SendAnswer{})
|
|
||||||
when := at
|
|
||||||
stale.RefusedAt, stale.RefusedApplied = &when, 44
|
|
||||||
if s := strings.Join(lastSendLines(stale), "\n"); !strings.Contains(s, "refused it at 2026-10-11 18:41:00") ||
|
|
||||||
!strings.Contains(s, "generation 44") {
|
|
||||||
t.Errorf("a send refused for its generation is not said so:\n%s", s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStatusJSONCarriesEachMachinesLastSend(t *testing.T) {
|
|
||||||
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.UTC)
|
|
||||||
asked := answers{nodes: []inventory.Node{{Name: "anchor"}, {Name: "laptop"}},
|
|
||||||
lastSent: map[string]inventory.Send{"laptop": aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeRefused,
|
|
||||||
Refused: "unknown field", At: &at})}}
|
|
||||||
body, err := statusAsJSON(asked)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var got struct {
|
|
||||||
LastSent []map[string]any `json:"lastSent"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(body, &got); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(got.LastSent) != 1 {
|
|
||||||
t.Fatalf("status --json carries %d last sends, want laptop's only:\n%s", len(got.LastSent), body)
|
|
||||||
}
|
|
||||||
s := got.LastSent[0]
|
|
||||||
if s["node"] != "laptop" || s["sequence"] != float64(12) || s["epoch"] != float64(57) ||
|
|
||||||
s["generation"] != float64(40) || s["sender"] != "a one-shot push by jochen at a shell on anchor" ||
|
|
||||||
s["digest"] != "d12" || s["sentAt"] == nil || s["answer"] != "refused" || s["refused"] != "unknown field" ||
|
|
||||||
s["answeredAt"] == nil {
|
|
||||||
t.Fatalf("laptop's last send reads %v", s)
|
|
||||||
}
|
|
||||||
if modules, _ := s["modules"].([]any); len(modules) != 2 || modules[0] != "docker" {
|
|
||||||
t.Fatalf("the modules it named read %v", s["modules"])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// And through the store: a send recorded and refused by the machine is what `node show` and `status --json`
|
|
||||||
// read back.
|
|
||||||
func TestNodeShowAndStatusReadTheLastSendFromTheStore(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
record, err := open.inventory.NodeByName(ctx, "laptop")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := open.inventory.RecordSentWith(ctx, record.ID, "d12", nil, 57, 40, inventory.Send{Sequence: 12,
|
|
||||||
Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12",
|
|
||||||
Modules: []string{"docker", "sudo"}}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeRefused,
|
|
||||||
Declared: "d12", Refused: "unknown field \"generation\""}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
shown := printed(t, func() error { return showNode(ctx, open, "laptop") })
|
|
||||||
for _, want := range []string{"a one-shot push by jochen at a shell on anchor", "lease epoch 57", "generation 40",
|
|
||||||
"docker, sudo", "unknown field \"generation\""} {
|
|
||||||
if !strings.Contains(shown, want) {
|
|
||||||
t.Errorf("node show laptop does not say %q:\n%s", want, shown)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
body := printed(t, func() error { return statusFor(ctx, open, true) })
|
|
||||||
var got struct {
|
|
||||||
LastSent []struct {
|
|
||||||
Node string `json:"node"`
|
|
||||||
Sender string `json:"sender"`
|
|
||||||
Answer string `json:"answer"`
|
|
||||||
} `json:"lastSent"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal([]byte(body), &got); err != nil {
|
|
||||||
t.Fatalf("%v:\n%s", err, body)
|
|
||||||
}
|
|
||||||
if len(got.LastSent) != 1 || got.LastSent[0].Node != "laptop" || got.LastSent[0].Answer != "refused" ||
|
|
||||||
got.LastSent[0].Sender != "a one-shot push by jochen at a shell on anchor" {
|
|
||||||
t.Fatalf("status --json's last sends read %+v", got.LastSent)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,222 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A module assigned to a machine and left out of its declaration is said (novox/hq issue 380).
|
|
||||||
//
|
|
||||||
// **An omission is a finding, never a refusal of the push** (ADR 0163, rule 6): the machine is sent everything
|
|
||||||
// else, and the module's held things are kept. Until issue 380 the only place it showed was `plan`: nfs-server was
|
|
||||||
// assigned to the home server for days, every send left it out for a setting nobody gave, and the operator believed
|
|
||||||
// it ran. So the self-check (D1) judges every machine as the next send would (Resolution.LeftOutBecause, the send's
|
|
||||||
// own judgement) and raises a condition on that machine for each module it leaves out:
|
|
||||||
//
|
|
||||||
// - a setting nobody gave (catalogue.UnsetSettingError) is the operator's to give: kind needs-operator, naming
|
|
||||||
// the module, the setting and the command that sets it;
|
|
||||||
// - any other cause is said as a module not working is: kind left-out, a warning with the reason as evidence.
|
|
||||||
//
|
|
||||||
// Never urgent and never escalated by age (as ADR 0283 decision 5): nothing the machine ran was undone. It clears
|
|
||||||
// on the first run that no longer finds it — the module composed again, or no longer assigned. `status` and
|
|
||||||
// `node show` list the same modules under "assigned, not applied" with the same reason.
|
|
||||||
//
|
|
||||||
// A module left out because its stored manifest has a key this controller does not know is not raised here: the
|
|
||||||
// catalogue's own unknown-field condition says it once for the whole mesh (ADR 0262); it is still listed.
|
|
||||||
|
|
||||||
const (
|
|
||||||
// probeLeftOutID is the self-check's probe that raises and clears these conditions: D1, which already
|
|
||||||
// resolves every machine (probeDeclarations), so the judgement costs no resolution of its own.
|
|
||||||
probeLeftOutID = "D1"
|
|
||||||
// kindLeftOut is a module left out for any cause but a setting nobody gave; it is also every such condition's
|
|
||||||
// token, whichever its kind, so a cause that changes is the same condition said anew.
|
|
||||||
kindLeftOut = "left-out"
|
|
||||||
)
|
|
||||||
|
|
||||||
// leftOutModule is one module of a machine's set that its declaration leaves out, and why.
|
|
||||||
type leftOutModule struct {
|
|
||||||
Module string
|
|
||||||
// Setting is the setting nobody gave, when that is the cause.
|
|
||||||
Setting string
|
|
||||||
// Why is the declaration's own reason, whole.
|
|
||||||
Why string
|
|
||||||
// Unread is a stored manifest this controller cannot read whole (said by the catalogue's condition).
|
|
||||||
Unread bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// leftOutOf is every module of a machine's resolution that a push would leave out, sorted. Pure: the judgement
|
|
||||||
// the push makes (catalogue.Resolution.LeftOutBecause), nothing allocated.
|
|
||||||
func leftOutOf(plan catalogue.Resolution, settings catalogue.SettingsBy, adopted bool) []leftOutModule {
|
|
||||||
because := plan.LeftOutBecause(settings, adopted)
|
|
||||||
out := make([]leftOutModule, 0, len(because))
|
|
||||||
for module, why := range because {
|
|
||||||
l := leftOutModule{Module: module, Why: oneLine(why.Error())}
|
|
||||||
var unset *catalogue.UnsetSettingError
|
|
||||||
var unread *catalogue.UnreadManifestError
|
|
||||||
switch {
|
|
||||||
case errors.As(why, &unset):
|
|
||||||
l.Setting = unset.Setting
|
|
||||||
case errors.As(why, &unread):
|
|
||||||
l.Unread = true
|
|
||||||
}
|
|
||||||
out = append(out, l)
|
|
||||||
}
|
|
||||||
sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module })
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// settingCommand is the command that gives a module's setting on one machine.
|
|
||||||
func settingCommand(module, setting, node string) string {
|
|
||||||
return fmt.Sprintf("`settings set %s '{%q: …}' --node %s`", module, setting, node)
|
|
||||||
}
|
|
||||||
|
|
||||||
// reason is why a module is left out, as `status`, `node show` and the condition's summary say it: for a setting
|
|
||||||
// nobody gave, the setting and the command that gives it; otherwise the declaration's own words.
|
|
||||||
func (l leftOutModule) reason(node string) string {
|
|
||||||
if l.Setting != "" {
|
|
||||||
return fmt.Sprintf("nothing sets its setting %q, so every send leaves it out of its declaration — %s sets it", l.Setting,
|
|
||||||
settingCommand(l.Module, l.Setting, node))
|
|
||||||
}
|
|
||||||
return "every send leaves it out of its declaration: " + l.Why
|
|
||||||
}
|
|
||||||
|
|
||||||
// leftOutObservations are the conditions a machine's left-out modules raise, one each.
|
|
||||||
func leftOutObservations(node string, left []leftOutModule) []conditions.Observation {
|
|
||||||
var out []conditions.Observation
|
|
||||||
for _, l := range left {
|
|
||||||
if l.Unread {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
out = append(out, leftOutObservation(node, l))
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// leftOutObservation is one module left out of one machine's declaration: needs-operator for a setting nobody
|
|
||||||
// gave, left-out otherwise; a warning either way, the operator's to resolve.
|
|
||||||
func leftOutObservation(node string, l leftOutModule) conditions.Observation {
|
|
||||||
o := conditions.Observation{Scope: conditions.ScopeModule, ID: l.Module + "." + node, Token: kindLeftOut,
|
|
||||||
Kind: kindLeftOut, Machine: node, Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
|
||||||
Summary: fmt.Sprintf("%s is assigned to %s and not applied: %s", l.Module, node, l.reason(node)),
|
|
||||||
Said: l.Why}
|
|
||||||
w := leftOutWords(l.Module, node, l.Setting)
|
|
||||||
if l.Setting != "" {
|
|
||||||
o.Kind = kindNeedsOperator
|
|
||||||
} else {
|
|
||||||
// The words of why may name a path or an address, which the operator's channel withholds: the
|
|
||||||
// summary sends them to the evidence, and `plan` says them in full.
|
|
||||||
o.Summary = fmt.Sprintf("%s is assigned to %s and not applied: every send leaves it out of its declaration, "+
|
|
||||||
"because what is set for it does not fit its manifest — the evidence and `plan %s` say why", l.Module, node, node)
|
|
||||||
}
|
|
||||||
o.Headline, o.Explanation, o.Needs, o.Resolved = w.Headline, w.Explanation, w.Needs, w.Resolved
|
|
||||||
return o
|
|
||||||
}
|
|
||||||
|
|
||||||
// leftOutWords is what the operator reads of a module left out (ADR 0253): plain, the act named.
|
|
||||||
func leftOutWords(module, node, setting string) words {
|
|
||||||
w := words{
|
|
||||||
Headline: fmt.Sprintf("%s is not applied on %s", module, node),
|
|
||||||
Explanation: fmt.Sprintf("%s is assigned to %s, and every send to %s leaves it out of the declaration because what "+
|
|
||||||
"is set for it does not fit its manifest. Nothing of it changes there; the rest of %s is sent as usual.",
|
|
||||||
module, node, node, node),
|
|
||||||
Needs: fmt.Sprintf("read why in the details, then change what is set for %s or unassign it.", module),
|
|
||||||
Resolved: fmt.Sprintf("%s on %s is no longer left out", module, node),
|
|
||||||
}
|
|
||||||
if setting != "" {
|
|
||||||
w.Explanation = fmt.Sprintf("%s is assigned to %s, and every send to %s leaves it out of the declaration because "+
|
|
||||||
"nothing sets its setting %s. Nothing of it runs there until it is set; the rest of %s is sent as usual.",
|
|
||||||
module, node, node, setting, node)
|
|
||||||
w.Needs = fmt.Sprintf("set %s for %s on %s, or approve it when it is proposed to you.", setting, module, node)
|
|
||||||
// A setting's name that is not plain (a dotted key) is in the summary instead.
|
|
||||||
if _, ok := conditions.PlainWords(w, node); !ok {
|
|
||||||
w.Explanation = fmt.Sprintf("%s is assigned to %s, and every send to %s leaves it out of the declaration because a "+
|
|
||||||
"setting it needs is not set. Nothing of it runs there until it is set; the details name it.",
|
|
||||||
module, node, node)
|
|
||||||
w.Needs = fmt.Sprintf("set what %s needs on %s; the details name the setting.", module, node)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return w
|
|
||||||
}
|
|
||||||
|
|
||||||
// notAppliedLines is a machine's "assigned, not applied" as `node show` prints it: one line a module, with the
|
|
||||||
// reason its condition says.
|
|
||||||
func notAppliedLines(node string, left []leftOutModule) []string {
|
|
||||||
if len(left) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
lines := []string{"", " assigned, not applied:"}
|
|
||||||
for _, l := range left {
|
|
||||||
lines = append(lines, fmt.Sprintf(" %-22s %s", l.Module, l.reason(node)))
|
|
||||||
}
|
|
||||||
return lines
|
|
||||||
}
|
|
||||||
|
|
||||||
// machineNotApplied is one module assigned to a machine and left out of its declaration, in `status --json`.
|
|
||||||
type machineNotApplied struct {
|
|
||||||
Node string `json:"node"`
|
|
||||||
Module string `json:"module"`
|
|
||||||
Setting string `json:"setting,omitempty"`
|
|
||||||
Reason string `json:"reason"`
|
|
||||||
// UnreadManifest is a module left out because this controller cannot read its manifest whole: it raises no
|
|
||||||
// condition of its own on the machine, since the catalogue's unknown-field condition says it once (ADR 0262).
|
|
||||||
UnreadManifest bool `json:"unread-manifest,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// notApplied is every machine's left-out modules, in a stated order, as `status --json` carries them.
|
|
||||||
func notApplied(left map[string][]leftOutModule) []machineNotApplied {
|
|
||||||
names := make([]string, 0, len(left))
|
|
||||||
for name := range left {
|
|
||||||
names = append(names, name)
|
|
||||||
}
|
|
||||||
sort.Strings(names)
|
|
||||||
var out []machineNotApplied
|
|
||||||
for _, name := range names {
|
|
||||||
for _, l := range left[name] {
|
|
||||||
out = append(out, machineNotApplied{Node: name, Module: l.Module, Setting: l.Setting, Reason: l.reason(name),
|
|
||||||
UnreadManifest: l.Unread})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// printNotApplied is status's "assigned, not applied", every machine's.
|
|
||||||
func printNotApplied(left map[string][]leftOutModule) {
|
|
||||||
rows := notApplied(left)
|
|
||||||
if len(rows) == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
unread := 0
|
|
||||||
for _, r := range rows {
|
|
||||||
if r.UnreadManifest {
|
|
||||||
unread++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Which raise a condition is said, not implied (review of #223): one whose manifest this controller cannot
|
|
||||||
// read is listed here and raises none of its own on the machine — the catalogue's condition says it.
|
|
||||||
raises := "each raises a condition on its machine"
|
|
||||||
switch {
|
|
||||||
case unread == len(rows):
|
|
||||||
raises = "none raises a condition on its machine: this controller cannot read their manifests, which the " +
|
|
||||||
"catalogue's own condition says"
|
|
||||||
case unread > 0:
|
|
||||||
raises += fmt.Sprintf(", except the %d whose manifest this controller cannot read, which the catalogue's own "+
|
|
||||||
"condition says", unread)
|
|
||||||
}
|
|
||||||
fmt.Printf("%d module(s) assigned, not applied — every send leaves them out of their declaration; %s:\n",
|
|
||||||
len(rows), raises)
|
|
||||||
for _, r := range rows {
|
|
||||||
fmt.Printf(" %-12s %-22s %s\n", r.Node, r.Module, r.Reason)
|
|
||||||
}
|
|
||||||
fmt.Println()
|
|
||||||
}
|
|
||||||
|
|
||||||
// isLeftOutCondition is whether a condition is a module left out of its declaration, which the machine's health
|
|
||||||
// statements neither raise nor clear: by its token, which every one carries whatever its kind.
|
|
||||||
func isLeftOutCondition(c conditions.Condition) bool {
|
|
||||||
return c.Subject.Scope == conditions.ScopeModule && strings.HasSuffix(c.Key, "."+kindLeftOut)
|
|
||||||
}
|
|
||||||
@@ -1,259 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"os"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// novox/hq issue 380: nfs-server was assigned to the home server for days and every send left it out — "nfs-server
|
|
||||||
// has a file that says ${setting:shares}, and nothing sets shares for it" — and nothing but `plan` said so. The
|
|
||||||
// manifest is the catalogue's own at the commit that added it (mesh-catalog 48fba44), which still says
|
|
||||||
// ${setting:shares}; the reason below is the one the live push printed.
|
|
||||||
|
|
||||||
// leftOutNFS is the resolution of a machine assigned that nfs-server, with the settings given.
|
|
||||||
func leftOutNFS(t *testing.T) catalogue.Resolution {
|
|
||||||
t.Helper()
|
|
||||||
raw, err := os.ReadFile("testdata/left-out/nfs-server.json")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
m, err := catalogue.ParseManifest(raw)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return catalogue.Resolution{Modules: []catalogue.Manifest{m}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sharesGiven is the operator's setting for it on the machine.
|
|
||||||
func sharesGiven(value string) catalogue.SettingsBy {
|
|
||||||
return catalogue.SettingsBy{"nfs-server": {{From: "anchor", Values: map[string]any{"shares": value}}}}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAModuleLeftOutForASettingNobodyGaveNeedsTheOperatorNamingTheSettingAndTheCommand(t *testing.T) {
|
|
||||||
left := leftOutOf(leftOutNFS(t), nil, false)
|
|
||||||
if len(left) != 1 || left[0].Module != "nfs-server" || left[0].Setting != "shares" {
|
|
||||||
t.Fatalf("left out: %+v", left)
|
|
||||||
}
|
|
||||||
if !strings.Contains(left[0].Why, `nothing sets "shares" for it`) {
|
|
||||||
t.Fatalf("the reason is not the push's own: %s", left[0].Why)
|
|
||||||
}
|
|
||||||
obs := leftOutObservations("anchor", left)
|
|
||||||
if len(obs) != 1 {
|
|
||||||
t.Fatalf("raised %+v", obs)
|
|
||||||
}
|
|
||||||
o := obs[0]
|
|
||||||
if o.Key() != "module.nfs-server.anchor.left-out" || o.Kind != kindNeedsOperator || o.Machine != "anchor" ||
|
|
||||||
o.Severity != conditions.Warning || o.Resolver != conditions.ResolverOperator {
|
|
||||||
t.Fatalf("raised %s as %s, %s, by %s, on %q", o.Key(), o.Kind, o.Severity, o.Resolver, o.Machine)
|
|
||||||
}
|
|
||||||
for _, want := range []string{"nfs-server", "anchor", `"shares"`, "`settings set nfs-server '{\"shares\": …}' --node anchor`"} {
|
|
||||||
if !strings.Contains(o.Summary, want) {
|
|
||||||
t.Errorf("the summary does not name %s: %s", want, o.Summary)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if o.Said != left[0].Why {
|
|
||||||
t.Errorf("the evidence is not the reason the send gives: %s", o.Said)
|
|
||||||
}
|
|
||||||
plainExample(t, o, "nfs-server is not applied on anchor",
|
|
||||||
"Needs you: set shares for nfs-server on anchor, or approve it when it is proposed to you. nfs-server is assigned to "+
|
|
||||||
"anchor, and every send to anchor leaves it out of the declaration because nothing sets its setting shares. "+
|
|
||||||
"Nothing of it runs there until it is set; the rest of anchor is sent as usual.")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAModuleLeftOutForAnotherCauseIsAWarningWithTheReasonAsEvidence(t *testing.T) {
|
|
||||||
// A setting stored that its manifest can no longer take: one with a line break (issue 339).
|
|
||||||
left := leftOutOf(leftOutNFS(t), sharesGiven("library=/srv/library\nmedia=/srv/media"), false)
|
|
||||||
if len(left) != 1 || left[0].Setting != "" {
|
|
||||||
t.Fatalf("left out: %+v", left)
|
|
||||||
}
|
|
||||||
obs := leftOutObservations("anchor", left)
|
|
||||||
if len(obs) != 1 {
|
|
||||||
t.Fatalf("raised %+v", obs)
|
|
||||||
}
|
|
||||||
o := obs[0]
|
|
||||||
if o.Key() != "module.nfs-server.anchor.left-out" || o.Kind != kindLeftOut || o.Severity != conditions.Warning {
|
|
||||||
t.Fatalf("raised %s as %s, %s", o.Key(), o.Kind, o.Severity)
|
|
||||||
}
|
|
||||||
if !strings.Contains(o.Said, "holds a line break") || strings.Contains(o.Summary, "/srv/") {
|
|
||||||
t.Fatalf("the reason is not the evidence, or the summary carries it to the channel:\n%s\n%s", o.Summary, o.Said)
|
|
||||||
}
|
|
||||||
plainExample(t, o, "nfs-server is not applied on anchor",
|
|
||||||
"Needs you: read why in the details, then change what is set for nfs-server or unassign it. nfs-server is assigned to "+
|
|
||||||
"anchor, and every send to anchor leaves it out of the declaration because what is set for it does not fit "+
|
|
||||||
"its manifest. Nothing of it changes there; the rest of anchor is sent as usual.")
|
|
||||||
}
|
|
||||||
|
|
||||||
// The self-check raises it, keeps it a warning however long it stands, and clears it when the module composes
|
|
||||||
// again or is no longer assigned; a machine's health statement neither clears nor raises it.
|
|
||||||
func TestALeftOutModulesConditionClearsWhenItComposesAgainOrIsUnassigned(t *testing.T) {
|
|
||||||
plan, settings := leftOutNFS(t), catalogue.SettingsBy(nil)
|
|
||||||
withProbes(t, probe{ID: probeLeftOutID, Asserts: "the test's", Kind: kindLeftOut, Phase: 1,
|
|
||||||
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
|
|
||||||
return leftOutObservations("anchor", leftOutOf(plan, settings, false)), nil
|
|
||||||
}})
|
|
||||||
store := conditions.NewInMemory()
|
|
||||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
|
||||||
defer k.Close(context.Background())
|
|
||||||
d := &doctor{keeper: k, teller: &conditions.Told{}, host: "anchor"}
|
|
||||||
key := "module.nfs-server.anchor.left-out"
|
|
||||||
openOnes := func() map[string]conditions.Condition {
|
|
||||||
t.Helper()
|
|
||||||
open, err := k.Open(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
out := map[string]conditions.Condition{}
|
|
||||||
for _, c := range open {
|
|
||||||
out[c.Key] = c
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
d.runOnce(t.Context(), "a test")
|
|
||||||
c, raised := openOnes()[key]
|
|
||||||
if !raised || c.Kind != kindNeedsOperator || c.Severity != conditions.Warning {
|
|
||||||
t.Fatalf("a module left out raised %+v", openOnes())
|
|
||||||
}
|
|
||||||
// A statement from the machine that says nothing of it — the module runs nothing there — leaves it open.
|
|
||||||
if err := judgeModuleHealth(t.Context(), nil, k, "anchor", map[string][]inventory.ResourceHealth{}, nil,
|
|
||||||
time.Now().Add(72*time.Hour)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, still := openOnes()[key]; !still {
|
|
||||||
t.Fatal("a health statement that says nothing of the module cleared its left-out condition")
|
|
||||||
}
|
|
||||||
d.runOnce(t.Context(), "a test")
|
|
||||||
if c := openOnes()[key]; c.Severity != conditions.Warning {
|
|
||||||
t.Fatalf("after a health statement and days, it is %+v", openOnes())
|
|
||||||
}
|
|
||||||
|
|
||||||
// The setting given: it composes, and the condition clears.
|
|
||||||
settings = sharesGiven("library=/srv/library")
|
|
||||||
d.runOnce(t.Context(), "a test")
|
|
||||||
if _, still := openOnes()[key]; still {
|
|
||||||
t.Fatalf("composed again, still open: %+v", openOnes())
|
|
||||||
}
|
|
||||||
|
|
||||||
// Left out again, then unassigned: no longer in the machine's set, and it clears.
|
|
||||||
settings = nil
|
|
||||||
d.runOnce(t.Context(), "a test")
|
|
||||||
if _, raised := openOnes()[key]; !raised {
|
|
||||||
t.Fatal("left out again and not raised")
|
|
||||||
}
|
|
||||||
plan = catalogue.Resolution{}
|
|
||||||
d.runOnce(t.Context(), "a test")
|
|
||||||
if _, still := openOnes()[key]; still {
|
|
||||||
t.Fatalf("unassigned, still open: %+v", openOnes())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTheLeftOutProbeIsInTheRegistry(t *testing.T) {
|
|
||||||
for _, p := range probeRegistry {
|
|
||||||
if p.ID == probeLeftOutID {
|
|
||||||
if p.run == nil || !slices.Contains(p.Raises, kindLeftOut) || !slices.Contains(p.Raises, kindNeedsOperator) {
|
|
||||||
t.Fatalf("%+v", p)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
t.Fatalf("no probe %s: a module left out is said nowhere", probeLeftOutID)
|
|
||||||
}
|
|
||||||
|
|
||||||
// status and node show list it under "assigned, not applied" with the reason the condition says, and status is
|
|
||||||
// not well while there is one.
|
|
||||||
func TestStatusAndNodeListAModuleAssignedAndNotApplied(t *testing.T) {
|
|
||||||
left := map[string][]leftOutModule{"anchor": leftOutOf(leftOutNFS(t), nil, false)}
|
|
||||||
reason := leftOutObservations("anchor", left["anchor"])[0].Summary
|
|
||||||
asked := answers{leftOut: left}
|
|
||||||
if asked.well() {
|
|
||||||
t.Fatal("a mesh with a module assigned and not applied is called well")
|
|
||||||
}
|
|
||||||
shown := printed(t, func() error { return printStatus(asked) })
|
|
||||||
if !strings.Contains(shown, "1 module(s) assigned, not applied") || !strings.Contains(shown, "nfs-server") ||
|
|
||||||
!strings.Contains(shown, "`settings set nfs-server '{\"shares\": …}' --node anchor` sets it") {
|
|
||||||
t.Fatalf("status says:\n%s", shown)
|
|
||||||
}
|
|
||||||
if !strings.Contains(reason, left["anchor"][0].reason("anchor")) {
|
|
||||||
t.Fatalf("status and the condition say different reasons:\n%s\n%s", shown, reason)
|
|
||||||
}
|
|
||||||
body, err := statusAsJSON(asked)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var doc struct {
|
|
||||||
NotApplied []machineNotApplied `json:"not-applied"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(body, &doc); err != nil || len(doc.NotApplied) != 1 ||
|
|
||||||
doc.NotApplied[0].Setting != "shares" || doc.NotApplied[0].Node != "anchor" {
|
|
||||||
t.Fatalf("status --json: %v %s", err, body)
|
|
||||||
}
|
|
||||||
lines := strings.Join(notAppliedLines("anchor", left["anchor"]), "\n")
|
|
||||||
if !strings.Contains(lines, "assigned, not applied:") || !strings.Contains(lines, "nfs-server") ||
|
|
||||||
!strings.Contains(lines, `nothing sets its setting "shares"`) {
|
|
||||||
t.Fatalf("node show says:\n%s", lines)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The skip is this probe's alone (review of #223): a part waiting for the operator (ADR 0283) keeps its own
|
|
||||||
// needs-operator condition, `module.<m>.<node>.needs-operator`, which still clears on the first statement that no
|
|
||||||
// longer names it — beside a left-out condition on the same machine, which stays.
|
|
||||||
func TestAWaitsNeedsOperatorStillClearsWhenItsStatementStopsNamingItBesideALeftOutOne(t *testing.T) {
|
|
||||||
store := conditions.NewInMemory()
|
|
||||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
|
||||||
defer k.Close(context.Background())
|
|
||||||
left := leftOutObservations("anchor", leftOutOf(leftOutNFS(t), nil, false))
|
|
||||||
if err := k.Reconcile(t.Context(), probeLeftOutID, left); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
waiting := map[string][]inventory.ResourceHealth{"notes": {{Module: "notes", Resource: "server", State: link.StateWaiting,
|
|
||||||
Waits: []inventory.Wait{{Setting: "domain", What: "the domain it serves"}}}}}
|
|
||||||
if err := judgeModuleHealth(t.Context(), nil, k, "anchor", waiting, map[string]int{"notes": 2}, time.Now()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
waitKey, leftKey := needsOperatorKey("notes", "anchor"), "module.nfs-server.anchor.left-out"
|
|
||||||
open := func() map[string]conditions.Condition {
|
|
||||||
t.Helper()
|
|
||||||
list, err := k.Open(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
out := map[string]conditions.Condition{}
|
|
||||||
for _, c := range list {
|
|
||||||
out[c.Key] = c
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
if c, raised := open()[waitKey]; !raised || c.Kind != kindNeedsOperator {
|
|
||||||
t.Fatalf("the wait raised %+v", open())
|
|
||||||
}
|
|
||||||
if c := open()[leftKey]; c.Kind != kindNeedsOperator {
|
|
||||||
t.Fatalf("the left-out condition is not open as needs-operator: %+v", open())
|
|
||||||
}
|
|
||||||
// D1 runs again and still finds nfs-server left out: its reconcile clears only what D1 raised, never the wait,
|
|
||||||
// which the machine's statement raised.
|
|
||||||
if err := k.Reconcile(t.Context(), probeLeftOutID, left); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, still := open()[waitKey]; !still {
|
|
||||||
t.Fatalf("D1's reconcile cleared the wait's needs-operator: %+v", open())
|
|
||||||
}
|
|
||||||
if err := judgeModuleHealth(t.Context(), nil, k, "anchor", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, still := open()[waitKey]; still {
|
|
||||||
t.Fatalf("the statement no longer names the wait, and its needs-operator is still open: %+v", open())
|
|
||||||
}
|
|
||||||
if _, still := open()[leftKey]; !still {
|
|
||||||
t.Fatalf("the left-out condition was cleared by a health statement: %+v", open())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -311,7 +311,7 @@ func usage() {
|
|||||||
the self-check: the last verdict, a run now, the probes, the signals' ages
|
the self-check: the last verdict, a run now, the probes, the signals' ages
|
||||||
healers [--days N] [--json] the healers, what they did lately, and their brake (to-be 45 §7)
|
healers [--days N] [--json] the healers, what they did lately, and their brake (to-be 45 §7)
|
||||||
durations [--kind K] [--days N] [--json]
|
durations [--kind K] [--days N] [--json]
|
||||||
apply, heartbeat, plan-tier, build and walk-phase durations
|
apply, heartbeat, plan-tier and build durations, per machine or module
|
||||||
collection [--json] kept archives held/unheld by a manifest, and what the sweep may let go
|
collection [--json] kept archives held/unheld by a manifest, and what the sweep may let go
|
||||||
builder issue <name> a broker account for a build machine, scoped to build work,
|
builder issue <name> a broker account for a build machine, scoped to build work,
|
||||||
delivered as the builder module's broker secret (module add it first)
|
delivered as the builder module's broker secret (module add it first)
|
||||||
|
|||||||
@@ -392,8 +392,7 @@ func TestACheckByHandClonesWhatTheSeatClones(t *testing.T) {
|
|||||||
for dir, refs := range map[string]map[string]string{
|
for dir, refs := range map[string]map[string]string{
|
||||||
"mesh-catalog": {"mesh-catalog": "main"},
|
"mesh-catalog": {"mesh-catalog": "main"},
|
||||||
"mesh-host": {"mesh-host": "c0ffee"},
|
"mesh-host": {"mesh-host": "c0ffee"},
|
||||||
"mesh-controller": {"mesh-controller": "c0ffee", "mesh-controller-main": "main", "mesh-lab": "main",
|
"mesh-controller": {"mesh-controller": "c0ffee", "mesh-controller-main": "main", "mesh-lab": "main"},
|
||||||
"mesh-sdk": sdkPinnedByGoMod(t)},
|
|
||||||
} {
|
} {
|
||||||
got := besideRefs(dir, "c0ffee")
|
got := besideRefs(dir, "c0ffee")
|
||||||
for d, ref := range refs {
|
for d, ref := range refs {
|
||||||
@@ -413,24 +412,3 @@ func TestACheckByHandClonesWhatTheSeatClones(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// sdkPinnedByGoMod is the SDK commit this tree's go.mod pins, read from the file rather than the build, so
|
|
||||||
// sdkPinned is held to what the repository says (novox/hq issue 449).
|
|
||||||
func sdkPinnedByGoMod(t *testing.T) string {
|
|
||||||
t.Helper()
|
|
||||||
raw, err := os.ReadFile(filepath.Join("..", "..", "go.mod"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, line := range strings.Split(string(raw), "\n") {
|
|
||||||
fields := strings.Fields(line)
|
|
||||||
if len(fields) >= 2 && fields[0] == sdkModule {
|
|
||||||
if m := pseudoCommit.FindStringSubmatch(fields[1]); m != nil {
|
|
||||||
return m[1]
|
|
||||||
}
|
|
||||||
return "go/" + fields[1]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
t.Fatalf("go.mod requires no %s", sdkModule)
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -74,21 +74,9 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke
|
|||||||
kept := inventory.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target,
|
kept := inventory.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target,
|
||||||
State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts,
|
State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts,
|
||||||
Check: r.Check, Needs: r.Needs, Account: r.Account, Root: r.Root}
|
Check: r.Check, Needs: r.Needs, Account: r.Account, Root: r.Root}
|
||||||
for _, w := range r.Waits {
|
|
||||||
kept.Waits = append(kept.Waits, inventory.Wait{Part: w.Part, Secret: w.Secret, Setting: w.Setting, What: w.What})
|
|
||||||
}
|
|
||||||
resources = append(resources, kept)
|
resources = append(resources, kept)
|
||||||
}
|
if r.State == link.StateUnhealthy && r.Module != "" {
|
||||||
// **A wait for the operator is checked before it is excused** (novox/hq ADR 0283): a waiting resource whose
|
unhealthy[r.Module] = append(unhealthy[r.Module], kept)
|
||||||
// wait does not check out is judged unhealthy, saying why; one that does is kept beside the unhealthy ones, so
|
|
||||||
// judgeModuleHealth can say it as needs-operator. What is stored is what the machine said.
|
|
||||||
var wf operatorWaitFacts
|
|
||||||
if mods := waitingModules(resources); len(mods) > 0 {
|
|
||||||
readWaitFacts(ctx, inv, node, mods, nil, &wf)
|
|
||||||
}
|
|
||||||
for _, r := range checkWaiting(node, resources, wf) {
|
|
||||||
if (r.State == link.StateUnhealthy || r.State == link.StateWaiting) && r.Module != "" {
|
|
||||||
unhealthy[r.Module] = append(unhealthy[r.Module], r)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
streaks := map[string]int{}
|
streaks := map[string]int{}
|
||||||
@@ -147,18 +135,17 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
|||||||
}
|
}
|
||||||
standing := map[string]conditions.Condition{}
|
standing := map[string]conditions.Condition{}
|
||||||
for _, c := range open {
|
for _, c := range open {
|
||||||
// A module left out of the composition is the self-check's to raise and clear, never a statement's
|
if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound) &&
|
||||||
// (novox/hq issue 380): its needs-operator is not cleared for not being in what the machine runs.
|
c.Subject.Machine == node {
|
||||||
if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound ||
|
|
||||||
c.Kind == kindNeedsOperator) &&
|
|
||||||
c.Subject.Machine == node && !isLeftOutCondition(c) {
|
|
||||||
standing[c.Key] = c
|
standing[c.Key] = c
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
hold, err := readHoldingFor(ctx, inv, open)
|
var hold *holding
|
||||||
if err != nil {
|
if inv != nil {
|
||||||
|
if hold, err = readHolding(ctx, inv, open); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
}
|
||||||
var problems []string
|
var problems []string
|
||||||
modules := make([]string, 0, len(unhealthy))
|
modules := make([]string, 0, len(unhealthy))
|
||||||
for m := range unhealthy {
|
for m := range unhealthy {
|
||||||
@@ -172,39 +159,6 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
|||||||
heldOn := map[string]string{}
|
heldOn := map[string]string{}
|
||||||
providers := map[catalogue.Chosen]bool{}
|
providers := map[catalogue.Chosen]bool{}
|
||||||
for _, m := range modules {
|
for _, m := range modules {
|
||||||
// **A part that waits for the operator is said as that** (novox/hq ADR 0283): its waits already checked,
|
|
||||||
// the operator's, never urgent, its words naming the act.
|
|
||||||
if waits, waiting := operatorWait(m, unhealthy[m]); waiting {
|
|
||||||
o := needsOperatorObservation(m, node, waits, unhealthy[m])
|
|
||||||
// **A provider waiting for the operator says who waits on it** (novox/hq issue 405), as one waiting for a
|
|
||||||
// login does: its consumers are held under it.
|
|
||||||
if hold != nil {
|
|
||||||
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
|
|
||||||
}
|
|
||||||
seen[o.Key()] = true
|
|
||||||
became[m] = kindNeedsOperator
|
|
||||||
if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if _, err := k.Observe(ctx, o); err != nil {
|
|
||||||
problems = append(problems, err.Error())
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
// **A wait for the operator beside anything else is said too** (novox/hq issue 405): a module with a checked
|
|
||||||
// wait beside a new login owed, a directory used as found or a fault of its own is said as that, and the
|
|
||||||
// operator's secret or setting it waits for was not mentioned until the other cleared. Its needs-operator
|
|
||||||
// condition stands beside the other, on the same looks; what follows judges the rest without the wait.
|
|
||||||
if waits, rest := besideAWait(m, unhealthy[m]); len(waits) > 0 {
|
|
||||||
o := needsOperatorObservation(m, node, waits, waitingOf(m, unhealthy[m]))
|
|
||||||
seen[o.Key()] = true
|
|
||||||
if _, isOpen := standing[o.Key()]; streaks[m] >= moduleUnhealthyAfter || isOpen {
|
|
||||||
if _, err := k.Observe(ctx, o); err != nil {
|
|
||||||
problems = append(problems, err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
unhealthy[m] = rest
|
|
||||||
}
|
|
||||||
// **A directory used as found is said as that** (novox/hq issue 339): the operator's to hand over at the
|
// **A directory used as found is said as that** (novox/hq issue 339): the operator's to hand over at the
|
||||||
// machine, never urgent — nothing is broken by the wait that a person was not told of — and its own kind,
|
// machine, never urgent — nothing is broken by the wait that a person was not told of — and its own kind,
|
||||||
// so the gate never reads it as a fault of the build that happened to be sent beside it.
|
// so the gate never reads it as a fault of the build that happened to be sent beside it.
|
||||||
@@ -241,24 +195,13 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
|||||||
}
|
}
|
||||||
o := moduleUnhealthyObservation(m, node, unhealthy[m])
|
o := moduleUnhealthyObservation(m, node, unhealthy[m])
|
||||||
if hold != nil {
|
if hold != nil {
|
||||||
if by, held := hold.heldWith(node, m, unhealthy[m]); held {
|
if p, held := hold.heldUnder(node, m, unhealthy[m]); held {
|
||||||
// Held under the provider's condition: nothing of its own, and the provider's says it waits. The
|
// Held under the provider's condition: nothing of its own, and the provider's says it waits.
|
||||||
// clearing line says which the provider is: unhealthy, or waiting for the operator (issue 405).
|
heldOn[o.Key()] = p.Module + " on " + p.Node
|
||||||
p := by.provider
|
|
||||||
heldOn[o.Key()] = p.Module + " on " + p.Node + ", which is unhealthy"
|
|
||||||
if len(by.waits) > 0 {
|
|
||||||
heldOn[o.Key()] = p.Module + " on " + p.Node + ", which waits for you"
|
|
||||||
}
|
|
||||||
providers[p] = true
|
providers[p] = true
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
|
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
|
||||||
// A provider that waits for the operator for a part this finding cannot be matched to does not hold it
|
|
||||||
// (novox/hq issue 405): raised as its own, and saying the provider waits, so neither is hidden.
|
|
||||||
if p, waiting := hold.waitingUncovered(node, m, unhealthy[m]); waiting {
|
|
||||||
o.Said += fmt.Sprintf("; %s on %s waits for you, but not for anything %s is known to need, so %s's "+
|
|
||||||
"fault is said on its own", p.Module, p.Node, m, m)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
seen[o.Key()] = true
|
seen[o.Key()] = true
|
||||||
became[m] = kindModuleUnhealthy
|
became[m] = kindModuleUnhealthy
|
||||||
@@ -285,11 +228,8 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
|||||||
if c.Kind == kindUsedAsFound {
|
if c.Kind == kindUsedAsFound {
|
||||||
why = fmt.Sprintf("%s says no directory of %s is used as found any more", node, module)
|
why = fmt.Sprintf("%s says no directory of %s is used as found any more", node, module)
|
||||||
}
|
}
|
||||||
if c.Kind == kindNeedsOperator {
|
|
||||||
why = fmt.Sprintf("%s says %s no longer waits for the operator", node, module)
|
|
||||||
}
|
|
||||||
if on, held := heldOn[key]; held {
|
if on, held := heldOn[key]; held {
|
||||||
why = fmt.Sprintf("what %s finds on %s waits on %s: held under its condition", module, node, on)
|
why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on)
|
||||||
}
|
}
|
||||||
// **A condition that became the other kind** is not "working again" (issue 318 review): its clearing
|
// **A condition that became the other kind** is not "working again" (issue 318 review): its clearing
|
||||||
// line says what it became.
|
// line says what it became.
|
||||||
@@ -298,12 +238,6 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
|||||||
case c.Kind == kindModuleUnhealthy && became[module] == kindReloginNeeded:
|
case c.Kind == kindModuleUnhealthy && became[module] == kindReloginNeeded:
|
||||||
why = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
|
why = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
|
||||||
resolved = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
|
resolved = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
|
||||||
case c.Kind == kindModuleUnhealthy && became[module] == kindNeedsOperator:
|
|
||||||
why = fmt.Sprintf("%s on %s now only waits for the operator", module, node)
|
|
||||||
resolved = fmt.Sprintf("%s on %s now only waits for you", module, node)
|
|
||||||
case c.Kind == kindNeedsOperator && became[module] == kindModuleUnhealthy:
|
|
||||||
why = fmt.Sprintf("%s on %s no longer only waits for the operator, and is not healthy", module, node)
|
|
||||||
resolved = fmt.Sprintf("What %s on %s waited for is given, and it still does not work", module, node)
|
|
||||||
case c.Kind == kindReloginNeeded && became[module] == kindModuleUnhealthy:
|
case c.Kind == kindReloginNeeded && became[module] == kindModuleUnhealthy:
|
||||||
why = fmt.Sprintf("%s on %s no longer waits for a new login, and is not healthy", module, node)
|
why = fmt.Sprintf("%s on %s no longer waits for a new login, and is not healthy", module, node)
|
||||||
resolved = fmt.Sprintf("The new login on %s is done, and %s still does not work", node, module)
|
resolved = fmt.Sprintf("The new login on %s is done, and %s still does not work", node, module)
|
||||||
@@ -328,74 +262,36 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
|||||||
// sayWaiters observes a provider's open condition again, with who waits on it, from its machine's newest
|
// sayWaiters observes a provider's open condition again, with who waits on it, from its machine's newest
|
||||||
// statement. Nothing when its condition is not open: it is raised by its own statements, on its own looks.
|
// statement. Nothing when its condition is not open: it is raised by its own statements, on its own looks.
|
||||||
func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p catalogue.Chosen, now time.Time) error {
|
func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p catalogue.Chosen, now time.Time) error {
|
||||||
// Its statement as it was judged, its waits checked (novox/hq issue 450): a wait that failed the check is
|
var raisedAt *conditions.Condition
|
||||||
// unhealthy, so the condition built here is the one its own statement raised, and who waits on it is listed.
|
for i, c := range hold.open {
|
||||||
|
if c.Key == moduleUnhealthyKey(p.Module, p.Node) || c.Key == reloginKey(p.Module, p.Node) {
|
||||||
|
raisedAt = &hold.open[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if raisedAt == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
var rs []inventory.ResourceHealth
|
var rs []inventory.ResourceHealth
|
||||||
for _, r := range hold.checked(p.Node) {
|
for _, r := range hold.healths[p.Node].Resources {
|
||||||
if r.Module == p.Module && (r.State == link.StateUnhealthy || r.State == link.StateWaiting) {
|
if r.Module == p.Module && r.State == link.StateUnhealthy {
|
||||||
rs = append(rs, r)
|
rs = append(rs, r)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(rs) == 0 {
|
if len(rs) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
// The condition its own statement says it under: needs-operator while it only waits for the operator (novox/hq
|
o := moduleUnhealthyObservation(p.Module, p.Node, rs)
|
||||||
// issue 405), else that for the rest of it, a wait beside it said on its own.
|
if said, waits := personWait(p.Module, p.Node, rs); waits {
|
||||||
var o conditions.Observation
|
o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rs)
|
||||||
if waits, waiting := operatorWait(p.Module, rs); waiting {
|
|
||||||
o = needsOperatorObservation(p.Module, p.Node, waits, rs)
|
|
||||||
} else {
|
|
||||||
_, rest := besideAWait(p.Module, rs)
|
|
||||||
o = moduleUnhealthyObservation(p.Module, p.Node, rest)
|
|
||||||
if said, waits := personWait(p.Module, p.Node, rest); waits {
|
|
||||||
o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rest)
|
|
||||||
}
|
}
|
||||||
}
|
if o.Key() != raisedAt.Key {
|
||||||
raised := false
|
return nil // its own statement says it next
|
||||||
for _, c := range hold.open {
|
|
||||||
raised = raised || c.Key == o.Key()
|
|
||||||
}
|
|
||||||
if !raised {
|
|
||||||
return nil // not open yet, or open as another kind: its own statement says it next
|
|
||||||
}
|
}
|
||||||
sayWaitingOn(&o, hold.waitersOn(p))
|
sayWaitingOn(&o, hold.waitersOn(p))
|
||||||
_, err := k.Observe(ctx, o)
|
_, err := k.Observe(ctx, o)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// besideAWait is, for a module with something not healthy beside a part waiting for the operator, the waits (checked
|
|
||||||
// already) and the rest without the waiting parts (novox/hq issue 405); no waits when nothing waits, or when the
|
|
||||||
// module only waits (operatorWait says that whole). Pure.
|
|
||||||
func besideAWait(module string, rs []inventory.ResourceHealth) ([]inventory.Wait, []inventory.ResourceHealth) {
|
|
||||||
if _, only := operatorWait(module, rs); only {
|
|
||||||
return nil, rs
|
|
||||||
}
|
|
||||||
var waits []inventory.Wait
|
|
||||||
var rest []inventory.ResourceHealth
|
|
||||||
for _, r := range rs {
|
|
||||||
if r.Module == module && r.State == link.StateWaiting {
|
|
||||||
waits = append(waits, r.Waits...)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
rest = append(rest, r)
|
|
||||||
}
|
|
||||||
if len(waits) == 0 {
|
|
||||||
return nil, rs
|
|
||||||
}
|
|
||||||
return waits, rest
|
|
||||||
}
|
|
||||||
|
|
||||||
// waitingOf is a module's waiting resources: the evidence of its wait.
|
|
||||||
func waitingOf(module string, rs []inventory.ResourceHealth) []inventory.ResourceHealth {
|
|
||||||
var out []inventory.ResourceHealth
|
|
||||||
for _, r := range rs {
|
|
||||||
if r.Module == module && r.State == link.StateWaiting {
|
|
||||||
out = append(out, r)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// reloginKey is a module's relogin-needed condition on a machine.
|
// reloginKey is a module's relogin-needed condition on a machine.
|
||||||
func reloginKey(module, node string) string {
|
func reloginKey(module, node string) string {
|
||||||
return conditions.Key(conditions.ScopeModule, module+"."+node, kindReloginNeeded)
|
return conditions.Key(conditions.ScopeModule, module+"."+node, kindReloginNeeded)
|
||||||
@@ -467,15 +363,12 @@ func waitingAccounts(module string, rs []inventory.ResourceHealth) []string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// sayWaitingOn adds to a module's condition the consumers held under it (to-be 48 §6): urgent while anyone
|
// sayWaitingOn adds to a module's condition the consumers held under it (to-be 48 §6): urgent while anyone
|
||||||
// waits on it, whether it is not working or waits for a new login. **A wait for the operator's secret or setting
|
// waits on it, whether it is not working or waits for a new login.
|
||||||
// stays a warning** (ADR 0283 decision 5, novox/hq issue 405): who waits on it is listed, and nothing escalates it.
|
|
||||||
func sayWaitingOn(o *conditions.Observation, waiters []string) {
|
func sayWaitingOn(o *conditions.Observation, waiters []string) {
|
||||||
if len(waiters) == 0 {
|
if len(waiters) == 0 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if o.Kind != kindNeedsOperator {
|
|
||||||
o.Severity = conditions.Urgent
|
o.Severity = conditions.Urgent
|
||||||
}
|
|
||||||
o.Said += "; " + waitingWords(waiters)
|
o.Said += "; " + waitingWords(waiters)
|
||||||
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
|
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
|
||||||
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
|
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
|
||||||
@@ -527,11 +420,6 @@ func reasonWords(r inventory.ResourceHealth) string {
|
|||||||
case "":
|
case "":
|
||||||
return "is unhealthy"
|
return "is unhealthy"
|
||||||
}
|
}
|
||||||
// A wait for the operator that did not check out is said as the controller found it (ADR 0283): names of
|
|
||||||
// secrets and settings only, never what the check itself said.
|
|
||||||
if strings.HasPrefix(r.Reason, waitRefusedPrefix) {
|
|
||||||
return r.Reason
|
|
||||||
}
|
|
||||||
// What a declared check found says an endpoint, a path or an address: evidence, never the summary the
|
// What a declared check found says an endpoint, a path or an address: evidence, never the summary the
|
||||||
// operator's channel carries (ADR 0234 §6). The summary names the check.
|
// operator's channel carries (ADR 0234 §6). The summary names the check.
|
||||||
if r.Check != "" {
|
if r.Check != "" {
|
||||||
@@ -623,9 +511,7 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
|
|||||||
if h.HeardAt.Before(since) {
|
if h.HeardAt.Before(since) {
|
||||||
return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module)
|
return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module)
|
||||||
}
|
}
|
||||||
// **A wait for the operator is checked first** (novox/hq ADR 0283): one that does not check out is unhealthy.
|
wait, waits := personWait(module, machine, h.Resources)
|
||||||
resources := checkWaiting(machine, h.Resources, f.waits)
|
|
||||||
wait, waits := personWait(module, machine, resources)
|
|
||||||
// **Only a build whose own send put the account in a new group is excused** (issue 318 review): read from
|
// **Only a build whose own send put the account in a new group is excused** (issue 318 review): read from
|
||||||
// what the controller sent, never from when the machine says the wait began — that time is the engine's
|
// what the controller sent, never from when the machine says the wait began — that time is the engine's
|
||||||
// memory, reset by its restart and moved by a change of words. A build that adds no account group cannot
|
// memory, reset by its restart and moved by a change of words. A build that adds no account group cannot
|
||||||
@@ -633,18 +519,11 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
|
|||||||
if waits && !f.groupsAdded[module] {
|
if waits && !f.groupsAdded[module] {
|
||||||
waits = false
|
waits = false
|
||||||
}
|
}
|
||||||
var found, onWaiting []string
|
var found []string
|
||||||
var forOperator []inventory.Wait
|
for _, r := range h.Resources {
|
||||||
for _, r := range resources {
|
|
||||||
if r.Module != module {
|
if r.Module != module {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
// **Any build is excused while its wait for the operator checks out** (ADR 0283 decision 4): a secret not
|
|
||||||
// given is owed by every build alike, so it is no fault of this one, and the verdict carries it.
|
|
||||||
if r.State == link.StateWaiting {
|
|
||||||
forOperator = append(forOperator, r.Waits...)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if waits && r.State == link.StateUnhealthy {
|
if waits && r.State == link.StateUnhealthy {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -662,16 +541,8 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
|
|||||||
return healthNotYet, fmt.Sprintf("its %s %s on %s is still starting", r.Kind, r.Resource, machine)
|
return healthNotYet, fmt.Sprintf("its %s %s on %s is still starting", r.Kind, r.Resource, machine)
|
||||||
case link.StateUnhealthy:
|
case link.StateUnhealthy:
|
||||||
if on, held := f.heldOn[module+"@"+machine]; held {
|
if on, held := f.heldOn[module+"@"+machine]; held {
|
||||||
// **Held under a provider that only waits for the operator** (novox/hq issue 405): a wait for a
|
|
||||||
// person, a pass carrying the wait (ADR 0254, ADR 0283 decision 4) — the walk never waits for the
|
|
||||||
// operator's secret, whichever module owes it.
|
|
||||||
if on.waits != "" {
|
|
||||||
onWaiting = append(onWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which waits for you: %s",
|
|
||||||
r.Kind, r.Resource, machine, on.on, on.waits))
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
return healthWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which is unhealthy", r.Kind,
|
return healthWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which is unhealthy", r.Kind,
|
||||||
r.Resource, machine, on.on)
|
r.Resource, machine, on)
|
||||||
}
|
}
|
||||||
return healthNotYet, fmt.Sprintf("its %s %s on %s %s", r.Kind, r.Resource, machine, reasonWords(r))
|
return healthNotYet, fmt.Sprintf("its %s %s on %s %s", r.Kind, r.Resource, machine, reasonWords(r))
|
||||||
default:
|
default:
|
||||||
@@ -679,14 +550,11 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
|
|||||||
reasonAfter(r.Reason))
|
reasonAfter(r.Reason))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if waits || len(found) > 0 || len(forOperator) > 0 || len(onWaiting) > 0 {
|
if waits || len(found) > 0 {
|
||||||
said := onWaiting
|
var said []string
|
||||||
if waits {
|
if waits {
|
||||||
said = append(said, wait)
|
said = append(said, wait)
|
||||||
}
|
}
|
||||||
if len(forOperator) > 0 {
|
|
||||||
said = append(said, operatorWaitSaid(module, machine, forOperator))
|
|
||||||
}
|
|
||||||
if len(found) > 0 {
|
if len(found) > 0 {
|
||||||
said = append(said, fmt.Sprintf("on %s, %s uses %s as found and waits for the operator to hand it over "+
|
said = append(said, fmt.Sprintf("on %s, %s uses %s as found and waits for the operator to hand it over "+
|
||||||
"(`nox node hand-over %s <directory>` on the control-node)", machine, module, strings.Join(found, ", "), machine))
|
"(`nox node hand-over %s <directory>` on the control-node)", machine, module, strings.Join(found, ", "), machine))
|
||||||
|
|||||||
@@ -3,12 +3,10 @@ package main
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
|
||||||
"reflect"
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/beside"
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/overlay"
|
"github.com/novox/mesh-controller/internal/overlay"
|
||||||
@@ -241,12 +239,13 @@ func TestATakeoverIsNotComposedForAHubPlacedOffItsTunnel(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// theResolver is the catalogue's dnsmasq module as it is (internal/beside).
|
// theResolver is the catalogue's dnsmasq module as it is, or the test is skipped where the
|
||||||
|
// catalogue is not beside this checkout.
|
||||||
func theResolver(t *testing.T) catalogue.Manifest {
|
func theResolver(t *testing.T) catalogue.Manifest {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
raw, err := os.ReadFile(filepath.Join(beside.Catalogue(t), "dnsmasq", "module.json"))
|
raw, err := os.ReadFile("../../../mesh-catalog/modules/dnsmasq/module.json")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Skipf("the catalogue is not beside this checkout: %v", err)
|
||||||
}
|
}
|
||||||
m, err := catalogue.ParseManifest(raw)
|
m, err := catalogue.ParseManifest(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -44,7 +44,7 @@ func nodeCommand(ctx context.Context, args []string) error {
|
|||||||
if len(args) != 2 {
|
if len(args) != 2 {
|
||||||
return errors.New("node show <name>")
|
return errors.New("node show <name>")
|
||||||
}
|
}
|
||||||
return showNode(ctx, open, args[1])
|
return showNode(ctx, inv, args[1])
|
||||||
case "add":
|
case "add":
|
||||||
return addNode(ctx, inv, args[1:])
|
return addNode(ctx, inv, args[1:])
|
||||||
|
|
||||||
@@ -787,8 +787,7 @@ func roughly(d time.Duration) string {
|
|||||||
//
|
//
|
||||||
// It is also where "what should it be configured as" is read. The same line that gates an
|
// It is also where "what should it be configured as" is read. The same line that gates an
|
||||||
// assignment carries `card1-DP-1`, and a person composing settings for that machine needs it.
|
// assignment carries `card1-DP-1`, and a person composing settings for that machine needs it.
|
||||||
func showNode(ctx context.Context, stored *stores, name string) error {
|
func showNode(ctx context.Context, inv *inventory.Inventory, name string) error {
|
||||||
inv := stored.inventory
|
|
||||||
node, err := inv.NodeByName(ctx, name)
|
node, err := inv.NodeByName(ctx, name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -804,19 +803,6 @@ func showNode(ctx context.Context, stored *stores, name string) error {
|
|||||||
if err := showMode(ctx, inv, node); err != nil {
|
if err := showMode(ctx, inv, node); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// What it was last sent, by whom, under which epoch, from which generation, naming which modules, and what
|
|
||||||
// it answered (novox/hq issue 485): the record of sends is read through this verb, never the store's reader.
|
|
||||||
// Unreadable is said, not taken for none.
|
|
||||||
switch last, found, err := inv.LastSend(ctx, name); {
|
|
||||||
case err != nil:
|
|
||||||
fmt.Printf(" what it was last sent could NOT be read: %v\n", err)
|
|
||||||
case !found:
|
|
||||||
fmt.Printf(" last sent no send recorded since the mesh began keeping them\n")
|
|
||||||
default:
|
|
||||||
for _, line := range lastSendLines(last) {
|
|
||||||
fmt.Println(line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Whom agents run as here, and whether that account can become root without a person (ADR 0266).
|
// Whom agents run as here, and whether that account can become root without a person (ADR 0266).
|
||||||
for _, line := range agentAccountLines(ctx, inv, node) {
|
for _, line := range agentAccountLines(ctx, inv, node) {
|
||||||
fmt.Println(line)
|
fmt.Println(line)
|
||||||
@@ -903,17 +889,6 @@ func showNode(ctx context.Context, stored *stores, name string) error {
|
|||||||
if len(assigned) > 0 {
|
if len(assigned) > 0 {
|
||||||
fmt.Printf("\n assigned: %s\n", strings.Join(assigned, ", "))
|
fmt.Printf("\n assigned: %s\n", strings.Join(assigned, ", "))
|
||||||
}
|
}
|
||||||
// And which of them a push leaves out, and why (novox/hq issue 380): judged as the push judges it, the same
|
|
||||||
// reason its condition says. Not computable is said, never read as "all applied".
|
|
||||||
plan, settings, err := planFor(ctx, stored, name)
|
|
||||||
switch {
|
|
||||||
case err != nil:
|
|
||||||
fmt.Printf("\n whether a send leaves any assigned module out is NOT known: %s\n", oneLine(err.Error()))
|
|
||||||
default:
|
|
||||||
for _, line := range notAppliedLines(name, leftOutOf(plan, settings, node.Adopted)) {
|
|
||||||
fmt.Println(line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,261 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A part that waits for the operator's secret or setting (novox/hq ADR 0283, issue 386).
|
|
||||||
//
|
|
||||||
// **A module's tool check may say it waits**: nothing of it is wrong but a part that cannot work until the operator
|
|
||||||
// gives one of its own secrets or one of its settings. The node-engine states such a resource `waiting`, with what
|
|
||||||
// it waits for. A module saying so is an assertion, so **the controller checks each wait before it excuses it**:
|
|
||||||
//
|
|
||||||
// - a wait for a secret names an own secret the module's manifest declares — by its name, or as a member of a
|
|
||||||
// secret family — said `"issued-by": "outside"`, and the store holds no value a person gave for it on that
|
|
||||||
// machine;
|
|
||||||
// - a wait for a setting names a setting the manifest declares (checked by name only: the controller cannot tell
|
|
||||||
// whether a free-form value covers a part, and the needs-operator condition is where a false one shows).
|
|
||||||
//
|
|
||||||
// An excused wait is read by the first-node gate as *waits for a person* (ADR 0254), a pass carried in the verdict,
|
|
||||||
// for any build of the module — a secret not given is owed by every build alike. It is said to the operator as
|
|
||||||
// `module.<module>.<machine>.needs-operator`, naming the act. A wait that fails the check is judged unhealthy, saying
|
|
||||||
// why, and raises the module's `unhealthy` condition.
|
|
||||||
|
|
||||||
// kindNeedsOperator is a module's condition while a part of it waits for the operator's secret or setting.
|
|
||||||
const kindNeedsOperator = "needs-operator"
|
|
||||||
|
|
||||||
// needsOperatorKey is a module's needs-operator condition on a machine.
|
|
||||||
func needsOperatorKey(module, node string) string {
|
|
||||||
return conditions.Key(conditions.ScopeModule, module+"."+node, kindNeedsOperator)
|
|
||||||
}
|
|
||||||
|
|
||||||
// operatorWaitFacts is what the controller holds to check a module's waits: the manifest judged per module, and per
|
|
||||||
// "<module>@<machine>" the own secrets a person gave there, with when. A module or a machine absent is not known,
|
|
||||||
// and no wait of it is excused.
|
|
||||||
type operatorWaitFacts struct {
|
|
||||||
manifests map[string]catalogue.Manifest
|
|
||||||
given map[string]map[string]time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkWait is nil when a wait is excused, and otherwise why not, in words. Pure.
|
|
||||||
func checkWait(module, machine string, w inventory.Wait, f operatorWaitFacts) error {
|
|
||||||
m, known := f.manifests[module]
|
|
||||||
if !known {
|
|
||||||
return fmt.Errorf("says it waits for %s, and the mesh holds no manifest of %s to check it against", waitNames(w), module)
|
|
||||||
}
|
|
||||||
switch {
|
|
||||||
case w.Secret != "" && w.Setting != "", w.Secret == "" && w.Setting == "":
|
|
||||||
return fmt.Errorf("says it waits, naming %s, where a wait names one secret or one setting", waitNames(w))
|
|
||||||
case w.Setting != "":
|
|
||||||
if _, declared := m.Settings[w.Setting]; !declared {
|
|
||||||
return fmt.Errorf("says it waits for the setting %s, which %s does not declare", w.Setting, module)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
own, _, declared := m.OwnSecrets.Lookup(w.Secret)
|
|
||||||
if !declared {
|
|
||||||
return fmt.Errorf("says it waits for the secret %s, which %s does not declare", w.Secret, module)
|
|
||||||
}
|
|
||||||
if own.IssuedBy != catalogue.IssuedOutside {
|
|
||||||
return fmt.Errorf("says it waits for the secret %s, which the mesh makes itself: only a secret issued outside "+
|
|
||||||
"the mesh waits for the operator", w.Secret)
|
|
||||||
}
|
|
||||||
given, readable := f.given[module+"@"+machine]
|
|
||||||
if !readable {
|
|
||||||
return fmt.Errorf("says it waits for the secret %s, and what was given on %s could not be read", w.Secret, machine)
|
|
||||||
}
|
|
||||||
if at, was := given[w.Secret]; was {
|
|
||||||
return fmt.Errorf("says it waits for the secret %s, which was given at %s", w.Secret,
|
|
||||||
at.UTC().Format("2006-01-02 15:04 MST"))
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// waitRefusedPrefix opens every reason checkWait gives, so the words of a refused wait are told from a check's own.
|
|
||||||
const waitRefusedPrefix = "says it waits"
|
|
||||||
|
|
||||||
// waitNames is what a wait names, as "the secret x" or "the setting y".
|
|
||||||
func waitNames(w inventory.Wait) string {
|
|
||||||
switch {
|
|
||||||
case w.Secret != "" && w.Setting != "":
|
|
||||||
return "the secret " + w.Secret + " and the setting " + w.Setting
|
|
||||||
case w.Secret != "":
|
|
||||||
return "the secret " + w.Secret
|
|
||||||
case w.Setting != "":
|
|
||||||
return "the setting " + w.Setting
|
|
||||||
}
|
|
||||||
return "nothing"
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkWaiting reads one machine's resources against the facts: every waiting resource whose waits all check out is
|
|
||||||
// kept as said; one with a wait that does not, or with no wait at all, is answered as unhealthy with why. Pure; the
|
|
||||||
// statement as kept is not changed.
|
|
||||||
func checkWaiting(machine string, rs []inventory.ResourceHealth, f operatorWaitFacts) []inventory.ResourceHealth {
|
|
||||||
out := make([]inventory.ResourceHealth, 0, len(rs))
|
|
||||||
for _, r := range rs {
|
|
||||||
if r.State == link.StateWaiting {
|
|
||||||
var why error
|
|
||||||
if len(r.Waits) == 0 {
|
|
||||||
why = fmt.Errorf("says it waits, and names nothing it waits for")
|
|
||||||
}
|
|
||||||
for _, w := range r.Waits {
|
|
||||||
if why == nil {
|
|
||||||
why = checkWait(r.Module, machine, w, f)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if why != nil {
|
|
||||||
r.State, r.Reason = link.StateUnhealthy, why.Error()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
out = append(out, r)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// operatorWait is whether everything not healthy of a module on a machine is waiting with its waits checked
|
|
||||||
// (checkWaiting already applied), and those waits. A module with anything unhealthy, starting or unknown beside it
|
|
||||||
// does not wait: it is judged as before.
|
|
||||||
func operatorWait(module string, rs []inventory.ResourceHealth) ([]inventory.Wait, bool) {
|
|
||||||
var waits []inventory.Wait
|
|
||||||
for _, r := range rs {
|
|
||||||
if r.Module != module {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
switch r.State {
|
|
||||||
case link.StateHealthy:
|
|
||||||
case link.StateWaiting:
|
|
||||||
waits = append(waits, r.Waits...)
|
|
||||||
default:
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return waits, len(waits) > 0
|
|
||||||
}
|
|
||||||
|
|
||||||
// operatorWaitSaid is a module's wait for the operator in one sentence, for the gate's verdict and the condition's
|
|
||||||
// summary: what the operator gives and what it names, and for a secret the line that opens the desk prompt.
|
|
||||||
func operatorWaitSaid(module, machine string, waits []inventory.Wait) string {
|
|
||||||
var parts []string
|
|
||||||
for _, w := range waits {
|
|
||||||
part := fmt.Sprintf("%s (%s", w.What, waitNames(w))
|
|
||||||
if w.Secret != "" {
|
|
||||||
part += fmt.Sprintf(", given with `nox secret ask %s %s %s`", machine, module, w.Secret)
|
|
||||||
}
|
|
||||||
parts = append(parts, part+")")
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("%s on %s waits for the operator: %s", module, machine, strings.Join(parts, "; "))
|
|
||||||
}
|
|
||||||
|
|
||||||
// needsOperatorObservation is a module whose only parts not healthy wait for the operator (ADR 0283): the operator's,
|
|
||||||
// a warning however long it stands, its plain words naming the act and never saying there is nothing to do.
|
|
||||||
func needsOperatorObservation(module, node string, waits []inventory.Wait, rs []inventory.ResourceHealth) conditions.Observation {
|
|
||||||
o := moduleUnhealthyObservation(module, node, rs)
|
|
||||||
o.Token, o.Kind, o.Resolver, o.Severity = kindNeedsOperator, kindNeedsOperator, conditions.ResolverOperator, conditions.Warning
|
|
||||||
o.Summary = operatorWaitSaid(module, node, waits)
|
|
||||||
w := needsOperatorWords(module, node, waits)
|
|
||||||
o.Headline, o.Explanation, o.Needs, o.Resolved, o.Actions = w.Headline, w.Explanation, w.Needs, w.Resolved, nil
|
|
||||||
return o
|
|
||||||
}
|
|
||||||
|
|
||||||
// needsOperatorWords is what the operator reads of a module waiting for them (ADR 0253, ADR 0283): the act, for a
|
|
||||||
// secret typed at the machine's desk prompt and for a setting approved when an agent proposes it. The secret's and
|
|
||||||
// the setting's names, and the line, are in the summary for whoever looks closer.
|
|
||||||
func needsOperatorWords(module, node string, waits []inventory.Wait) words {
|
|
||||||
var acts []string
|
|
||||||
seen := map[string]bool{}
|
|
||||||
secret := false
|
|
||||||
for _, w := range waits {
|
|
||||||
var act string
|
|
||||||
switch {
|
|
||||||
case w.Secret != "":
|
|
||||||
act, secret = fmt.Sprintf("type %s at %s's desk prompt", w.What, node), true
|
|
||||||
case w.Setting != "":
|
|
||||||
act = fmt.Sprintf("approve %s of %s on %s when it is proposed to you", w.Setting, module, node)
|
|
||||||
}
|
|
||||||
if act != "" && !seen[act] {
|
|
||||||
seen[act] = true
|
|
||||||
acts = append(acts, act)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
needs := strings.Join(acts, "; and ") + "."
|
|
||||||
// Plain words hold one sentence of at most conditions.NeedsMax characters: several acts are named in the
|
|
||||||
// summary instead.
|
|
||||||
if len(acts) == 0 || len(needs) > conditions.NeedsMax {
|
|
||||||
needs = fmt.Sprintf("give what %s waits for on %s; the details name each secret and setting.", module, node)
|
|
||||||
}
|
|
||||||
explanation := fmt.Sprintf("Part of %s on %s cannot work until you give what it waits for.", module, node)
|
|
||||||
if secret {
|
|
||||||
explanation += " A hidden prompt opens at the desk when the secret is asked for, and what you type there " +
|
|
||||||
"is sealed to the machine."
|
|
||||||
}
|
|
||||||
explanation += " Its update is in place and nothing was undone; it carries on by itself once it is given."
|
|
||||||
return words{
|
|
||||||
Headline: fmt.Sprintf("%s waits for you on %s", module, node),
|
|
||||||
Needs: needs,
|
|
||||||
Explanation: explanation,
|
|
||||||
Resolved: fmt.Sprintf("%s on %s no longer waits for you", module, node),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// readWaitFacts reads what the controller holds to check the waits of the modules named on one machine: the
|
|
||||||
// manifests (the catalogue's, or those given) and the secrets given there. A read that fails leaves that module
|
|
||||||
// unknown, so none of its waits is excused.
|
|
||||||
func readWaitFacts(ctx context.Context, inv *inventory.Inventory, machine string, modules []string,
|
|
||||||
manifests map[string]catalogue.Manifest, f *operatorWaitFacts) {
|
|
||||||
if f.manifests == nil {
|
|
||||||
f.manifests = map[string]catalogue.Manifest{}
|
|
||||||
}
|
|
||||||
if f.given == nil {
|
|
||||||
f.given = map[string]map[string]time.Time{}
|
|
||||||
}
|
|
||||||
if inv == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var shelf map[string]catalogue.Manifest
|
|
||||||
sort.Strings(modules)
|
|
||||||
for _, module := range modules {
|
|
||||||
if _, has := f.manifests[module]; !has {
|
|
||||||
if m, given := manifests[module]; given {
|
|
||||||
f.manifests[module] = m
|
|
||||||
} else {
|
|
||||||
if shelf == nil {
|
|
||||||
var err error
|
|
||||||
if shelf, err = inv.Catalogue(ctx); err != nil {
|
|
||||||
shelf = map[string]catalogue.Manifest{}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if m, known := shelf[module]; known {
|
|
||||||
f.manifests[module] = m
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if _, read := f.given[module+"@"+machine]; read {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if given, err := inv.GivenOwnSecrets(ctx, machine, module); err == nil {
|
|
||||||
f.given[module+"@"+machine] = given
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// waitingModules is every module with a waiting resource in a statement.
|
|
||||||
func waitingModules(rs []inventory.ResourceHealth) []string {
|
|
||||||
seen := map[string]bool{}
|
|
||||||
var out []string
|
|
||||||
for _, r := range rs {
|
|
||||||
if r.State == link.StateWaiting && r.Module != "" && !seen[r.Module] {
|
|
||||||
seen[r.Module] = true
|
|
||||||
out = append(out, r.Module)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
@@ -1,264 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A part that waits for the operator's secret or setting (novox/hq ADR 0283, issue 386).
|
|
||||||
|
|
||||||
var mountsManifest = catalogue.Manifest{Module: "mounts", Version: "1",
|
|
||||||
Settings: map[string]catalogue.SettingDeclaration{"smb-users": {}, "sources": {}},
|
|
||||||
OwnSecrets: catalogue.OwnSecrets{
|
|
||||||
"smb-password-*": {Path: "/s/smb-password-*.secret", IssuedBy: catalogue.IssuedOutside},
|
|
||||||
"broker": {Path: "/s/broker"},
|
|
||||||
"made": {Path: "/s/made", Taken: catalogue.TakenAtStart},
|
|
||||||
"licence": {Path: "/s/licence", IssuedBy: catalogue.IssuedOutside},
|
|
||||||
}}
|
|
||||||
|
|
||||||
var passwordWait = inventory.Wait{Part: "the source games", Secret: "smb-password-games",
|
|
||||||
What: "the password of the source games"}
|
|
||||||
|
|
||||||
var usernameWait = inventory.Wait{Part: "the source games", Setting: "smb-users", What: "the username of the source games"}
|
|
||||||
|
|
||||||
func waitingResource(waits ...inventory.Wait) inventory.ResourceHealth {
|
|
||||||
return inventory.ResourceHealth{Module: "mounts", Resource: "mounts.watch", Kind: "process",
|
|
||||||
Target: "mesh-mounts-watch.service", State: link.StateWaiting, Check: "tool",
|
|
||||||
Reason: "the source games waits for its password", Waits: waits}
|
|
||||||
}
|
|
||||||
|
|
||||||
func factsGiven(given map[string]time.Time) operatorWaitFacts {
|
|
||||||
return operatorWaitFacts{manifests: map[string]catalogue.Manifest{"mounts": mountsManifest},
|
|
||||||
given: map[string]map[string]time.Time{"mounts@workstation": given}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Rule 3: a wait is excused only when what it names is the module's, issued outside the mesh, and not given there.
|
|
||||||
func TestAWaitIsExcusedOnlyWhenItChecksOut(t *testing.T) {
|
|
||||||
at := time.Date(2026, 10, 10, 15, 8, 0, 0, time.UTC)
|
|
||||||
for _, c := range []struct {
|
|
||||||
name string
|
|
||||||
w inventory.Wait
|
|
||||||
f operatorWaitFacts
|
|
||||||
says string // "" when excused
|
|
||||||
}{
|
|
||||||
{"a member of an outside family, not given", passwordWait, factsGiven(nil), ""},
|
|
||||||
{"an outside secret by name, not given", inventory.Wait{Part: "p", Secret: "licence", What: "w"}, factsGiven(nil), ""},
|
|
||||||
{"a declared setting", usernameWait, factsGiven(nil), ""},
|
|
||||||
{"a member given", passwordWait, factsGiven(map[string]time.Time{"smb-password-games": at}), "was given at 2026-10-10 15:08"},
|
|
||||||
{"a secret not declared", inventory.Wait{Part: "p", Secret: "smb-credentials", What: "w"}, factsGiven(nil), "does not declare"},
|
|
||||||
{"a secret the mesh makes", inventory.Wait{Part: "p", Secret: "made", What: "w"}, factsGiven(nil), "mesh makes itself"},
|
|
||||||
{"the bus account", inventory.Wait{Part: "p", Secret: "broker", What: "w"}, factsGiven(nil), "mesh makes itself"},
|
|
||||||
{"a setting not declared", inventory.Wait{Part: "p", Setting: "logins", What: "w"}, factsGiven(nil), "does not declare"},
|
|
||||||
{"both", inventory.Wait{Part: "p", Secret: "licence", Setting: "smb-users", What: "w"}, factsGiven(nil), "one secret or one setting"},
|
|
||||||
{"neither", inventory.Wait{Part: "p", What: "w"}, factsGiven(nil), "one secret or one setting"},
|
|
||||||
{"no manifest known", passwordWait, operatorWaitFacts{}, "no manifest"},
|
|
||||||
{"what was given cannot be read", passwordWait,
|
|
||||||
operatorWaitFacts{manifests: map[string]catalogue.Manifest{"mounts": mountsManifest}}, "could not be read"},
|
|
||||||
} {
|
|
||||||
err := checkWait("mounts", "workstation", c.w, c.f)
|
|
||||||
switch {
|
|
||||||
case c.says == "" && err != nil:
|
|
||||||
t.Errorf("%s: refused: %v", c.name, err)
|
|
||||||
case c.says != "" && (err == nil || !strings.Contains(err.Error(), c.says)):
|
|
||||||
t.Errorf("%s: %v; want a refusal saying %q", c.name, err, c.says)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A waiting resource whose wait does not check out, or that names nothing, is judged unhealthy, saying why.
|
|
||||||
func TestAWaitThatFailsItsCheckIsUnhealthy(t *testing.T) {
|
|
||||||
given := factsGiven(map[string]time.Time{"smb-password-games": time.Now()})
|
|
||||||
got := checkWaiting("workstation", []inventory.ResourceHealth{waitingResource(passwordWait)}, given)
|
|
||||||
if got[0].State != link.StateUnhealthy || !strings.Contains(got[0].Reason, "was given") {
|
|
||||||
t.Fatalf("a wait for a secret given: %+v", got[0])
|
|
||||||
}
|
|
||||||
got = checkWaiting("workstation", []inventory.ResourceHealth{waitingResource()}, factsGiven(nil))
|
|
||||||
if got[0].State != link.StateUnhealthy || !strings.Contains(got[0].Reason, "names nothing") {
|
|
||||||
t.Fatalf("a wait naming nothing: %+v", got[0])
|
|
||||||
}
|
|
||||||
got = checkWaiting("workstation", []inventory.ResourceHealth{waitingResource(passwordWait, usernameWait)}, factsGiven(nil))
|
|
||||||
if got[0].State != link.StateWaiting {
|
|
||||||
t.Fatalf("two waits that check out: %+v", got[0])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Rule 4: the gate passes a module whose only parts not healthy wait for the operator, carrying the wait; anything
|
|
||||||
// else beside it is judged as before; and any build is excused, not only one that added something.
|
|
||||||
func TestTheGatePassesAWaitForTheOperatorCarriedAlong(t *testing.T) {
|
|
||||||
now := time.Now()
|
|
||||||
since := now.Add(-time.Minute)
|
|
||||||
healthy := inventory.ResourceHealth{Module: "mounts", Resource: "mounts.apply", Kind: "process",
|
|
||||||
Target: "mesh-mounts-apply.service", State: link.StateHealthy}
|
|
||||||
f := gateFacts{now: now, waits: factsGiven(nil), groupsAdded: map[string]bool{"mounts": false},
|
|
||||||
health: map[string]inventory.NodeHealth{"workstation": {Node: "workstation", HeardAt: now,
|
|
||||||
Resources: []inventory.ResourceHealth{healthy, waitingResource(passwordWait)}}}}
|
|
||||||
h, why := moduleHealthWord("mounts", "workstation", since, f)
|
|
||||||
if h != healthPerson || !strings.Contains(why, "waits for the operator: the password of the source games") ||
|
|
||||||
!strings.Contains(why, "nox secret ask workstation mounts smb-password-games") {
|
|
||||||
t.Fatalf("an excused wait reads %v %q; want a wait for a person naming the act", h, why)
|
|
||||||
}
|
|
||||||
// A second resource unhealthy beside it: not yet, as before.
|
|
||||||
down := healthy
|
|
||||||
down.State, down.Reason = link.StateUnhealthy, "down"
|
|
||||||
f.health["workstation"] = inventory.NodeHealth{Node: "workstation", HeardAt: now,
|
|
||||||
Resources: []inventory.ResourceHealth{down, waitingResource(passwordWait)}}
|
|
||||||
if h, why := moduleHealthWord("mounts", "workstation", since, f); h != healthNotYet {
|
|
||||||
t.Fatalf("a resource down beside the wait reads %v %q", h, why)
|
|
||||||
}
|
|
||||||
// The password given and the module still saying it waits: not excused.
|
|
||||||
f.waits = factsGiven(map[string]time.Time{"smb-password-games": now})
|
|
||||||
f.health["workstation"] = inventory.NodeHealth{Node: "workstation", HeardAt: now,
|
|
||||||
Resources: []inventory.ResourceHealth{healthy, waitingResource(passwordWait)}}
|
|
||||||
if h, why := moduleHealthWord("mounts", "workstation", since, f); h != healthNotYet || !strings.Contains(why, "was given") {
|
|
||||||
t.Fatalf("a wait for a secret given reads %v %q", h, why)
|
|
||||||
}
|
|
||||||
// Facts never read (no manifest): never excused.
|
|
||||||
f.waits = operatorWaitFacts{}
|
|
||||||
if h, _ := moduleHealthWord("mounts", "workstation", since, f); h != healthNotYet {
|
|
||||||
t.Fatalf("a wait nothing could check reads %v", h)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func needsOperatorOpen(t *testing.T, k *conditions.Keeper) (*conditions.Condition, []conditions.Condition) {
|
|
||||||
t.Helper()
|
|
||||||
open, err := k.Open(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for i, c := range open {
|
|
||||||
if c.Key == needsOperatorKey("mounts", "workstation") {
|
|
||||||
return &open[i], open
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil, open
|
|
||||||
}
|
|
||||||
|
|
||||||
// Rule 5: two statements of an excused wait raise needs-operator, the operator's, a warning however long, naming the
|
|
||||||
// act; a statement without it clears it.
|
|
||||||
func TestTheNeedsOperatorConditionNamesTheAct(t *testing.T) {
|
|
||||||
k, _ := withConditionsInMemory(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
rs := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}}
|
|
||||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 1}, time.Now()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got, _ := needsOperatorOpen(t, k); got != nil {
|
|
||||||
t.Fatal("raised on one statement")
|
|
||||||
}
|
|
||||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 2}, time.Now()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
got, open := needsOperatorOpen(t, k)
|
|
||||||
if got == nil {
|
|
||||||
t.Fatalf("not raised on two statements: %+v", open)
|
|
||||||
}
|
|
||||||
for _, c := range open {
|
|
||||||
if c.Kind == kindModuleUnhealthy {
|
|
||||||
t.Fatalf("raised as a fault too: %+v", c)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if got.Kind != kindNeedsOperator || got.Resolver != conditions.ResolverOperator || got.Severity != conditions.Warning {
|
|
||||||
t.Fatalf("the condition: %+v", got)
|
|
||||||
}
|
|
||||||
if !strings.Contains(got.Needs, "type the password of the source games at workstation's desk prompt") ||
|
|
||||||
!strings.Contains(got.Explanation, "hidden prompt opens at the desk") {
|
|
||||||
t.Fatalf("its needs do not name the act: %q", got.Needs)
|
|
||||||
}
|
|
||||||
if strings.Contains(strings.ToLower(got.Explanation), "nothing for you") || !strings.Contains(got.Explanation, "nothing was undone") {
|
|
||||||
t.Fatalf("its explanation: %q", got.Explanation)
|
|
||||||
}
|
|
||||||
if !strings.Contains(got.Summary, "smb-password-games") || !strings.Contains(got.Summary, "nox secret ask workstation mounts smb-password-games") {
|
|
||||||
t.Fatalf("its summary does not name the secret and the line: %q", got.Summary)
|
|
||||||
}
|
|
||||||
// Long open is still a warning: only the operator can end it.
|
|
||||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 3}, time.Now().Add(48*time.Hour)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got, _ := needsOperatorOpen(t, k); got == nil || got.Severity == conditions.Urgent {
|
|
||||||
t.Fatalf("after two days: %+v", got)
|
|
||||||
}
|
|
||||||
// Given: the next statement does not say it, and it clears.
|
|
||||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got, _ := needsOperatorOpen(t, k); got != nil {
|
|
||||||
t.Fatal("not cleared once given")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestASettingsWaitAsksForTheApproval(t *testing.T) {
|
|
||||||
k, _ := withConditionsInMemory(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
rs := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(usernameWait)}}
|
|
||||||
for i := 1; i <= 2; i++ {
|
|
||||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": i}, time.Now()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
got, _ := needsOperatorOpen(t, k)
|
|
||||||
if got == nil || !strings.Contains(got.Needs, "approve smb-users of mounts on workstation when it is proposed to you") {
|
|
||||||
t.Fatalf("the condition: %+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A wait that fails its check is the module's own fault: unhealthy, with why, and no needs-operator.
|
|
||||||
func TestAWaitThatFailsItsCheckRaisesUnhealthy(t *testing.T) {
|
|
||||||
k, _ := withConditionsInMemory(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
checked := checkWaiting("workstation", []inventory.ResourceHealth{waitingResource(passwordWait)},
|
|
||||||
factsGiven(map[string]time.Time{"smb-password-games": time.Now()}))
|
|
||||||
rs := map[string][]inventory.ResourceHealth{"mounts": checked}
|
|
||||||
for i := 1; i <= 2; i++ {
|
|
||||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": i}, time.Now()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
got, open := needsOperatorOpen(t, k)
|
|
||||||
if got != nil {
|
|
||||||
t.Fatalf("a wait for a secret given raised needs-operator: %+v", got)
|
|
||||||
}
|
|
||||||
unhealthy := false
|
|
||||||
for _, c := range open {
|
|
||||||
unhealthy = unhealthy || c.Key == moduleUnhealthyKey("mounts", "workstation")
|
|
||||||
}
|
|
||||||
if !unhealthy {
|
|
||||||
t.Fatalf("not raised as unhealthy: %+v", open)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A module that waited and is then broken says so when the wait clears, and the other way round.
|
|
||||||
func TestANeedsOperatorThatBecameUnhealthySaysSo(t *testing.T) {
|
|
||||||
k, _ := withConditionsInMemory(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
waiting := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}}
|
|
||||||
for i := 1; i <= 2; i++ {
|
|
||||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", waiting, map[string]int{"mounts": i}, time.Now()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
broken := waitingResource()
|
|
||||||
broken.State, broken.Reason, broken.Waits = link.StateUnhealthy, "the source games refused its login", nil
|
|
||||||
for i := 3; i <= 4; i++ {
|
|
||||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": {broken}},
|
|
||||||
map[string]int{"mounts": i}, time.Now()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
got, open := needsOperatorOpen(t, k)
|
|
||||||
if got != nil {
|
|
||||||
t.Fatal("needs-operator still open after it became a fault")
|
|
||||||
}
|
|
||||||
found := false
|
|
||||||
for _, c := range open {
|
|
||||||
found = found || c.Key == moduleUnhealthyKey("mounts", "workstation")
|
|
||||||
}
|
|
||||||
if !found {
|
|
||||||
t.Fatalf("the fault is not raised: %+v", open)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,125 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A delivery over its budget is loud (novox/hq ADR 0282 decision 7, issue 382): the self-check reads
|
|
||||||
// mesh-delivery's `times` and raises the warning `delivery.<class>.over-budget` when the newest delivery of a
|
|
||||||
// class whose delivery time is known took longer than its class's budget — three minutes for a leaf module (ADR
|
|
||||||
// 0304 decision 4), ten for a core module — naming the delivery and its longest phase. It clears when the next delivery of that class
|
|
||||||
// lands within its budget. Measurement only: the condition says, it never holds or acts on a delivery.
|
|
||||||
|
|
||||||
// probeBudgetsID is the probe that reads the delivery times.
|
|
||||||
const probeBudgetsID = "D16"
|
|
||||||
|
|
||||||
// kindOverBudget is what a delivery over its class's budget raises.
|
|
||||||
const kindOverBudget = "over-budget"
|
|
||||||
|
|
||||||
// deliveryBudgets are the budgets by class (ADR 0282 decision 1, the leaf's as ADR 0304 decision 4 changed it);
|
|
||||||
// the probe raises nothing for another class.
|
|
||||||
var deliveryBudgets = map[string]time.Duration{inventory.ClassLeaf: 3 * time.Minute, inventory.ClassCore: 10 * time.Minute}
|
|
||||||
|
|
||||||
// timesAnswer is what mesh-delivery's `times` answers, as far as the probe reads it.
|
|
||||||
type timesAnswer struct {
|
|
||||||
Classes []timesClass `json:"classes"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// timesClass is one class's line of `times`.
|
|
||||||
type timesClass struct {
|
|
||||||
Class string `json:"class"`
|
|
||||||
Latest *timesLatest `json:"latest,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// timesLatest is the newest delivery of a class whose delivery time is known.
|
|
||||||
type timesLatest struct {
|
|
||||||
ID string `json:"id"`
|
|
||||||
TookMS int64 `json:"took_ms"`
|
|
||||||
Longest string `json:"longest,omitempty"`
|
|
||||||
Landed string `json:"landed,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// deliveryTimes is what the delivery's owner says of its delivery times; nothing when no holder is on record or
|
|
||||||
// none answers (D3 says that one).
|
|
||||||
func deliveryTimes(ctx context.Context, conn *nats.Conn, held bool) (*timesAnswer, error) {
|
|
||||||
if !held {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
raw, err := askDeliveryOwner(ctx, conn, "times", map[string]any{})
|
|
||||||
if errors.Is(err, link.ErrNothingServes) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var a timesAnswer
|
|
||||||
if err := json.Unmarshal(raw, &a); err != nil {
|
|
||||||
return nil, fmt.Errorf("%s.times answered something unreadable: %w", catalogue.DeliverySeat, err)
|
|
||||||
}
|
|
||||||
return &a, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// overBudgetObservations are the conditions of the classes whose newest delivery took longer than its budget:
|
|
||||||
// strictly longer, so a delivery of exactly its budget is within it.
|
|
||||||
func overBudgetObservations(a *timesAnswer) []conditions.Observation {
|
|
||||||
if a == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
var out []conditions.Observation
|
|
||||||
for _, c := range a.Classes {
|
|
||||||
budget, ok := deliveryBudgets[c.Class]
|
|
||||||
if !ok || c.Latest == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
took := time.Duration(c.Latest.TookMS) * time.Millisecond
|
|
||||||
if took <= budget {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
longest := c.Latest.Longest
|
|
||||||
if longest == "" {
|
|
||||||
longest = "not known"
|
|
||||||
}
|
|
||||||
out = append(out, conditions.Observation{Scope: conditions.ScopeDelivery, ID: c.Class, Kind: kindOverBudget,
|
|
||||||
Severity: conditions.Warning,
|
|
||||||
Summary: fmt.Sprintf("the %s delivery %s took %s from its merge to running everywhere, over its budget of %s; "+
|
|
||||||
"its longest phase: %s — `mesh-delivery.times`", c.Class, c.Latest.ID, humanDuration(took),
|
|
||||||
humanDuration(budget), longest),
|
|
||||||
Said: fmt.Sprintf("%s took %s (budget %s), longest phase %s", c.Latest.ID, took.Round(time.Second), budget, longest),
|
|
||||||
Headline: fmt.Sprintf("A %s delivery took %s, over its %s budget", c.Class, humanDuration(took),
|
|
||||||
humanDuration(budget)),
|
|
||||||
Explanation: fmt.Sprintf("The delivery %s took %s from its merge until every machine ran it; a %s module is "+
|
|
||||||
"held to %s (ADR 0282). Most of the time went to %s. Nothing was held or changed because of this: it is "+
|
|
||||||
"a measurement.", c.Latest.ID, humanDuration(took), c.Class, humanDuration(budget), longest),
|
|
||||||
Resolved: fmt.Sprintf("the next %s delivery lands within %s", c.Class, humanDuration(budget)),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// probeBudgets is D16: the newest delivery of each class lands within its class's budget.
|
|
||||||
func probeBudgets(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
|
||||||
entries, err := d.open.inventory.Catalogued(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var conn *nats.Conn
|
|
||||||
if d.js != nil {
|
|
||||||
conn = d.js.Conn()
|
|
||||||
}
|
|
||||||
a, err := deliveryTimes(ctx, conn, deliverySeatHeld(entries))
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return overBudgetObservations(a), nil
|
|
||||||
}
|
|
||||||
@@ -1,112 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A leaf delivery of 3 minutes 10 seconds raises delivery.leaf.over-budget naming its longest phase; one of
|
|
||||||
// exactly three minutes, a core one of nine and a class without a budget raise nothing (ADR 0282 decision 7,
|
|
||||||
// the leaf's budget as ADR 0304 decision 4 changed it).
|
|
||||||
func TestADeliveryOverItsBudgetIsLoud(t *testing.T) {
|
|
||||||
a := ×Answer{Classes: []timesClass{
|
|
||||||
{Class: inventory.ClassLeaf, Latest: ×Latest{ID: "novox/mesh-catalog@abc", TookMS: (3*time.Minute + 10*time.Second).Milliseconds(),
|
|
||||||
Longest: "judgement 2m40s"}},
|
|
||||||
{Class: inventory.ClassCore, Latest: ×Latest{ID: "novox/mesh-controller@def", TookMS: (9 * time.Minute).Milliseconds(),
|
|
||||||
Longest: "build 1m"}},
|
|
||||||
{Class: "unclassed", Latest: ×Latest{ID: "x@y", TookMS: time.Hour.Milliseconds()}},
|
|
||||||
}}
|
|
||||||
obs := overBudgetObservations(a)
|
|
||||||
if len(obs) != 1 {
|
|
||||||
t.Fatalf("one class over its budget, got %d: %+v", len(obs), obs)
|
|
||||||
}
|
|
||||||
o := obs[0]
|
|
||||||
if o.Key() != "delivery.leaf.over-budget" || !strings.Contains(o.Summary, "judgement 2m40s") ||
|
|
||||||
!strings.Contains(o.Summary, "novox/mesh-catalog@abc") {
|
|
||||||
t.Fatalf("the condition names its delivery and longest phase: %s — %s", o.Key(), o.Summary)
|
|
||||||
}
|
|
||||||
// The next leaf delivery within its budget clears it: the probe raises nothing for the class.
|
|
||||||
a.Classes[0].Latest = ×Latest{ID: "novox/mesh-catalog@ghi", TookMS: (3 * time.Minute).Milliseconds()}
|
|
||||||
if obs := overBudgetObservations(a); len(obs) != 0 {
|
|
||||||
t.Fatalf("a delivery of exactly its budget is within it: %+v", obs)
|
|
||||||
}
|
|
||||||
a.Classes[1].Latest.TookMS = (10*time.Minute + time.Second).Milliseconds()
|
|
||||||
if obs := overBudgetObservations(a); len(obs) != 1 || obs[0].Key() != "delivery.core.over-budget" {
|
|
||||||
t.Fatalf("a core delivery over ten minutes: %+v", obs)
|
|
||||||
}
|
|
||||||
if obs := overBudgetObservations(nil); obs != nil {
|
|
||||||
t.Fatal("no answer raises nothing")
|
|
||||||
}
|
|
||||||
// No delivery of a class with a known time yet: nothing said of it.
|
|
||||||
if obs := overBudgetObservations(×Answer{Classes: []timesClass{{Class: inventory.ClassLeaf}}}); len(obs) != 0 {
|
|
||||||
t.Fatalf("a class with no delivery: %+v", obs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The probe's question is one the controller's grant names: a question the bus refuses checks nothing.
|
|
||||||
func TestTheControllerMayAskForTheDeliveryTimes(t *testing.T) {
|
|
||||||
found := false
|
|
||||||
for _, v := range broker.VerbsTheControllerAsksTheDeliveryOwner {
|
|
||||||
found = found || (v.Seat == catalogue.DeliverySeat && v.Verb == "times")
|
|
||||||
}
|
|
||||||
if !found {
|
|
||||||
t.Fatal("the grant does not name mesh-delivery.times")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A walk's class is core when it walks a module of the controller's own path or one holding the mesh's resolver,
|
|
||||||
// leaf otherwise; its window closed at its batch's window, or its maximum, never after its cut.
|
|
||||||
func TestAWalksClassAndWindowAreReadAtItsCut(t *testing.T) {
|
|
||||||
entries := []inventory.Entry{
|
|
||||||
{Manifest: catalogue.Manifest{Module: "dnsmasq", Claims: []catalogue.Claim{{Name: resolverSeat}}}},
|
|
||||||
{Manifest: catalogue.Manifest{Module: "gitea"}},
|
|
||||||
}
|
|
||||||
walk := func(modules ...string) inventory.Plan {
|
|
||||||
p := inventory.Plan{Modules: map[string]*inventory.PlanModule{}}
|
|
||||||
for _, m := range modules {
|
|
||||||
p.Modules[m] = &inventory.PlanModule{}
|
|
||||||
}
|
|
||||||
return p
|
|
||||||
}
|
|
||||||
for want, w := range map[string]inventory.Plan{
|
|
||||||
inventory.ClassLeaf: walk("gitea"), inventory.ClassCore: walk("gitea", "mesh-controller"),
|
|
||||||
} {
|
|
||||||
if got := classOf(w, entries); got != want {
|
|
||||||
t.Errorf("%v: %s, want %s", w.Modules, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if got := classOf(walk("dnsmasq"), entries); got != inventory.ClassCore {
|
|
||||||
t.Errorf("the resolver's holder is core: %s", got)
|
|
||||||
}
|
|
||||||
now := time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC)
|
|
||||||
batch := inventory.Plan{Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{
|
|
||||||
ClosesAt: now.Add(-20 * time.Second), AtMost: now.Add(5 * time.Minute)}}}
|
|
||||||
times := walkTimesAtCut(batch, walk("gitea"), entries, now)
|
|
||||||
if times.Cut == nil || !times.Cut.Equal(now) || times.WindowClosed == nil || !times.WindowClosed.Equal(now.Add(-20*time.Second)) ||
|
|
||||||
times.Class != inventory.ClassLeaf {
|
|
||||||
t.Fatalf("times at the cut: %+v", times)
|
|
||||||
}
|
|
||||||
batch.Delivery.Batch.AtMost = now.Add(-time.Minute)
|
|
||||||
if times := walkTimesAtCut(batch, walk("gitea"), entries, now); !times.WindowClosed.Equal(now.Add(-time.Minute)) {
|
|
||||||
t.Fatalf("a window closed at its maximum: %v", times.WindowClosed)
|
|
||||||
}
|
|
||||||
if times := walkTimesAtCut(inventory.Plan{Delivery: &inventory.PlanDelivery{Alone: true}}, walk("gitea"), entries, now); times.WindowClosed != nil {
|
|
||||||
t.Fatalf("a merge walked alone had no window: %v", times.WindowClosed)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// What each machine of the rest was sent is kept only for the machines the send reached.
|
|
||||||
func TestTheRestIsKeptForTheMachinesTheSendReached(t *testing.T) {
|
|
||||||
got := restOf([]string{"ace", "g14"}, []string{"ace", "shanks"}, map[string]inventory.SentDeclaration{"ace": {Digest: "d1"}})
|
|
||||||
if len(got) != 1 || got["ace"].Digest != "d1" {
|
|
||||||
t.Fatalf("rest: %+v", got)
|
|
||||||
}
|
|
||||||
if restOf([]string{"g14"}, []string{"ace"}, nil) != nil {
|
|
||||||
t.Fatal("no machine reached: nothing kept")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -221,24 +221,6 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
|||||||
w := usedAsFoundObservation(orModule(module), machineOr(o, "a machine"), o.Summary, nil)
|
w := usedAsFoundObservation(orModule(module), machineOr(o, "a machine"), o.Summary, nil)
|
||||||
return words{Headline: w.Headline, Explanation: w.Explanation, Needs: w.Needs, Resolved: w.Resolved}
|
return words{Headline: w.Headline, Explanation: w.Explanation, Needs: w.Needs, Resolved: w.Resolved}
|
||||||
}),
|
}),
|
||||||
kindNeedsOperator: worded(func(o conditions.Observation) words {
|
|
||||||
// The observation carries the act itself (ADR 0283); these are its words when only the kind is known.
|
|
||||||
module := ""
|
|
||||||
if o.Scope == conditions.ScopeModule && o.Machine != "" {
|
|
||||||
module = strings.TrimSuffix(o.ID, "."+o.Machine)
|
|
||||||
}
|
|
||||||
node := machineOr(o, "a machine")
|
|
||||||
w := needsOperatorWords(orModule(module), node, nil)
|
|
||||||
return w
|
|
||||||
}),
|
|
||||||
kindLeftOut: worded(func(o conditions.Observation) words {
|
|
||||||
// The observation carries its own words (novox/hq issue 380); these are its kind's alone.
|
|
||||||
module := ""
|
|
||||||
if o.Scope == conditions.ScopeModule && o.Machine != "" {
|
|
||||||
module = strings.TrimSuffix(o.ID, "."+o.Machine)
|
|
||||||
}
|
|
||||||
return leftOutWords(orModule(module), machineOr(o, "a machine"), "")
|
|
||||||
}),
|
|
||||||
kindProviderFailing: worded(func(o conditions.Observation) words {
|
kindProviderFailing: worded(func(o conditions.Observation) words {
|
||||||
thing, consumer := conditions.ThingWords(o), idPart(o, 2)
|
thing, consumer := conditions.ThingWords(o), idPart(o, 2)
|
||||||
if consumer == "" {
|
if consumer == "" {
|
||||||
@@ -614,19 +596,6 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
|||||||
Explanation: "The bus refused messages from a part of the mesh, so what they carried did not happen.",
|
Explanation: "The bus refused messages from a part of the mesh, so what they carried did not happen.",
|
||||||
Resolved: "Resolved: the bus takes the messages again"}
|
Resolved: "Resolved: the bus takes the messages again"}
|
||||||
}),
|
}),
|
||||||
kindBucketOrLogFilling: worded(func(o conditions.Observation) words {
|
|
||||||
return words{Headline: "A module's bucket or log on the bus is filling up",
|
|
||||||
Needs: "decide whether to raise its cap or have the module keep less.",
|
|
||||||
Explanation: "A bucket or log a module keeps on the bus holds three quarters of its cap or more. When it " +
|
|
||||||
"is full, the bus refuses what the module writes there.",
|
|
||||||
Resolved: "It has room again"}
|
|
||||||
}),
|
|
||||||
kindBucketOrLogUnread: worded(func(o conditions.Observation) words {
|
|
||||||
return words{Headline: "A module's bucket or log could not be read",
|
|
||||||
Explanation: "The controller could not read how full a bucket or log a module keeps on the bus is, so it " +
|
|
||||||
"cannot say whether it is filling up. It asks again at its next check.",
|
|
||||||
Resolved: "It can be read again"}
|
|
||||||
}),
|
|
||||||
"stream-wrong": worded(func(o conditions.Observation) words {
|
"stream-wrong": worded(func(o conditions.Observation) words {
|
||||||
return words{Headline: "Part of the bus's storage is wrong",
|
return words{Headline: "Part of the bus's storage is wrong",
|
||||||
Needs: "check the machine the bus runs on; the details say what is missing.",
|
Needs: "check the machine the bus runs on; the details say what is missing.",
|
||||||
@@ -849,9 +818,6 @@ func stalledWords(l stalledLine, o conditions.Observation) (headline, explanatio
|
|||||||
"to check it: the forge never announced it. It cannot merge until it is checked.", pull, long),
|
"to check it: the forge never announced it. It cannot merge until it is checked.", pull, long),
|
||||||
fmt.Sprintf("%s has a merge check now, or is closed", pull), needs, nil
|
fmt.Sprintf("%s has a merge check now, or is closed", pull), needs, nil
|
||||||
}
|
}
|
||||||
if l.State == "unanswered" {
|
|
||||||
return unansweredWords(l, o)
|
|
||||||
}
|
|
||||||
held := l.State
|
held := l.State
|
||||||
if held == "" {
|
if held == "" {
|
||||||
held = "held"
|
held = "held"
|
||||||
@@ -881,74 +847,6 @@ func stalledWords(l stalledLine, o conditions.Observation) (headline, explanatio
|
|||||||
fmt.Sprintf("Delivery of %s is no longer %s", name, held), needs, actions
|
fmt.Sprintf("Delivery of %s is no longer %s", name, held), needs, actions
|
||||||
}
|
}
|
||||||
|
|
||||||
// unansweredWords are the plain words of a pull request's head whose merge check was asked and has not answered
|
|
||||||
// within its bound (novox/hq issue 438): mesh-delivery holds no delivery of it, so there is nothing to stop, release
|
|
||||||
// or close, and no action is offered. The operator's acts are a new commit, which asks the check again, or a look
|
|
||||||
// at the build queue, where a check that never ran may still wait.
|
|
||||||
func unansweredWords(l stalledLine, o conditions.Observation) (headline, explanation, resolved, needs string,
|
|
||||||
actions []conditions.Action) {
|
|
||||||
repository, _, _ := strings.Cut(l.ID, "@")
|
|
||||||
pull := "A pull request of " + repoName(repository)
|
|
||||||
if l.Number > 0 {
|
|
||||||
pull = fmt.Sprintf("Pull request %s #%d", repoName(repository), l.Number)
|
|
||||||
}
|
|
||||||
long, limit := "for too long", ""
|
|
||||||
if d, err := time.ParseDuration(l.For); err == nil {
|
|
||||||
long = "for " + humanDuration(d)
|
|
||||||
}
|
|
||||||
if d, err := time.ParseDuration(l.Bound); err == nil {
|
|
||||||
limit = ", past its limit of " + humanDuration(d)
|
|
||||||
}
|
|
||||||
which := "Its merge check"
|
|
||||||
if said := uncheckedWaitWords(l); len(said) > 0 {
|
|
||||||
which = "Its merge check (" + strings.Join(said, ", ") + ")"
|
|
||||||
}
|
|
||||||
if o.Resolver == conditions.ResolverOperator {
|
|
||||||
needs = "push a new commit to its branch, which asks the check again, or see whether its check still waits " +
|
|
||||||
"in the build queue: mesh-controller.queue."
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("%s's merge check has not answered %s", pull, long),
|
|
||||||
fmt.Sprintf("%s is open on a branch that requires the merge check. %s was asked and has not answered %s%s. "+
|
|
||||||
"It cannot merge until its check answers.", pull, which, long, limit),
|
|
||||||
fmt.Sprintf("%s has an answer from its merge check, or is closed", pull), needs, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// uncheckedWaitWords are the merge checks an unanswered line waits on, as the operator reads them: from the checks
|
|
||||||
// given as data, each time in the controller's local zone, as every other time in a message — "mesh/merge-gate
|
|
||||||
// pending since 02:11" — never the UTC time inside mesh-delivery's finished words, which cannot be said again in
|
|
||||||
// another zone (novox/hq issue 443). A line from a mesh-delivery that gives no such data is said by its words.
|
|
||||||
func uncheckedWaitWords(l stalledLine) []string {
|
|
||||||
if len(l.Checks) == 0 {
|
|
||||||
return l.Waiting
|
|
||||||
}
|
|
||||||
out := make([]string, 0, len(l.Checks))
|
|
||||||
for _, c := range l.Checks {
|
|
||||||
switch c.State {
|
|
||||||
case "never-set":
|
|
||||||
out = append(out, c.Context+" never set")
|
|
||||||
case "pending":
|
|
||||||
out = append(out, c.Context+" pending"+sinceWords(c.Since))
|
|
||||||
default:
|
|
||||||
out = append(out, c.Context+" not answered")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// sinceWords is " since 02:11" in the controller's local zone, with its day when that is not today ("since 23:05 on
|
|
||||||
// 9 Oct"), so the time stays absolute; a time not given, or not readable, is said so and never made up.
|
|
||||||
func sinceWords(since string) string {
|
|
||||||
at, err := time.Parse(time.RFC3339, since)
|
|
||||||
if err != nil {
|
|
||||||
return ", since a time the forge did not say"
|
|
||||||
}
|
|
||||||
local, today := at.In(wordsZone()), wordsNow().In(wordsZone())
|
|
||||||
if local.YearDay() == today.YearDay() && local.Year() == today.Year() {
|
|
||||||
return " since " + local.Format("15:04")
|
|
||||||
}
|
|
||||||
return " since " + local.Format("15:04 on 2 Jan")
|
|
||||||
}
|
|
||||||
|
|
||||||
// causeWords is a hand-act's cause as a person says it.
|
// causeWords is a hand-act's cause as a person says it.
|
||||||
func causeWords(cause string) string {
|
func causeWords(cause string) string {
|
||||||
return strings.NewReplacer(".", " ", "_", " ").Replace(cause)
|
return strings.NewReplacer(".", " ", "_", " ").Replace(cause)
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -348,107 +347,3 @@ func TestAnUnannouncedPullRequestSaysToPushANewCommit(t *testing.T) {
|
|||||||
t.Fatalf("it reads %q / %q", o.Headline, o.Explanation)
|
t.Fatalf("it reads %q / %q", o.Headline, o.Explanation)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// **A pull request whose merge check never answered says which check, since when, and what to do** (novox/hq issue
|
|
||||||
// 438): mesh-delivery says one as a stalled line in state `unanswered`, the line below exactly as its test makes it.
|
|
||||||
// No delivery of the head is held, so there is nothing to stop, release or close: no action is offered, and the
|
|
||||||
// operator's acts are a new commit, which asks the check again, or a look at the build queue.
|
|
||||||
func TestAnUnansweredMergeCheckSaysWhichCheckAndWhatToDo(t *testing.T) {
|
|
||||||
var l stalledLine
|
|
||||||
if err := json.Unmarshal([]byte(`{"id":"novox/mesh-controller@c11222026a3b","number":212,"state":"unanswered",`+
|
|
||||||
`"waiting":["mesh/merge-gate pending since 2026-10-11T00:11:39Z","mesh/repo-check never set"],"for":"1h1m0s",`+
|
|
||||||
`"bound":"1h0m0s","h2":"none: no delivery of it is held here, so there is none to close — the operator's",`+
|
|
||||||
`"says":"novox/mesh-controller#212 is open on main, which requires the merge check, and its head's check `+
|
|
||||||
`started and never answered: mesh/merge-gate pending since 2026-10-11T00:11:39Z, mesh/repo-check never set. `+
|
|
||||||
`No delivery of it is held here, so nothing asks it again. A new commit on its branch announces it and asks `+
|
|
||||||
`its check"}`), &l); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
obs := stalledObservations([]stalledLine{l})
|
|
||||||
if len(obs) != 1 || obs[0].Resolver != conditions.ResolverOperator {
|
|
||||||
t.Fatalf("an unanswered merge check is not the operator's: %+v", obs)
|
|
||||||
}
|
|
||||||
o := obs[0]
|
|
||||||
t.Logf("headline: %s\nexplanation: %s\nneeds: %s\nresolved: %s", o.Headline, o.Explanation, o.Needs, o.Resolved)
|
|
||||||
if len(o.Actions) != 0 {
|
|
||||||
t.Fatalf("it offers an action a head with no delivery cannot take: %+v", o.Actions)
|
|
||||||
}
|
|
||||||
if strings.Contains(o.Needs, "stop") || strings.Contains(o.Needs, "release") || !strings.Contains(o.Needs, "commit") ||
|
|
||||||
!strings.Contains(o.Needs, "in the build queue: mesh-controller.queue") || strings.Contains(o.Needs, "build seat") {
|
|
||||||
t.Fatalf("it says to %q", o.Needs)
|
|
||||||
}
|
|
||||||
if o.Headline != "Pull request mesh-controller #212's merge check has not answered for 61 minutes" {
|
|
||||||
t.Fatalf("its headline reads %q", o.Headline)
|
|
||||||
}
|
|
||||||
for _, want := range []string{"mesh/merge-gate pending since 2026-10-11T00:11:39Z", "mesh/repo-check never set",
|
|
||||||
"past its limit of 60 minutes"} {
|
|
||||||
if !strings.Contains(o.Explanation, want) {
|
|
||||||
t.Fatalf("its explanation does not say %q: %q", want, o.Explanation)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if strings.Contains(o.Explanation, "never asked") || strings.Contains(o.Explanation, "never announced") {
|
|
||||||
t.Fatalf("it says the mesh was never asked, of a head whose check started: %q", o.Explanation)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A line from a mesh-delivery that names no checks still says the check did not answer, and offers nothing.
|
|
||||||
var bare stalledLine
|
|
||||||
if err := json.Unmarshal([]byte(`{"id":"novox/mesh-controller@c11222026a3b","number":212,"state":"unanswered",`+
|
|
||||||
`"for":"1h1m0s","bound":"1h0m0s","h2":"none: no delivery of it is held here, so there is none to close — the operator's"}`),
|
|
||||||
&bare); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
o = stalledObservations([]stalledLine{bare})[0]
|
|
||||||
if len(o.Actions) != 0 || !strings.Contains(o.Explanation, "has not answered") || strings.Contains(o.Headline, "Delivery of") {
|
|
||||||
t.Fatalf("a line naming no checks reads %q / %q, actions %+v", o.Headline, o.Explanation, o.Actions)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **An unanswered merge check's time reaches the operator in their own time, never as raw UTC** (novox/hq issue
|
|
||||||
// 443): mesh-delivery says each check waited on as data — its context, its state and since when, in RFC 3339 — beside
|
|
||||||
// the finished words it gave before, and the controller words it in its local zone, as every other time in a message.
|
|
||||||
// The line is the one mesh-delivery says, as in the report of issue 443, with the checks it now carries.
|
|
||||||
func TestAnUnansweredMergeChecksTimeIsSaidInLocalTime(t *testing.T) {
|
|
||||||
// The operator's zone given through the seam, never by writing time.Local: other tests' goroutines read it, and
|
|
||||||
// the build seat runs the suite under the race detector.
|
|
||||||
wasZone, wasNow := wordsZone, wordsNow
|
|
||||||
wordsZone = func() *time.Location { return time.FixedZone("CEST", 2*60*60) }
|
|
||||||
wordsNow = func() time.Time { return time.Date(2026, 10, 11, 1, 12, 39, 0, time.UTC) }
|
|
||||||
t.Cleanup(func() { wordsZone, wordsNow = wasZone, wasNow })
|
|
||||||
|
|
||||||
var l stalledLine
|
|
||||||
if err := json.Unmarshal([]byte(`{"id":"novox/mesh-controller@c11222026a3b","number":212,"state":"unanswered",`+
|
|
||||||
`"waiting":["mesh/merge-gate pending since 2026-10-11T00:11:39Z","mesh/repo-check never set"],`+
|
|
||||||
`"checks":[{"context":"mesh/merge-gate","state":"pending","since":"2026-10-11T00:11:39Z"},`+
|
|
||||||
`{"context":"mesh/repo-check","state":"never-set"}],"for":"1h1m0s",`+
|
|
||||||
`"bound":"1h0m0s","h2":"none: no delivery of it is held here, so there is none to close — the operator's",`+
|
|
||||||
`"says":"novox/mesh-controller#212 is open on main, which requires the merge check, and its head's check `+
|
|
||||||
`started and never answered: mesh/merge-gate pending since 2026-10-11T00:11:39Z, mesh/repo-check never set. `+
|
|
||||||
`No delivery of it is held here, so nothing asks it again. A new commit on its branch announces it and asks `+
|
|
||||||
`its check"}`), &l); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
o := stalledObservations([]stalledLine{l})[0]
|
|
||||||
t.Logf("explanation: %s", o.Explanation)
|
|
||||||
for _, raw := range []string{"2026-10-11T00:11:39Z", "00:11", "UTC"} {
|
|
||||||
if strings.Contains(o.Explanation, raw) || strings.Contains(o.Headline, raw) {
|
|
||||||
t.Fatalf("the operator reads %q: %q / %q", raw, o.Headline, o.Explanation)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, want := range []string{"mesh/merge-gate pending since 02:11", "mesh/repo-check never set"} {
|
|
||||||
if !strings.Contains(o.Explanation, want) {
|
|
||||||
t.Fatalf("its explanation does not say %q: %q", want, o.Explanation)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A check pending since another day than today says which day, so the time stays absolute.
|
|
||||||
l.Checks[0].Since = "2026-10-09T21:05:00Z"
|
|
||||||
if o = stalledObservations([]stalledLine{l})[0]; !strings.Contains(o.Explanation, "mesh/merge-gate pending since 23:05 on 9 Oct") {
|
|
||||||
t.Fatalf("a check pending since an earlier day reads %q", o.Explanation)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A pending check whose time the forge did not say is said so, with no time made up.
|
|
||||||
l.Checks[0].Since = ""
|
|
||||||
if o = stalledObservations([]stalledLine{l})[0]; !strings.Contains(o.Explanation, "mesh/merge-gate pending, since a time the forge did not say") {
|
|
||||||
t.Fatalf("a pending check without its time reads %q", o.Explanation)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -420,8 +420,7 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
|||||||
out := sendable{Resources: composed.Resources, Adoption: adoption,
|
out := sendable{Resources: composed.Resources, Adoption: adoption,
|
||||||
Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
|
Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
|
||||||
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld,
|
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld,
|
||||||
unbound: with.Unbound, foreseen: composed.Foreseen, unplaced: composed.Unplaced,
|
unbound: with.Unbound, foreseen: composed.Foreseen, unplaced: composed.Unplaced}
|
||||||
modules: namedModules(plan, composed.LeftOut)}
|
|
||||||
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
|
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
|
||||||
// 0221). Read only on the send path: a question about what would be sent records nothing.
|
// 0221). Read only on the send path: a question about what would be sent records nothing.
|
||||||
if choosing == Allocating {
|
if choosing == Allocating {
|
||||||
@@ -665,33 +664,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
|||||||
needed := map[string]map[string]string{}
|
needed := map[string]map[string]string{}
|
||||||
foreseen := map[string]map[string]bool{}
|
foreseen := map[string]map[string]bool{}
|
||||||
for _, m := range plan.Modules {
|
for _, m := range plan.Modules {
|
||||||
// **A secret family is never made** (novox/hq ADR 0283): each member a person gave on this machine is
|
for name := range m.OwnSecrets {
|
||||||
// placed, and one not given is nothing — the module says it waits for it.
|
|
||||||
for _, family := range m.OwnSecrets.Families() {
|
|
||||||
members, err := inv.GivenMembers(ctx, node, m.Module, family)
|
|
||||||
if err != nil {
|
|
||||||
return catalogue.Rendering{}, inventory.Node{}, err
|
|
||||||
}
|
|
||||||
for _, g := range members {
|
|
||||||
if _, fam, ok := m.OwnSecrets.Lookup(g.Name); !ok || fam != family {
|
|
||||||
continue // a longer family's member, or a name no longer of this family
|
|
||||||
}
|
|
||||||
if !g.Current {
|
|
||||||
if choosing == Allocating {
|
|
||||||
return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf(
|
|
||||||
"%s on %s holds %q, which was given to the mesh rather than made by it, and %s has "+
|
|
||||||
"since generated a new sealing key. The mesh cannot make another; give it again",
|
|
||||||
m.Module, node, g.Name, node)
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if needed[m.Module] == nil {
|
|
||||||
needed[m.Module] = map[string]string{}
|
|
||||||
}
|
|
||||||
needed[m.Module][g.Name] = g.Sealed
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for name := range m.OwnSecrets.Plain() {
|
|
||||||
// Minted on the send path and only read on every other. Making one is an insert, and
|
// Minted on the send path and only read on every other. Making one is an insert, and
|
||||||
// a question that writes is a question that can block against the machine it is about.
|
// a question that writes is a question that can block against the machine it is about.
|
||||||
var sealed string
|
var sealed string
|
||||||
@@ -1291,11 +1264,6 @@ func planCommand(ctx context.Context, args []string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// What the machine was last told, by whom and from which assignment generation (novox/hq issue 234),
|
|
||||||
// beside what it would be told now. A record that cannot be read is said, not taken for none.
|
|
||||||
if err := writeLastSend(ctx, os.Stdout, open.inventory, args[0]); err != nil {
|
|
||||||
fmt.Printf("what %s was last sent could not be read: %v\n", args[0], err)
|
|
||||||
}
|
|
||||||
// Which modules a push would leave out, and why — said before the plan, since the plan is of
|
// Which modules a push would leave out, and why — said before the plan, since the plan is of
|
||||||
// what the machine would be told (novox/hq ADR 0163, rule 6). Judged, never composed: `plan`
|
// what the machine would be told (novox/hq ADR 0163, rule 6). Judged, never composed: `plan`
|
||||||
// without --json allocates nothing.
|
// without --json allocates nothing.
|
||||||
|
|||||||
@@ -278,13 +278,6 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
|||||||
plans = append(plans, recent...)
|
plans = append(plans, recent...)
|
||||||
// **A failed walk whose earlier merges are walked alone is not retried** (novox/hq ADR 0276): the search for
|
// **A failed walk whose earlier merges are walked alone is not retried** (novox/hq ADR 0276): the search for
|
||||||
// the merge that brought the failure answers them now, and a retried walk would name them twice.
|
// the merge that brought the failure answers them now, and a retried walk would name them twice.
|
||||||
// **A walk stopped without some merges is not retried** (novox/hq ADR 0299): the merges it left out were stopped,
|
|
||||||
// and the others are another walk's now.
|
|
||||||
if p.Delivery != nil && (p.Delivery.WalkedAgainBy != "" || len(p.Delivery.LeftOut) > 0) {
|
|
||||||
return "", fmt.Errorf("%s was stopped without %s: those merges were stopped, and %s walks the others; a "+
|
|
||||||
"newer merge, or `rebuild <module>`, builds again", p.ID, mergesLeftOutWords(p.Delivery.LeftOut),
|
|
||||||
orWords(p.Delivery.WalkedAgainBy, "nothing"))
|
|
||||||
}
|
|
||||||
if p.Delivery != nil && len(p.Delivery.Merges) > 0 {
|
if p.Delivery != nil && len(p.Delivery.Merges) > 0 {
|
||||||
kept, err := inv.MergesOf(ctx, p.ID)
|
kept, err := inv.MergesOf(ctx, p.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -295,24 +288,13 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
|||||||
"those walks answer them; a newer merge, or `rebuild <module>`, builds again", p.ID)
|
"those walks answer them; a newer merge, or `rebuild <module>`, builds again", p.ID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Settled from the build records before anything is judged (novox/hq issue 457): a build of the tier
|
|
||||||
// that failed after the plan did is as failed as the one that failed it. What toRetry says is the
|
|
||||||
// failed set every step below works from.
|
|
||||||
var failed []string
|
|
||||||
if p.Tier < len(p.Tiers) {
|
|
||||||
recorded, byID, err := recordsOfAsked(ctx, inv, &p, p.Tiers[p.Tier])
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
failed = toRetry(&p, recorded, byID)
|
|
||||||
}
|
|
||||||
if err := retryRefusal(p, plans); err != nil {
|
if err := retryRefusal(p, plans); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
if again := unjudgedAtGate(p); len(failed) == 0 && len(again) > 0 {
|
if again := unjudgedAtGate(p); len(failedIn(p)) == 0 && len(again) > 0 {
|
||||||
return retryTierWhole(ctx, open, &p, again)
|
return retryTierWhole(ctx, open, &p, again)
|
||||||
}
|
}
|
||||||
if len(failed) == 0 {
|
if len(failedIn(p)) == 0 {
|
||||||
return retryRollouts(ctx, open, &p)
|
return retryRollouts(ctx, open, &p)
|
||||||
}
|
}
|
||||||
entries, err := inv.Catalogued(ctx)
|
entries, err := inv.Catalogued(ctx)
|
||||||
@@ -323,6 +305,7 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
|||||||
for _, e := range entries {
|
for _, e := range entries {
|
||||||
byName[e.Manifest.Module] = e
|
byName[e.Manifest.Module] = e
|
||||||
}
|
}
|
||||||
|
failed := failedIn(p)
|
||||||
var asked []string
|
var asked []string
|
||||||
for _, m := range failed {
|
for _, m := range failed {
|
||||||
askModule(ctx, &p, m, byName)
|
askModule(ctx, &p, m, byName)
|
||||||
@@ -542,16 +525,3 @@ func retryTierWhole(ctx context.Context, open *stores, p *inventory.Plan, again
|
|||||||
func sendAgain(s *inventory.PlanModule) {
|
func sendAgain(s *inventory.PlanModule) {
|
||||||
s.First, s.FirstAt, s.Gate, s.GatedBy, s.Previous, s.Why = nil, nil, nil, "", "", ""
|
s.First, s.FirstAt, s.Gate, s.GatedBy, s.Previous, s.Why = nil, nil, nil, "", "", ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// toRetry is the modules a retry asks again: every one of the tier that failed, the plan's state
|
|
||||||
// settled from the build records first (novox/hq issue 457). A plan fails on the first failure in its
|
|
||||||
// tier, and an outcome arriving after that finds no open plan to answer — it is kept only in the build
|
|
||||||
// records. Read from the plan alone, a retry asked only the build that failed first, and the records
|
|
||||||
// then failed the plan again on the next: each failed build of a tier took a retry of its own. What
|
|
||||||
// still runs is left asked, and its outcome is the plan's once the retry sets it building.
|
|
||||||
func toRetry(p *inventory.Plan, recorded map[string][]inventory.Build, byID map[string]inventory.Build) []string {
|
|
||||||
if p.Tier < len(p.Tiers) {
|
|
||||||
settleFromRecords(p, p.Tiers[p.Tier], recorded, byID)
|
|
||||||
}
|
|
||||||
return failedIn(*p)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,57 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"reflect"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A retry asks every failed build of the tier, not one (novox/hq issue 457). Seen 2026-10-11: gitea
|
|
||||||
// and plex both failed in tier 0 while the registry was held still; gitea's failure failed the plan,
|
|
||||||
// and plex's, arriving after, found no open plan and was kept only in the build records. The first
|
|
||||||
// retry asked gitea alone, the records then failed the plan again on plex, and a second retry asked
|
|
||||||
// plex.
|
|
||||||
func TestARetryAsksEveryFailedBuildOfTheTier(t *testing.T) {
|
|
||||||
asked := time.Date(2026, 10, 11, 1, 29, 0, 0, time.UTC)
|
|
||||||
failedAt := asked.Add(2 * time.Minute)
|
|
||||||
p := inventory.Plan{ID: "plan-457", State: inventory.PlanFailed, Tier: 0,
|
|
||||||
Tiers: [][]string{{"gitea", "plex"}}, Note: "gitea failed to build in tier 0",
|
|
||||||
Modules: map[string]*inventory.PlanModule{
|
|
||||||
"gitea": {State: "failed", AskedAt: &asked, Build: "build-gitea", Why: "cannot reach the registry"},
|
|
||||||
"plex": {State: "asked", AskedAt: &asked, Build: "build-plex"},
|
|
||||||
}}
|
|
||||||
byID := map[string]inventory.Build{
|
|
||||||
"build-plex": {ID: "build-plex", Module: "plex", At: failedAt, Failed: "cannot reach the registry"},
|
|
||||||
}
|
|
||||||
|
|
||||||
if got := toRetry(&p, nil, byID); !reflect.DeepEqual(got, []string{"gitea", "plex"}) {
|
|
||||||
t.Fatalf("a retry of a tier where gitea and plex failed asks %v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A build of the tier still running when the plan failed is not asked again: its outcome is the plan's
|
|
||||||
// once the retry sets it building, and one that is recorded built is taken as built.
|
|
||||||
func TestARetryLeavesABuildThatRunsOrWorked(t *testing.T) {
|
|
||||||
asked := time.Date(2026, 10, 11, 1, 29, 0, 0, time.UTC)
|
|
||||||
builtAt := asked.Add(3 * time.Minute)
|
|
||||||
p := inventory.Plan{ID: "plan-457", State: inventory.PlanFailed, Tier: 0,
|
|
||||||
Tiers: [][]string{{"a", "b", "c"}},
|
|
||||||
Modules: map[string]*inventory.PlanModule{
|
|
||||||
"a": {State: "failed", AskedAt: &asked, Build: "build-a", Why: "broken"},
|
|
||||||
"b": {State: "asked", AskedAt: &asked, Build: "build-b"},
|
|
||||||
"c": {State: "asked", AskedAt: &asked, Build: "build-c"},
|
|
||||||
}}
|
|
||||||
byID := map[string]inventory.Build{"build-c": {ID: "build-c", Module: "c", At: builtAt, Commit: "c0ffee"}}
|
|
||||||
|
|
||||||
if got := toRetry(&p, nil, byID); !reflect.DeepEqual(got, []string{"a"}) {
|
|
||||||
t.Fatalf("asks %v, want a alone", got)
|
|
||||||
}
|
|
||||||
if s := p.Modules["c"]; s.State != "built" || s.Commit != "c0ffee" {
|
|
||||||
t.Errorf("c, recorded built, is %+v", s)
|
|
||||||
}
|
|
||||||
if s := p.Modules["b"]; s.State != "asked" {
|
|
||||||
t.Errorf("b, still building, is %+v", s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -78,11 +78,6 @@ func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation
|
|||||||
return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err)
|
return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err)
|
||||||
}
|
}
|
||||||
var problems, foreseen []string
|
var problems, foreseen []string
|
||||||
// Each module the next send leaves out of this machine's declaration (novox/hq issue 380), judged from this
|
|
||||||
// resolution as the send judges it: a finding, never a refusal.
|
|
||||||
if err == nil {
|
|
||||||
out = append(out, leftOutObservations(n.Name, leftOutOf(plan, settings, n.Adopted))...)
|
|
||||||
}
|
|
||||||
if err == nil && gensErr == nil {
|
if err == nil && gensErr == nil {
|
||||||
var declared sendable
|
var declared sendable
|
||||||
if declared, err = declarationWith(ctx, open, n.Name, plan, settings, gens, Foreseeing); err == nil {
|
if declared, err = declarationWith(ctx, open, n.Name, plan, settings, gens, Foreseeing); err == nil {
|
||||||
|
|||||||
@@ -37,32 +37,6 @@ type holding struct {
|
|||||||
shelf map[string]catalogue.Manifest
|
shelf map[string]catalogue.Manifest
|
||||||
// providers memoises providerFor by machine, consumer and provision.
|
// providers memoises providerFor by machine, consumer and provision.
|
||||||
providers map[string]providerLookup
|
providers map[string]providerLookup
|
||||||
// waits is what the waits of a statement are checked against, read as they are asked for (novox/hq issue 450).
|
|
||||||
waits operatorWaitFacts
|
|
||||||
}
|
|
||||||
|
|
||||||
// checked is a machine's newest statement as the controller judges it: each wait checked (ADR 0283 decision 3), so
|
|
||||||
// a wait that does not check out reads as unhealthy, as it did when the statement was judged (novox/hq issue 450).
|
|
||||||
// What is stored is what the machine said, the waits unchecked; read as stored, a provider whose wait failed its
|
|
||||||
// check seems to wait for the operator while its unhealthy condition is open.
|
|
||||||
func (h *holding) checked(machine string) []inventory.ResourceHealth {
|
|
||||||
rs := h.healths[machine].Resources
|
|
||||||
mods := waitingModules(rs)
|
|
||||||
if len(mods) == 0 {
|
|
||||||
return rs
|
|
||||||
}
|
|
||||||
if h.waits.manifests == nil {
|
|
||||||
h.waits.manifests = map[string]catalogue.Manifest{}
|
|
||||||
}
|
|
||||||
for _, module := range mods {
|
|
||||||
if _, has := h.waits.manifests[module]; !has {
|
|
||||||
if m, ok := h.manifestOf(module); ok {
|
|
||||||
h.waits.manifests[module] = m
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
readWaitFacts(h.ctx, h.inv, machine, mods, nil, &h.waits)
|
|
||||||
return checkWaiting(machine, rs, h.waits)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type providerLookup struct {
|
type providerLookup struct {
|
||||||
@@ -79,15 +53,6 @@ func readHolding(ctx context.Context, inv *inventory.Inventory, open []condition
|
|||||||
return &holding{ctx: ctx, inv: inv, healths: healths, open: open, providers: map[string]providerLookup{}}, nil
|
return &holding{ctx: ctx, inv: inv, healths: healths, open: open, providers: map[string]providerLookup{}}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// readHoldingFor is how a judging reads its holding: nothing without a store. A variable so a test can hand a
|
|
||||||
// judging the record it holds under (novox/hq issue 405).
|
|
||||||
var readHoldingFor = func(ctx context.Context, inv *inventory.Inventory, open []conditions.Condition) (*holding, error) {
|
|
||||||
if inv == nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return readHolding(ctx, inv, open)
|
|
||||||
}
|
|
||||||
|
|
||||||
// heldFinding says a resource's state is a finding of its declared check that names a provision: what
|
// heldFinding says a resource's state is a finding of its declared check that names a provision: what
|
||||||
// may be held. Down and restarting are liveness, the resource's own.
|
// may be held. Down and restarting are liveness, the resource's own.
|
||||||
func heldFinding(r inventory.ResourceHealth) bool {
|
func heldFinding(r inventory.ResourceHealth) bool {
|
||||||
@@ -141,137 +106,42 @@ func (h *holding) lookUpProvider(machine, consumer, provision string) (catalogue
|
|||||||
return catalogue.Chosen{}, false
|
return catalogue.Chosen{}, false
|
||||||
}
|
}
|
||||||
|
|
||||||
// providerState is how a provider stands on the record: unhealthy when its unhealthy condition is open or its
|
// unhealthy says a provider is unhealthy on the record: its condition is open, or its machine's newest
|
||||||
// machine's newest statement says a resource of it is unhealthy; else waiting for the operator when that statement
|
// statement says a resource of it is unhealthy.
|
||||||
// says a resource of it waits, with the waits it names, or its needs-operator condition is open (waits then
|
func (h *holding) unhealthy(p catalogue.Chosen) bool {
|
||||||
// unknown); else healthy.
|
key := moduleUnhealthyKey(p.Module, p.Node)
|
||||||
func (h *holding) providerState(p catalogue.Chosen) (unhealthy, waiting bool, waits []inventory.Wait) {
|
|
||||||
for _, c := range h.open {
|
for _, c := range h.open {
|
||||||
if c.Key == moduleUnhealthyKey(p.Module, p.Node) {
|
if c.Key == key {
|
||||||
return true, false, nil
|
return true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, r := range h.checked(p.Node) {
|
for _, r := range h.healths[p.Node].Resources {
|
||||||
if r.Module != p.Module {
|
if r.Module == p.Module && r.State == link.StateUnhealthy {
|
||||||
continue
|
return true
|
||||||
}
|
|
||||||
switch r.State {
|
|
||||||
case link.StateUnhealthy:
|
|
||||||
return true, false, nil
|
|
||||||
case link.StateWaiting:
|
|
||||||
waiting = true
|
|
||||||
waits = append(waits, r.Waits...)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !waiting {
|
return false
|
||||||
for _, c := range h.open {
|
|
||||||
waiting = waiting || c.Key == needsOperatorKey(p.Module, p.Node)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false, waiting, waits
|
|
||||||
}
|
|
||||||
|
|
||||||
// manifestOf is a module's manifest from the catalogue, read once; false when it cannot be read.
|
|
||||||
func (h *holding) manifestOf(module string) (catalogue.Manifest, bool) {
|
|
||||||
if h.shelf == nil && h.inv != nil {
|
|
||||||
shelf, err := h.inv.Catalogue(h.ctx)
|
|
||||||
if err != nil {
|
|
||||||
return catalogue.Manifest{}, false
|
|
||||||
}
|
|
||||||
h.shelf = shelf
|
|
||||||
}
|
|
||||||
m, ok := h.shelf[module]
|
|
||||||
return m, ok
|
|
||||||
}
|
|
||||||
|
|
||||||
// covering is the waits of a provider that cover a provision it gives (novox/hq issue 405): a module that waits
|
|
||||||
// says nothing else of it is wrong and names each part that waits (ADR 0283 decision 1), so only a consumer of
|
|
||||||
// the waiting part waits on it. A wait covers a provision when its part is that provision, or the secret it waits
|
|
||||||
// for is the provision's shared credential (ADR 0158). Nothing when no wait can be matched: a consumer failing
|
|
||||||
// then is not held, since holding it would hide a fault that may be its own.
|
|
||||||
func (h *holding) covering(p catalogue.Chosen, provision string, waits []inventory.Wait) []inventory.Wait {
|
|
||||||
credential := ""
|
|
||||||
if m, ok := h.manifestOf(p.Module); ok {
|
|
||||||
credential, _ = m.SharedCredentialOf(provision)
|
|
||||||
}
|
|
||||||
var out []inventory.Wait
|
|
||||||
for _, w := range waits {
|
|
||||||
if w.Part == provision || (w.Secret != "" && w.Secret == credential) {
|
|
||||||
out = append(out, w)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// heldUnderProvider is the provider a consumer's findings are held under, and — when that provider only waits for the
|
|
||||||
// operator, for the part the consumer needs — the waits that hold it.
|
|
||||||
type heldUnderProvider struct {
|
|
||||||
provider catalogue.Chosen
|
|
||||||
// waits is set when every finding held waits on a provider that only waits for the operator: the consumer's
|
|
||||||
// gate then reads as ADR 0254's waits for a person, a pass with the wait carried (ADR 0283 decision 4).
|
|
||||||
waits []inventory.Wait
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// heldUnder is the provider a consumer's unhealthy resources wait on: when every one of them is a finding
|
// heldUnder is the provider a consumer's unhealthy resources wait on: when every one of them is a finding
|
||||||
// of a check naming a provision whose provider for this consumer is unhealthy on the record, or waits for the
|
// of a check naming a provision whose provider for this consumer is unhealthy on the record. False when any
|
||||||
// operator for the part that gives it (novox/hq issue 405). False when any is the consumer's own.
|
// is the consumer's own.
|
||||||
func (h *holding) heldUnder(machine, module string, rs []inventory.ResourceHealth) (catalogue.Chosen, bool) {
|
func (h *holding) heldUnder(machine, module string, rs []inventory.ResourceHealth) (catalogue.Chosen, bool) {
|
||||||
by, held := h.heldWith(machine, module, rs)
|
var on catalogue.Chosen
|
||||||
return by.provider, held
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *holding) heldWith(machine, module string, rs []inventory.ResourceHealth) (heldUnderProvider, bool) {
|
|
||||||
var on heldUnderProvider
|
|
||||||
onlyWaits := true
|
|
||||||
for _, r := range rs {
|
for _, r := range rs {
|
||||||
if r.State != link.StateUnhealthy {
|
if r.State != link.StateUnhealthy {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if !heldFinding(r) {
|
if !heldFinding(r) {
|
||||||
return heldUnderProvider{}, false
|
|
||||||
}
|
|
||||||
p, ok := h.providerFor(machine, module, r.Needs)
|
|
||||||
if !ok || (p.Node == machine && p.Module == module) {
|
|
||||||
return heldUnderProvider{}, false
|
|
||||||
}
|
|
||||||
unhealthy, waiting, waits := h.providerState(p)
|
|
||||||
switch {
|
|
||||||
case unhealthy:
|
|
||||||
onlyWaits = false
|
|
||||||
case waiting:
|
|
||||||
covered := h.covering(p, r.Needs, waits)
|
|
||||||
if len(covered) == 0 {
|
|
||||||
return heldUnderProvider{}, false
|
|
||||||
}
|
|
||||||
on.waits = append(on.waits, covered...)
|
|
||||||
default:
|
|
||||||
return heldUnderProvider{}, false
|
|
||||||
}
|
|
||||||
on.provider = p
|
|
||||||
}
|
|
||||||
if !onlyWaits {
|
|
||||||
on.waits = nil
|
|
||||||
}
|
|
||||||
return on, on.provider.Module != ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// waitingUncovered is a provider of a consumer's failing finding that waits for the operator for a part the
|
|
||||||
// finding cannot be matched to (novox/hq issue 405): the consumer is raised on its own, and its condition says
|
|
||||||
// the provider waits, so neither is hidden.
|
|
||||||
func (h *holding) waitingUncovered(machine, module string, rs []inventory.ResourceHealth) (catalogue.Chosen, bool) {
|
|
||||||
for _, r := range rs {
|
|
||||||
if r.State != link.StateUnhealthy || !heldFinding(r) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
p, ok := h.providerFor(machine, module, r.Needs)
|
|
||||||
if !ok || (p.Node == machine && p.Module == module) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if unhealthy, waiting, waits := h.providerState(p); !unhealthy && waiting && len(h.covering(p, r.Needs, waits)) == 0 {
|
|
||||||
return p, true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return catalogue.Chosen{}, false
|
return catalogue.Chosen{}, false
|
||||||
|
}
|
||||||
|
p, ok := h.providerFor(machine, module, r.Needs)
|
||||||
|
if !ok || (p.Node == machine && p.Module == module) || !h.unhealthy(p) {
|
||||||
|
return catalogue.Chosen{}, false
|
||||||
|
}
|
||||||
|
on = p
|
||||||
|
}
|
||||||
|
return on, on.Module != ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// waitersOn is every consumer held under a provider, as "<module> on <machine>", sorted.
|
// waitersOn is every consumer held under a provider, as "<module> on <machine>", sorted.
|
||||||
@@ -295,8 +165,8 @@ func (h *holding) waitersOn(p catalogue.Chosen) []string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// heldModules is, for one machine's statement, each module whose finding is held, with the provider.
|
// heldModules is, for one machine's statement, each module whose finding is held, with the provider.
|
||||||
func (h *holding) heldModules(machine string) map[string]heldUnderProvider {
|
func (h *holding) heldModules(machine string) map[string]catalogue.Chosen {
|
||||||
out := map[string]heldUnderProvider{}
|
out := map[string]catalogue.Chosen{}
|
||||||
byModule := map[string][]inventory.ResourceHealth{}
|
byModule := map[string][]inventory.ResourceHealth{}
|
||||||
for _, r := range h.healths[machine].Resources {
|
for _, r := range h.healths[machine].Resources {
|
||||||
if r.Module != "" && r.State == link.StateUnhealthy {
|
if r.Module != "" && r.State == link.StateUnhealthy {
|
||||||
@@ -304,7 +174,7 @@ func (h *holding) heldModules(machine string) map[string]heldUnderProvider {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
for module, rs := range byModule {
|
for module, rs := range byModule {
|
||||||
if on, held := h.heldWith(machine, module, rs); held {
|
if on, held := h.heldUnder(machine, module, rs); held {
|
||||||
out[module] = on
|
out[module] = on
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -357,15 +357,9 @@ func declare(ctx context.Context, args []string) error {
|
|||||||
// The epoch it carried, if a person wrote one in, is what the machine heard.
|
// The epoch it carried, if a person wrote one in, is what the machine heard.
|
||||||
var carried struct {
|
var carried struct {
|
||||||
Epoch uint64 `json:"epoch"`
|
Epoch uint64 `json:"epoch"`
|
||||||
Sequence int64 `json:"sequence"`
|
|
||||||
Generation int64 `json:"generation"`
|
|
||||||
}
|
}
|
||||||
_ = json.Unmarshal(raw, &carried)
|
_ = json.Unmarshal(raw, &carried)
|
||||||
// And recorded as every send is (novox/hq issue 234): by hand, from what it carried; the modules it
|
if _, err := recordSent(ctx, inv, node, raw, nil, carried.Epoch); err != nil {
|
||||||
// named are not known, since the mesh did not compose it.
|
|
||||||
if _, err := recordSent(ctx, inv, node, raw, nil, carried.Epoch, sentRecord{sequence: carried.Sequence,
|
|
||||||
epoch: carried.Epoch, generation: carried.Generation, toldGeneration: carried.Generation,
|
|
||||||
sender: senderOf(callerOf(ctx)) + ", a declaration sent by hand"}); err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw))
|
fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw))
|
||||||
@@ -804,7 +798,7 @@ func composeEach(names []string, allot func(node string) (order, error),
|
|||||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
numbered.stamp(&declared)
|
declared.Sequence, declared.Epoch = numbered.sequence, numbered.epoch
|
||||||
if len(declared.Resources) == 0 {
|
if len(declared.Resources) == 0 {
|
||||||
// Sent, not skipped (novox/hq issue 127). A node whose declaration composes to
|
// Sent, not skipped (novox/hq issue 127). A node whose declaration composes to
|
||||||
// nothing may have HELD something before — the broker opening a placement gave it,
|
// nothing may have HELD something before — the broker opening a placement gave it,
|
||||||
@@ -1022,8 +1016,7 @@ func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (stri
|
|||||||
}
|
}
|
||||||
// After it is away, not before. A digest recorded for something that failed to send would make
|
// After it is away, not before. A digest recorded for something that failed to send would make
|
||||||
// the machine look current for a declaration it never received.
|
// the machine look current for a declaration it never received.
|
||||||
digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds, s.declared.Epoch,
|
digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds, s.declared.Epoch)
|
||||||
sentRecordOf(ctx, s.declared))
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
@@ -1235,18 +1228,7 @@ func sendToEach(ctx context.Context, open *stores, names []string) ([]string, er
|
|||||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
numbered.stamp(&declared)
|
declared.Sequence, declared.Epoch = numbered.sequence, numbered.epoch
|
||||||
// **The grants step sends the user list and nothing else** (novox/hq issue 490): judged on this very
|
|
||||||
// declaration, the one sent, so nothing composed between a check and the send can slip through.
|
|
||||||
if everyBuildKept(ctx, name) {
|
|
||||||
other, err := grantsOnlyIn(ctx, open, name, plan, declared)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if other != "" {
|
|
||||||
return nil, fmt.Errorf("%w: %s", errNotGrantsOnly, other)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
reportLeftOut(name, declared)
|
reportLeftOut(name, declared)
|
||||||
sending = append(sending, readyNode{name, declared})
|
sending = append(sending, readyNode{name, declared})
|
||||||
}
|
}
|
||||||
@@ -1311,15 +1293,6 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, se
|
|||||||
if _, err := broker.RaiseBuckets(broker.OnConn(bus.Conn), buckets); err != nil {
|
if _, err := broker.RaiseBuckets(broker.OnConn(bus.Conn), buckets); err != nil {
|
||||||
return fmt.Errorf("the modules' state could not be asserted on the bus: %w", err)
|
return fmt.Errorf("the modules' state could not be asserted on the bus: %w", err)
|
||||||
}
|
}
|
||||||
// **And every declared log, for the same reason** (novox/hq ADR 0297 §2): a membership names its
|
|
||||||
// logs, and a module whose log does not exist fails its first append.
|
|
||||||
logs, err := open.inventory.DeclaredLogs(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("the modules' logs could not be read, so no log was asserted: %w", err)
|
|
||||||
}
|
|
||||||
if _, err := broker.RaiseLogs(broker.OnConn(bus.Conn), logs); err != nil {
|
|
||||||
return fmt.Errorf("the modules' logs could not be asserted on the bus: %w", err)
|
|
||||||
}
|
|
||||||
// Every membership is tried, and the first failure named once.
|
// Every membership is tried, and the first failure named once.
|
||||||
issued := 0
|
issued := 0
|
||||||
refused := map[string]error{}
|
refused := map[string]error{}
|
||||||
@@ -1420,11 +1393,6 @@ func wouldSendFrom(ctx context.Context, open *stores,
|
|||||||
if declared.Epoch, err = open.inventory.SentEpoch(ctx, n.ID); err != nil {
|
if declared.Epoch, err = open.inventory.SentEpoch(ctx, n.ID); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
// And the generation it was last sent, for the same reason (novox/hq issue 234): an assignment
|
|
||||||
// elsewhere in the mesh is not a change of this machine.
|
|
||||||
if declared.Generation, err = open.inventory.SentGeneration(ctx, n.ID); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
body, err := declared.Body()
|
body, err := declared.Body()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -1503,28 +1471,13 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string)
|
|||||||
fmt.Printf("the bus holds state nothing declares any more, kept because it is data: %s — "+
|
fmt.Printf("the bus holds state nothing declares any more, kept because it is data: %s — "+
|
||||||
"removing it is a person's act\n", strings.Join(undeclared, ", "))
|
"removing it is a person's act\n", strings.Join(undeclared, ", "))
|
||||||
}
|
}
|
||||||
// Every module's log (novox/hq ADR 0297), from the catalogue, as its buckets: one that nothing
|
|
||||||
// declares any more is said and kept — a log is a module's record, and no path of the mesh removes it.
|
|
||||||
logs, err := inv.DeclaredLogs(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
unlogged, err := broker.RaiseLogs(js, logs)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if len(unlogged) > 0 {
|
|
||||||
fmt.Printf("the bus holds logs nothing declares any more, kept because they are data: %s — "+
|
|
||||||
"removing one is a person's act\n", strings.Join(unlogged, ", "))
|
|
||||||
}
|
|
||||||
// And how every module hears what it consumes: asserted with the rest above, counted here.
|
// And how every module hears what it consumes: asserted with the rest above, counted here.
|
||||||
hearing, err := moduleConsumerCount(ctx, inv)
|
hearing, err := moduleConsumerCount(ctx, inv)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
fmt.Printf("the bus at %s has its streams, %d machine(s) can hear a declaration, %d module(s) "+
|
fmt.Printf("the bus at %s has its streams, %d machine(s) can hear a declaration, %d module(s) "+
|
||||||
"can hear what they consume, %d bucket(s) of state and %d log(s)\n", broker.BareAddress(address), len(names), hearing,
|
"can hear what they consume, and %d bucket(s) of state\n", broker.BareAddress(address), len(names), hearing, len(buckets))
|
||||||
len(buckets), len(logs))
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1578,12 +1531,6 @@ var epochForActs = func(ctx context.Context) (uint64, error) { return theLease.e
|
|||||||
type order struct {
|
type order struct {
|
||||||
sequence int64
|
sequence int64
|
||||||
epoch uint64
|
epoch uint64
|
||||||
// generation is the assignment generation read before the composition, and readsGeneration whether the
|
|
||||||
// machine's node-engine said it reads one (novox/hq issue 234); acting is the lease epoch the sender acts
|
|
||||||
// under, whether or not the machine is sent it — both kept for the record of the send.
|
|
||||||
generation int64
|
|
||||||
readsGeneration bool
|
|
||||||
acting uint64
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// allot takes the next sequence for a machine — the number its next declaration carries — under the
|
// allot takes the next sequence for a machine — the number its next declaration carries — under the
|
||||||
@@ -1597,7 +1544,6 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, e
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return order{}, err
|
return order{}, err
|
||||||
}
|
}
|
||||||
acting := epoch
|
|
||||||
if epoch > 0 {
|
if epoch > 0 {
|
||||||
reads, err := inv.ReadsEpoch(ctx, record.ID)
|
reads, err := inv.ReadsEpoch(ctx, record.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1607,24 +1553,11 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, e
|
|||||||
epoch = 0
|
epoch = 0
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// **The generation is read here, before the composition reads a single assignment** (novox/hq issue
|
|
||||||
// 234). A generation only grows, so one read before is never newer than the view composed after it:
|
|
||||||
// the declaration may claim a generation older than its content, never newer — and a claim newer than
|
|
||||||
// the content is exactly the stale send the machine must be able to refuse.
|
|
||||||
generation, err := inv.AssignmentGeneration(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return order{}, err
|
|
||||||
}
|
|
||||||
readsGeneration, err := inv.ReadsGeneration(ctx, record.ID)
|
|
||||||
if err != nil {
|
|
||||||
return order{}, err
|
|
||||||
}
|
|
||||||
seq, err := inv.NextSequence(ctx, record.ID)
|
seq, err := inv.NextSequence(ctx, record.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return order{}, err
|
return order{}, err
|
||||||
}
|
}
|
||||||
return order{sequence: seq, epoch: epoch, generation: generation, readsGeneration: readsGeneration,
|
return order{sequence: seq, epoch: epoch}, nil
|
||||||
acting: acting}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// recordSent writes down what a machine was just sent, and returns the digest.
|
// recordSent writes down what a machine was just sent, and returns the digest.
|
||||||
@@ -1639,7 +1572,7 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, e
|
|||||||
// And the build of each module it carried (novox/hq issue 259, ADR 0221), nil when that is not known:
|
// And the build of each module it carried (novox/hq issue 259, ADR 0221), nil when that is not known:
|
||||||
// what tells a machine held back by a policy or a plan from one a push left behind.
|
// what tells a machine held back by a policy or a plan from one a push left behind.
|
||||||
func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte,
|
func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte,
|
||||||
builds map[string]string, epoch uint64, sent sentRecord) (string, error) {
|
builds map[string]string, epoch uint64) (string, error) {
|
||||||
kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
record, err := inv.NodeByName(kept, node)
|
record, err := inv.NodeByName(kept, node)
|
||||||
@@ -1647,21 +1580,7 @@ func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body
|
|||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
digest := digestOf(body)
|
digest := digestOf(body)
|
||||||
// The digest and the generation it carried are written in one transaction (novox/hq issue 234), so the
|
if err := inv.RecordSentUnder(kept, record.ID, digest, builds, epoch); err != nil {
|
||||||
// would-send is never stamped with a generation the machine was not sent; and the send itself, who sent
|
|
||||||
// it and from which generation, beside them. A record of the send that cannot be written does not make a
|
|
||||||
// send that is away look failed: it is said, loudly, and the machine's own record stands.
|
|
||||||
err = inv.RecordSentWith(kept, record.ID, digest, builds, epoch, sent.toldGeneration, inventory.Send{
|
|
||||||
Sequence: sent.sequence, Epoch: int64(sent.epoch), Sender: sent.sender, Generation: sent.generation,
|
|
||||||
Digest: digest, Modules: sent.modules})
|
|
||||||
var unrecorded *inventory.SendNotRecordedError
|
|
||||||
switch {
|
|
||||||
case errors.As(err, &unrecorded):
|
|
||||||
fmt.Fprintf(os.Stderr, "mesh-controller: SEND NOT RECORDED: %s was sent declaration %s (sequence %d), and who "+
|
|
||||||
"sent it and from which generation could not be written down, so a refusal of it will name no sender "+
|
|
||||||
"(novox/hq issue 234): %v\n", node, short(digest), sent.sequence, unrecorded.Err)
|
|
||||||
fmt.Printf("%s: sent, and the record of who sent it could NOT be written: %v\n", node, unrecorded.Err)
|
|
||||||
case err != nil:
|
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
// And what it was, summarised, so the next push can be compared with it (novox/hq ADR 0217).
|
// And what it was, summarised, so the next push can be compared with it (novox/hq ADR 0217).
|
||||||
|
|||||||
@@ -69,9 +69,6 @@ type meshStatus struct {
|
|||||||
// **A document without this said an outage was a well mesh.** Read from what each machine
|
// **A document without this said an outage was a well mesh.** Read from what each machine
|
||||||
// reported, so it is the machine's account and not the mesh's take-time listing.
|
// reported, so it is the machine's account and not the mesh's take-time listing.
|
||||||
Untaken []machineUntaken `json:"untaken,omitempty"`
|
Untaken []machineUntaken `json:"untaken,omitempty"`
|
||||||
// NotApplied is every module assigned to a machine and left out of its composition, with why (novox/hq
|
|
||||||
// issue 380). Absent when every module composes.
|
|
||||||
NotApplied []machineNotApplied `json:"not-applied,omitempty"`
|
|
||||||
// Filtered is every converged machine that is not filtered by the mesh alone (novox/hq ADR
|
// Filtered is every converged machine that is not filtered by the mesh alone (novox/hq ADR
|
||||||
// 0168), one entry per rule set the mesh did not write — the found firewall in force again,
|
// 0168), one entry per rule set the mesh did not write — the found firewall in force again,
|
||||||
// or a chain nobody speaks for. Absent when every converged machine is filtered by the mesh
|
// or a chain nobody speaks for. Absent when every converged machine is filtered by the mesh
|
||||||
@@ -109,33 +106,6 @@ type meshStatus struct {
|
|||||||
// why they could not be read.
|
// why they could not be read.
|
||||||
Pending []pendingStatus `json:"pending,omitempty"`
|
Pending []pendingStatus `json:"pending,omitempty"`
|
||||||
PendingUnread string `json:"pendingUnread,omitempty"`
|
PendingUnread string `json:"pendingUnread,omitempty"`
|
||||||
// LastSent is every machine's last recorded send — who sent it, under which lease epoch, from which
|
|
||||||
// assignment generation, naming which modules — and what the machine answered of it (novox/hq issue 485).
|
|
||||||
// A machine the mesh has recorded no send to is absent. LastSentUnread says why it could not be read.
|
|
||||||
LastSent []machineLastSent `json:"lastSent"`
|
|
||||||
LastSentUnread string `json:"lastSentUnread,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// machineLastSent is one machine's last send as status says it.
|
|
||||||
type machineLastSent struct {
|
|
||||||
Node string `json:"node"`
|
|
||||||
Sequence int64 `json:"sequence"`
|
|
||||||
SentAt time.Time `json:"sentAt"`
|
|
||||||
Sender string `json:"sender"`
|
|
||||||
Epoch int64 `json:"epoch"`
|
|
||||||
Generation int64 `json:"generation"`
|
|
||||||
Digest string `json:"digest"`
|
|
||||||
Modules []string `json:"modules"`
|
|
||||||
// Answer is what the machine said of it: applied, failed or refused, as its report has it; empty when it
|
|
||||||
// has not reported on it. Refused is its words when it refused it whole, Failed how many resources failed.
|
|
||||||
Answer string `json:"answer"`
|
|
||||||
Refused string `json:"refused,omitempty"`
|
|
||||||
Failed int `json:"failed,omitempty"`
|
|
||||||
AnsweredAt *time.Time `json:"answeredAt,omitempty"`
|
|
||||||
// RefusedForGeneration is when the machine refused it for coming from an older assignment generation than
|
|
||||||
// it applied, and AppliedGeneration that generation; absent when it did not.
|
|
||||||
RefusedForGeneration *time.Time `json:"refusedForGeneration,omitempty"`
|
|
||||||
AppliedGeneration int64 `json:"appliedGeneration,omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// pendingStatus is one pending assignment as status says it.
|
// pendingStatus is one pending assignment as status says it.
|
||||||
@@ -281,7 +251,6 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
out.Unheld = asked.unheld
|
out.Unheld = asked.unheld
|
||||||
out.NotApplied = notApplied(asked.leftOut)
|
|
||||||
out.HandActsThisWeek, out.HandActsUnread = asked.handActs, asked.handActsUnread
|
out.HandActsThisWeek, out.HandActsUnread = asked.handActs, asked.handActsUnread
|
||||||
out.HealsThisWeek, out.HealsUnread = asked.heals, asked.healsUnread
|
out.HealsThisWeek, out.HealsUnread = asked.heals, asked.healsUnread
|
||||||
// In brief, as `conditions` lists them: status leads with every open condition, and their whole
|
// In brief, as `conditions` lists them: status leads with every open condition, and their whole
|
||||||
@@ -294,7 +263,6 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
|||||||
out.Pending = append(out.Pending, pendingStatus{Node: p.Node, Module: p.Module, State: p.State,
|
out.Pending = append(out.Pending, pendingStatus{Node: p.Node, Module: p.Module, State: p.State,
|
||||||
Build: p.Build, Repository: p.Repository, Path: p.Path, Since: p.Since, Settled: p.Settled, Note: p.Note})
|
Build: p.Build, Repository: p.Repository, Path: p.Path, Since: p.Since, Settled: p.Settled, Note: p.Note})
|
||||||
}
|
}
|
||||||
out.LastSent, out.LastSentUnread = lastSentStatus(asked.lastSent), asked.lastSentUnread
|
|
||||||
for name := range asked.refused {
|
for name := range asked.refused {
|
||||||
out.Unresolved = append(out.Unresolved, machineUnresolved{
|
out.Unresolved = append(out.Unresolved, machineUnresolved{
|
||||||
Node: name, Problem: asked.refused[name]})
|
Node: name, Problem: asked.refused[name]})
|
||||||
@@ -351,20 +319,3 @@ func say(value any) error {
|
|||||||
fmt.Println(string(body))
|
fmt.Println(string(body))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// lastSentStatus is every machine's last send as status --json says it, by machine name (novox/hq issue 485).
|
|
||||||
func lastSentStatus(sends map[string]inventory.Send) []machineLastSent {
|
|
||||||
out := []machineLastSent{}
|
|
||||||
for _, s := range sends {
|
|
||||||
modules := s.Modules
|
|
||||||
if modules == nil {
|
|
||||||
modules = []string{}
|
|
||||||
}
|
|
||||||
out = append(out, machineLastSent{Node: s.NodeName, Sequence: s.Sequence, SentAt: s.SentAt, Sender: s.Sender,
|
|
||||||
Epoch: s.Epoch, Generation: s.Generation, Digest: s.Digest, Modules: modules, Answer: s.Answer.Outcome,
|
|
||||||
Refused: s.Answer.Refused, Failed: s.Answer.Failed, AnsweredAt: s.Answer.At,
|
|
||||||
RefusedForGeneration: s.RefusedAt, AppliedGeneration: s.RefusedApplied})
|
|
||||||
}
|
|
||||||
sort.Slice(out, func(i, j int) bool { return out[i].Node < out[j].Node })
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -61,57 +61,20 @@ func keptExcept(ctx context.Context) (map[string]bool, bool) {
|
|||||||
return skip, on
|
return skip, on
|
||||||
}
|
}
|
||||||
|
|
||||||
type keepEveryBuildKey struct{}
|
|
||||||
|
|
||||||
// keepingEveryBuild is a context whose sends compose every module of one machine — recorded or rolling out —
|
|
||||||
// at the build that machine was last sent (novox/hq issue 490): the grants step, which sends the machine
|
|
||||||
// holding the bus its new user list and nothing of any module's new code. That code waits there for a gate
|
|
||||||
// like any other move; the list must not, or the first machine's gate is judged against a bus that refuses
|
|
||||||
// what the change newly grants.
|
|
||||||
//
|
|
||||||
// **That machine alone.** Composing one machine resolves the others too — who is on the private network,
|
|
||||||
// who answers a requirement — on the same context, and those are no part of the step's send: they are
|
|
||||||
// composed as any send composes them. Kept for every machine, a machine whose last send is not known
|
|
||||||
// refused the bus's machine's composition.
|
|
||||||
func keepingEveryBuild(ctx context.Context, node string) context.Context {
|
|
||||||
return context.WithValue(ctx, keepEveryBuildKey{}, node)
|
|
||||||
}
|
|
||||||
|
|
||||||
// everyBuildKept is whether this context keeps every module of this machine at the build it runs.
|
|
||||||
func everyBuildKept(ctx context.Context, node string) bool {
|
|
||||||
on, _ := ctx.Value(keepEveryBuildKey{}).(string)
|
|
||||||
return on != "" && on == node
|
|
||||||
}
|
|
||||||
|
|
||||||
// recordedKept is, for a send under keepingRecorded, every recorded module the machine was last sent a
|
// recordedKept is, for a send under keepingRecorded, every recorded module the machine was last sent a
|
||||||
// build of that the mesh's build is not identical to: module → the commit it keeps. Nil when the context
|
// build of that the mesh's build is not identical to: module → the commit it keeps. Nil when the context
|
||||||
// keeps nothing, or when what the machine was last sent is not known (it is then held whole elsewhere —
|
// keeps nothing, or when what the machine was last sent is not known (it is then held whole elsewhere —
|
||||||
// ADR 0221).
|
// ADR 0221).
|
||||||
//
|
|
||||||
// Under keepingEveryBuild, every module the machine was sent is kept, whatever its policy (novox/hq issue
|
|
||||||
// 490), and a machine whose last send is not known refuses the send: there is nothing to keep it at, and
|
|
||||||
// composing the mesh's builds would be the very move the grants step must not make.
|
|
||||||
func recordedKept(ctx context.Context, open *stores, node string) (map[string]string, error) {
|
func recordedKept(ctx context.Context, open *stores, node string) (map[string]string, error) {
|
||||||
every := everyBuildKept(ctx, node)
|
|
||||||
skip, on := keptExcept(ctx)
|
skip, on := keptExcept(ctx)
|
||||||
if !on && !every {
|
if !on {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
if every {
|
|
||||||
skip = nil
|
|
||||||
}
|
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
sent, known, err := inv.SentBuilds(ctx, node)
|
sent, known, err := inv.SentBuilds(ctx, node)
|
||||||
if err != nil {
|
if err != nil || !known {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if !known {
|
|
||||||
if every {
|
|
||||||
return nil, fmt.Errorf("what %s was last sent is not known, so the bus's user list cannot be sent "+
|
|
||||||
"there alone with every build kept (novox/hq issue 490)", node)
|
|
||||||
}
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
current, err := inv.CurrentBuilds(ctx)
|
current, err := inv.CurrentBuilds(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -120,7 +83,7 @@ func recordedKept(ctx context.Context, open *stores, node string) (map[string]st
|
|||||||
out := map[string]string{}
|
out := map[string]string{}
|
||||||
for m, was := range sent {
|
for m, was := range sent {
|
||||||
now, held := current[m]
|
now, held := current[m]
|
||||||
if !held || (now.RollOut && !every) || skip[m] || was == "" || sameCommit(was, now.Commit) {
|
if !held || now.RollOut || skip[m] || was == "" || sameCommit(was, now.Commit) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if f == nil {
|
if f == nil {
|
||||||
@@ -155,10 +118,6 @@ func keepRecorded(ctx context.Context, open *stores, node string, shelf map[stri
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if !found && everyBuildKept(ctx, node) {
|
|
||||||
return nil, fmt.Errorf("%s runs %s's build %s, which the build records no longer hold: the bus's user "+
|
|
||||||
"list cannot be sent there alone with it kept (novox/hq issue 490)", node, m, short(kept[m]))
|
|
||||||
}
|
|
||||||
if !found {
|
if !found {
|
||||||
return nil, fmt.Errorf("%s records rather than rolls out, and %s runs its build %s, which the build "+
|
return nil, fmt.Errorf("%s records rather than rolls out, and %s runs its build %s, which the build "+
|
||||||
"records no longer hold: this send cannot keep it and does not move it — `push %s` sends the new "+
|
"records no longer hold: this send cannot keep it and does not move it — `push %s` sends the new "+
|
||||||
|
|||||||
@@ -1,111 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// What relates two changes (novox/hq ADR 0304 decision 1, as ADR 0306 decision 3 reads it): two changes are
|
|
||||||
// related when they share a repository, a module, a delivery group's head, or an edge of the module graph
|
|
||||||
// between their modules. A module is every one the walk builds or sends: the modules its merges moved and the
|
|
||||||
// modules that depend on them, not only the ones the merges touched. Unrelated changes may walk beside each
|
|
||||||
// other (ADR 0304 decision 2, ADR 0282 decision 3); nothing walks so yet, and nothing but tests reads this.
|
|
||||||
|
|
||||||
// footprint is what one change reaches: its repositories (lower case), its modules, and its delivery group
|
|
||||||
// heads (the head branch names of its merges). unknown says a merge of it was kept without the modules it
|
|
||||||
// moves, so what it reaches is not known: it is related to every change, never wrongly to none.
|
|
||||||
type footprint struct {
|
|
||||||
repos, modules, heads map[string]bool
|
|
||||||
unknown string
|
|
||||||
}
|
|
||||||
|
|
||||||
func newFootprint() footprint {
|
|
||||||
return footprint{repos: map[string]bool{}, modules: map[string]bool{}, heads: map[string]bool{}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// footprintOfWalk is a walk's footprint: the repositories it carries and the modules it plans. A walk keeps no
|
|
||||||
// head of its merges; kept are the merges the record holds for it, whose announcements say their heads, and
|
|
||||||
// may be nil.
|
|
||||||
func footprintOfWalk(p inventory.Plan, kept []inventory.BatchedMerge) footprint {
|
|
||||||
f := newFootprint()
|
|
||||||
for _, c := range p.Carried() {
|
|
||||||
f.repos[strings.ToLower(c.Repository)] = true
|
|
||||||
}
|
|
||||||
for name := range p.Modules {
|
|
||||||
f.modules[name] = true
|
|
||||||
}
|
|
||||||
for _, m := range kept {
|
|
||||||
f.repos[strings.ToLower(m.Repository)] = true
|
|
||||||
if h := announcedOf(m).Head; h != "" {
|
|
||||||
f.heads[h] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return f
|
|
||||||
}
|
|
||||||
|
|
||||||
// footprintOfBatch is an assembling batch's footprint: each merge's repository and head, and the modules its
|
|
||||||
// merges moved when heard, widened to their dependents (planOfMoves). A merge kept with no moves (one owed a
|
|
||||||
// later look, or kept for its packaging) is judged again at the cut and may move anything then: its batch is
|
|
||||||
// unknown, related to everything.
|
|
||||||
func footprintOfBatch(merges []inventory.BatchedMerge, edges []inventory.Edge) footprint {
|
|
||||||
f := newFootprint()
|
|
||||||
var moved []string
|
|
||||||
for _, m := range merges {
|
|
||||||
k := announcedOf(m)
|
|
||||||
f.repos[strings.ToLower(m.Repository)] = true
|
|
||||||
if k.Head != "" {
|
|
||||||
f.heads[k.Head] = true
|
|
||||||
}
|
|
||||||
if len(k.Moves) == 0 && f.unknown == "" {
|
|
||||||
f.unknown = "the modules a merge of " + m.Repository + " moves are not known until its batch is cut"
|
|
||||||
}
|
|
||||||
moved = append(moved, k.Moves...)
|
|
||||||
}
|
|
||||||
for _, name := range reachableFrom(moved, edges) {
|
|
||||||
f.modules[name] = true
|
|
||||||
}
|
|
||||||
return f
|
|
||||||
}
|
|
||||||
|
|
||||||
// related says whether two changes are related, and the first reason in plain words; false and empty when
|
|
||||||
// they are not.
|
|
||||||
func related(a, b footprint, edges []inventory.Edge) (bool, string) {
|
|
||||||
if a.unknown != "" {
|
|
||||||
return true, a.unknown
|
|
||||||
}
|
|
||||||
if b.unknown != "" {
|
|
||||||
return true, b.unknown
|
|
||||||
}
|
|
||||||
if r := firstShared(a.repos, b.repos); r != "" {
|
|
||||||
return true, "both change " + r
|
|
||||||
}
|
|
||||||
if m := firstShared(a.modules, b.modules); m != "" {
|
|
||||||
return true, "both move " + m
|
|
||||||
}
|
|
||||||
if h := firstShared(a.heads, b.heads); h != "" {
|
|
||||||
return true, "both are members of group " + h
|
|
||||||
}
|
|
||||||
for _, e := range edges {
|
|
||||||
if a.modules[e.From] && b.modules[e.To] || b.modules[e.From] && a.modules[e.To] {
|
|
||||||
return true, e.From + " depends on " + e.To + " (" + e.Kind + ")"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false, ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// firstShared is the first key, in order, both sets hold; empty when none.
|
|
||||||
func firstShared(a, b map[string]bool) string {
|
|
||||||
var both []string
|
|
||||||
for k := range a {
|
|
||||||
if b[k] {
|
|
||||||
both = append(both, k)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(both) == 0 {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
sort.Strings(both)
|
|
||||||
return both[0]
|
|
||||||
}
|
|
||||||
@@ -1,110 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// heardMerge is a kept merge of a repository from a head branch that moved modules.
|
|
||||||
func heardMerge(repository, head string, moves ...string) inventory.BatchedMerge {
|
|
||||||
m := inventory.BatchedMerge{Repository: repository, Branch: "main", Commit: "c-" + repository}
|
|
||||||
m.Event, _ = json.Marshal(keptMerge{SourceMoved: link.SourceMoved{Head: head}, Moves: moves})
|
|
||||||
return m
|
|
||||||
}
|
|
||||||
|
|
||||||
// walkOf is a walk carrying one repository's commit and planning modules.
|
|
||||||
func walkOf(repository string, modules ...string) inventory.Plan {
|
|
||||||
p := inventory.Plan{Repository: repository, Commit: "w-" + repository, Modules: map[string]*inventory.PlanModule{}}
|
|
||||||
for _, name := range modules {
|
|
||||||
p.Modules[name] = &inventory.PlanModule{}
|
|
||||||
}
|
|
||||||
return p
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRelatedChanges: two changes are related by a repository, a module, a group head or a graph edge between
|
|
||||||
// their modules, and by a module reached only through dependents (novox/hq ADR 0304 decision 1, ADR 0306
|
|
||||||
// decision 3); otherwise they are not.
|
|
||||||
func TestRelatedChanges(t *testing.T) {
|
|
||||||
edges := []inventory.Edge{
|
|
||||||
{From: "nzbget", To: "base-image", Kind: inventory.EdgeStandsOn},
|
|
||||||
{From: "app", To: "build-machine", Kind: inventory.EdgeBuiltBy},
|
|
||||||
}
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
a, b footprint
|
|
||||||
why string
|
|
||||||
}{
|
|
||||||
{"repository, case aside",
|
|
||||||
footprintOfWalk(walkOf("novox/Mesh-Catalog", "notes"), nil),
|
|
||||||
footprintOfBatch([]inventory.BatchedMerge{heardMerge("NOVOX/mesh-catalog", "fix/a", "app")}, nil),
|
|
||||||
"both change novox/mesh-catalog"},
|
|
||||||
{"module",
|
|
||||||
footprintOfWalk(walkOf("novox/one", "nzbget"), nil),
|
|
||||||
footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/two", "fix/b", "nzbget")}, nil),
|
|
||||||
"both move nzbget"},
|
|
||||||
{"edge, walk depends on batch",
|
|
||||||
footprintOfWalk(walkOf("novox/one", "nzbget"), nil),
|
|
||||||
footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/two", "fix/b", "base-image")}, nil),
|
|
||||||
"nzbget depends on base-image (stands-on)"},
|
|
||||||
{"edge, batch depends on walk",
|
|
||||||
footprintOfWalk(walkOf("novox/one", "build-machine"), nil),
|
|
||||||
footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/two", "fix/b", "app")}, nil),
|
|
||||||
"app depends on build-machine (built-by)"},
|
|
||||||
{"group head",
|
|
||||||
footprintOfWalk(walkOf("novox/one", "notes"), []inventory.BatchedMerge{heardMerge("novox/one", "feat/x")}),
|
|
||||||
footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/two", "feat/x", "app")}, nil),
|
|
||||||
"both are members of group feat/x"},
|
|
||||||
{"module reached through dependents",
|
|
||||||
footprintOfWalk(walkOf("novox/one", "nzbget"), nil),
|
|
||||||
footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/two", "fix/b", "base-image")}, edges),
|
|
||||||
"both move nzbget"},
|
|
||||||
{"unrelated",
|
|
||||||
footprintOfWalk(walkOf("novox/one", "notes"), []inventory.BatchedMerge{heardMerge("novox/one", "feat/x")}),
|
|
||||||
footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/two", "feat/y", "app")}, edges),
|
|
||||||
""},
|
|
||||||
}
|
|
||||||
for _, c := range cases {
|
|
||||||
t.Run(c.name, func(t *testing.T) {
|
|
||||||
for _, pair := range [][2]footprint{{c.a, c.b}, {c.b, c.a}} {
|
|
||||||
is, why := related(pair[0], pair[1], edges)
|
|
||||||
if is != (c.why != "") || why != c.why {
|
|
||||||
t.Errorf("related = %v, %q; want %q", is, why, c.why)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestFootprintOfBatchWidensToDependents: a batch's modules are what its merges moved and every module that
|
|
||||||
// depends on them, along the edges that widen a walk.
|
|
||||||
func TestFootprintOfBatchWidensToDependents(t *testing.T) {
|
|
||||||
edges := []inventory.Edge{
|
|
||||||
{From: "nzbget", To: "base-image", Kind: inventory.EdgeStandsOn},
|
|
||||||
{From: "base-image", To: "build-machine", Kind: inventory.EdgeBuiltBy},
|
|
||||||
}
|
|
||||||
f := footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/two", "", "base-image")}, edges)
|
|
||||||
if !f.modules["base-image"] || !f.modules["nzbget"] || f.modules["build-machine"] || len(f.heads) != 0 {
|
|
||||||
t.Errorf("footprint = %+v; want base-image and nzbget, no head", f)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A batch holding a merge kept with no moves (one owed a later look, judged again at the cut) is related to
|
|
||||||
// every change, in both orders: what it will move is not known, and an unknown change is never unrelated
|
|
||||||
// (issue 525 review).
|
|
||||||
func TestABatchWhoseMovesAreNotKnownIsRelatedToEverything(t *testing.T) {
|
|
||||||
unknown := footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/one", "fix/a", "app"), heardMerge("novox/two", "fix/b")}, nil)
|
|
||||||
other := footprintOfWalk(walkOf("novox/three", "nzbget"), nil)
|
|
||||||
for _, pair := range [][2]footprint{{unknown, other}, {other, unknown}} {
|
|
||||||
if ok, why := related(pair[0], pair[1], nil); !ok || !strings.Contains(why, "not known") {
|
|
||||||
t.Fatalf("an unknown batch read as %v (%q)", ok, why)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
known := footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/one", "fix/a", "app")}, nil)
|
|
||||||
if ok, why := related(known, other, nil); ok {
|
|
||||||
t.Fatalf("a known, unrelated batch read as related: %q", why)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+12
-190
@@ -2,7 +2,6 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
"errors"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -254,19 +253,8 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
// The grants step composes every module at the build its machine was last sent (novox/hq issue 490):
|
|
||||||
// a move it keeps is not made. Read, not assumed, so a move it could not keep is still refused here.
|
|
||||||
var keeps map[string]string
|
|
||||||
if everyBuildKept(ctx, n) {
|
|
||||||
if keeps, err = recordedKept(ctx, open, n); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var waiting []string
|
var waiting []string
|
||||||
for _, mv := range moves {
|
for _, mv := range moves {
|
||||||
if was, kept := keeps[mv.Module]; kept && sameCommit(was, mv.From) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if !scope.judged[n] && !scope.modules[mv.Module] {
|
if !scope.judged[n] && !scope.modules[mv.Module] {
|
||||||
waiting = append(waiting, fmt.Sprintf("%s %s → %s", mv.Module, short(mv.From), short(mv.To)))
|
waiting = append(waiting, fmt.Sprintf("%s %s → %s", mv.Module, short(mv.From), short(mv.To)))
|
||||||
}
|
}
|
||||||
@@ -295,18 +283,15 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
|
|||||||
// owns are the moves the send exists for — every module of a plan's tier whose first machine this is, in
|
// owns are the moves the send exists for — every module of a plan's tier whose first machine this is, in
|
||||||
// one send (novox/hq issue 281): a send carries the machine's whole declaration (ADR 0221), so a send per
|
// one send (novox/hq issue 281): a send carries the machine's whole declaration (ADR 0221), so a send per
|
||||||
// module was the same declaration sent again and again, each one setting aside the one before.
|
// module was the same declaration sent again and again, each one setting aside the one before.
|
||||||
//
|
func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, []string, error) {
|
||||||
// And the grants step first, when the send changes what the bus's user list must say (novox/hq issue 490):
|
|
||||||
// answered for the gate to keep, nil when there was none.
|
|
||||||
func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, []string, *inventory.GrantsStep, error) {
|
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
f, err := readMoveFacts(ctx, inv)
|
f, err := readMoveFacts(ctx, inv)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
moves, err := machineMoves(ctx, open, f, node, true)
|
moves, err := machineMoves(ctx, open, f, node, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
own := func(module string) bool {
|
own := func(module string) bool {
|
||||||
return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module })
|
return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module })
|
||||||
@@ -316,7 +301,7 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if id := f.walkedBy(mv.Module, node, mv.To); id != "" {
|
if id := f.walkedBy(mv.Module, node, mv.To); id != "" {
|
||||||
return nil, nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
return nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
||||||
mv.Module, short(mv.To), node, id)
|
mv.Module, short(mv.To), node, id)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -324,7 +309,7 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
|
|||||||
// plan, or a release, on this machine is not crossed with a newer build; this send waits for its gate.
|
// plan, or a release, on this machine is not crossed with a newer build; this send waits for its gate.
|
||||||
for _, o := range owns {
|
for _, o := range owns {
|
||||||
if id := f.walkedBy(o.Module, node, o.To); id != "" {
|
if id := f.walkedBy(o.Module, node, o.To); id != "" {
|
||||||
return nil, nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
return nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
||||||
o.Module, short(o.To), node, id)
|
o.Module, short(o.To), node, id)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -338,22 +323,22 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(moves) == 0 && len(owns) == 0 {
|
if len(moves) == 0 && len(owns) == 0 {
|
||||||
return nil, nil, nil, nil
|
return nil, nil, nil
|
||||||
}
|
}
|
||||||
for i := range moves {
|
for i := range moves {
|
||||||
if moves[i].Build == "" {
|
if moves[i].Build == "" {
|
||||||
if moves[i].Build, err = inv.BuildOf(ctx, moves[i].Module, moves[i].To); err != nil {
|
if moves[i].Build, err = inv.BuildOf(ctx, moves[i].Module, moves[i].To); err != nil {
|
||||||
return nil, nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
sort.Slice(moves, func(i, j int) bool { return moves[i].Module < moves[j].Module })
|
sort.Slice(moves, func(i, j int) bool { return moves[i].Module < moves[j].Module })
|
||||||
sayRecreations(ctx, open, moves)
|
sayRecreations(ctx, open, moves)
|
||||||
sent, grants, err := sendJudged(ctx, open, node)
|
sent, err := sendRollout(withScope(ctx, sendScope{judged: map[string]bool{node: true}}), open, []string{node})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
return moves, sent, grants, nil
|
return moves, sent, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// passCarried keeps a pass as the verdict of every build the gate judged beside its own module.
|
// passCarried keeps a pass as the verdict of every build the gate judged beside its own module.
|
||||||
@@ -689,7 +674,7 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
|||||||
r.Next++
|
r.Next++
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
moves, sent, grants, err := gatedSend(ctx, open, node, nil)
|
moves, sent, err := gatedSend(ctx, open, node, nil)
|
||||||
if errors.Is(err, errWalkedElsewhere) {
|
if errors.Is(err, errWalkedElsewhere) {
|
||||||
note := fmt.Sprintf("waiting before %s: %v", node, err)
|
note := fmt.Sprintf("waiting before %s: %v", node, err)
|
||||||
changed := p.Note != note
|
changed := p.Note != note
|
||||||
@@ -704,12 +689,8 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
|||||||
r.Next++
|
r.Next++
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves, Sent: sentNow(ctx, open.inventory, sent),
|
r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves, Sent: sentNow(ctx, open.inventory, sent)}
|
||||||
Grants: grants}
|
|
||||||
p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves))
|
p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves))
|
||||||
if said := grantsSaid(grants); said != "" {
|
|
||||||
p.Note += "; " + said
|
|
||||||
}
|
|
||||||
if said := recreationsSaid(moves); said != "" {
|
if said := recreationsSaid(moves); said != "" {
|
||||||
p.Note += "; " + said
|
p.Note += "; " + said
|
||||||
}
|
}
|
||||||
@@ -935,162 +916,3 @@ func recreationsSaid(moves []inventory.CarriedMove) string {
|
|||||||
}
|
}
|
||||||
return strings.Join(said, "; ")
|
return strings.Join(said, "; ")
|
||||||
}
|
}
|
||||||
|
|
||||||
// The grants step (novox/hq issue 490).
|
|
||||||
//
|
|
||||||
// The bus's user list — every module's grants, composed mesh-wide from the catalogue — travels only in the
|
|
||||||
// declaration of the machine holding the bus. A send to any other machine that changed it put that machine
|
|
||||||
// first (issue 249), unless a build waited there for a gate: then ungatedIn left it out, and said so. On
|
|
||||||
// 2026-10-11 a merge gave restic a new health tool; its walk sent restic to its first machine and judged it
|
|
||||||
// there, the machine's node-engine asked the tool and the bus refused it ("this host's grant does not name
|
|
||||||
// …"), because the machine holding the bus also runs restic, whose new build waited for that very gate. The
|
|
||||||
// gate could not pass; a person pushed the bus's machine by hand, which carried restic's new build there
|
|
||||||
// unjudged.
|
|
||||||
//
|
|
||||||
// So before a gated send, when the user list composed now is not the one the machine holding the bus was
|
|
||||||
// last sent, that machine is sent its declaration with **every build kept at the one it runs**: the new
|
|
||||||
// list, and nothing of any module's new code. Said on the gate (`plans`), and, when it cannot be sent, said
|
|
||||||
// with why and passed over: the send goes on as it did before, and its gate says what it finds.
|
|
||||||
|
|
||||||
// brokerBehindOf is brokerBehind: a variable so a test of the walk needs no store.
|
|
||||||
var brokerBehindOf = brokerBehind
|
|
||||||
|
|
||||||
// grantsStep sends the machine holding the bus its user list alone, ahead of a gated send to node, when the
|
|
||||||
// list changed; answers what it did, or nil when there was nothing to send.
|
|
||||||
func grantsStep(ctx context.Context, open *stores, node string) *inventory.GrantsStep {
|
|
||||||
holder, behind, err := brokerBehindOf(ctx, open, []string{node})
|
|
||||||
if err != nil {
|
|
||||||
fmt.Printf("grants step before %s: whether the bus's user list changed could not be read: %v\n", node, err)
|
|
||||||
return &inventory.GrantsStep{Node: "the machine holding the bus", Failed: err.Error()}
|
|
||||||
}
|
|
||||||
if holder == "" || holder == node || !behind {
|
|
||||||
// No bus with a user list, the gate's own machine holds it (its send carries the list first), or the
|
|
||||||
// list is the one already sent.
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if _, err := sendRollout(keepingEveryBuild(withScope(ctx, sendScope{}), holder), open, []string{holder}); err != nil {
|
|
||||||
fmt.Printf("grants step: the bus's user list could not be sent to %s ahead of %s, which is sent without "+
|
|
||||||
"it — the bus may refuse what the send newly grants: %v\n", holder, node, err)
|
|
||||||
return &inventory.GrantsStep{Node: holder, Failed: err.Error()}
|
|
||||||
}
|
|
||||||
now := time.Now().UTC()
|
|
||||||
fmt.Printf("grants step: %s sent the bus's user list alone, every build there kept, before %s is judged\n",
|
|
||||||
holder, node)
|
|
||||||
return &inventory.GrantsStep{Node: holder, At: &now}
|
|
||||||
}
|
|
||||||
|
|
||||||
// errNotGrantsOnly is a grants step refused unsent: its declaration would change more than the user list.
|
|
||||||
var errNotGrantsOnly = errors.New("the grants step would change more than the bus's user list, and is not sent")
|
|
||||||
|
|
||||||
// grantsOnlyIn is what a grants step's declaration, composed for sending, would change on the machine holding
|
|
||||||
// the bus besides its user list, against what it was last sent (novox/hq issue 490); empty when nothing else.
|
|
||||||
// Judged by sendToEach on the very declaration it then sends — one composition, judged, then sent or refused
|
|
||||||
// unsent. Kept builds are not the whole of a declaration: a new assignment, a settings change, an assignment
|
|
||||||
// taken away, a provision's logins are composed as the mesh holds them now, and a step that carried any of
|
|
||||||
// it would be the unjudged change this step exists to avoid. Compared resource by resource with the summary
|
|
||||||
// the last send kept (ADR 0217): exact, rather than a list of the cases that can differ. A module there that
|
|
||||||
// was never sent is a new assignment, refused whatever its resources.
|
|
||||||
func grantsOnlyIn(ctx context.Context, open *stores, holder string, plan catalogue.Resolution, declared sendable) (string, error) {
|
|
||||||
inv := open.inventory
|
|
||||||
sent, known, err := inv.SentBuilds(ctx, holder)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if !known {
|
|
||||||
return fmt.Sprintf("what %s was last sent is not known", holder), nil
|
|
||||||
}
|
|
||||||
var other []string
|
|
||||||
listID := ""
|
|
||||||
for _, m := range plan.Modules {
|
|
||||||
if _, was := sent[m.Module]; !was {
|
|
||||||
other = append(other, m.Module+" newly assigned")
|
|
||||||
}
|
|
||||||
if m.BusUsers != "" && m.ClaimsSeat(catalogue.BrokerSeat) {
|
|
||||||
listID = m.Module + "." + catalogue.BusUsersID()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var facts *moveFacts
|
|
||||||
for m, was := range sent {
|
|
||||||
now, carried := declared.Builds[m]
|
|
||||||
if !carried || sameCommit(now, was) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if facts == nil {
|
|
||||||
f, err := readMoveFacts(ctx, inv)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
facts = &f
|
|
||||||
}
|
|
||||||
if !facts.identical(m, was, now) {
|
|
||||||
other = append(other, fmt.Sprintf("%s %s → %s", m, short(was), short(now)))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, f := range declared.foreseen {
|
|
||||||
other = append(other, f+" would be minted")
|
|
||||||
}
|
|
||||||
body, err := declared.Body()
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
after, err := summarize(body)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
prev, err := inv.SentSummary(ctx, holder)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if len(prev) == 0 {
|
|
||||||
return fmt.Sprintf("what %s was last sent is not kept for comparison", holder), nil
|
|
||||||
}
|
|
||||||
var before []sentResource
|
|
||||||
if err := json.Unmarshal(prev, &before); err != nil {
|
|
||||||
return "", fmt.Errorf("what %s was last sent is kept in a form this version does not read: %w", holder, err)
|
|
||||||
}
|
|
||||||
other = append(other, otherThanTheList(diffSent(before, after), listID)...)
|
|
||||||
sort.Strings(other)
|
|
||||||
if len(other) == 0 {
|
|
||||||
return "", nil
|
|
||||||
}
|
|
||||||
return "its send would change more than the bus's user list: " + strings.Join(other, "; "), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// otherThanTheList is what a diff against the last send changes besides the user list's content: every
|
|
||||||
// resource added, removed or changed, but the list's own resource when its content is all that changed —
|
|
||||||
// its path, mode and every other field must be as last sent (novox/hq issue 490).
|
|
||||||
func otherThanTheList(d sentDiff, listID string) []string {
|
|
||||||
var other []string
|
|
||||||
for _, id := range d.Added {
|
|
||||||
other = append(other, "+"+id)
|
|
||||||
}
|
|
||||||
for _, id := range d.Removed {
|
|
||||||
other = append(other, "-"+id)
|
|
||||||
}
|
|
||||||
for _, c := range d.Changed {
|
|
||||||
if c.ID == listID && len(c.Fields) == 1 && c.Fields[0] == "content" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
other = append(other, fmt.Sprintf("~%s (%s)", c.ID, strings.Join(c.Fields, ", ")))
|
|
||||||
}
|
|
||||||
return other
|
|
||||||
}
|
|
||||||
|
|
||||||
// sendJudged is a gated send's sends: the grants step when the user list changed, then the machine judged.
|
|
||||||
func sendJudged(ctx context.Context, open *stores, node string) ([]string, *inventory.GrantsStep, error) {
|
|
||||||
grants := grantsStep(ctx, open, node)
|
|
||||||
sent, err := sendRollout(withScope(ctx, sendScope{judged: map[string]bool{node: true}}), open, []string{node})
|
|
||||||
return sent, grants, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// grantsSaid is a grants step as `plans` and a walk's note say it.
|
|
||||||
func grantsSaid(g *inventory.GrantsStep) string {
|
|
||||||
switch {
|
|
||||||
case g == nil:
|
|
||||||
return ""
|
|
||||||
case g.Failed != "":
|
|
||||||
return fmt.Sprintf(grantsStepWord+" to %s FAILED, sent without it: %s", g.Node, g.Failed)
|
|
||||||
default:
|
|
||||||
return fmt.Sprintf(grantsStepWord+": %s sent the bus's user list first, its builds kept", g.Node)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -510,14 +510,6 @@ func advancePlans(ctx context.Context, open *stores) {
|
|||||||
advanceHeld(ctx, open)
|
advanceHeld(ctx, open)
|
||||||
}
|
}
|
||||||
|
|
||||||
// keepWalkPhases keeps where the walks that ended lately spent their time (novox/hq ADR 0282), outside the hold
|
|
||||||
// on the plans so it never lengthens it: measured, never acted on, and an error only said.
|
|
||||||
func keepWalkPhases(ctx context.Context, open *stores) {
|
|
||||||
if err := recordWalkPhases(ctx, open.inventory, time.Now()); err != nil {
|
|
||||||
fmt.Printf("plans: the phases of the walks ended lately could not be kept: %v\n", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// advanceHeld is advancePlans for a caller already holding the plans.
|
// advanceHeld is advancePlans for a caller already holding the plans.
|
||||||
func advanceHeld(ctx context.Context, open *stores) {
|
func advanceHeld(ctx context.Context, open *stores) {
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
@@ -649,10 +641,27 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
// the controller in its first tier: the build that produced the new one is recorded, and the
|
// the controller in its first tier: the build that produced the new one is recorded, and the
|
||||||
// plan never hears it. The record is the fact; a build recorded after the ask is that tier's
|
// plan never hears it. The record is the fact; a build recorded after the ask is that tier's
|
||||||
// outcome, whoever was listening.
|
// outcome, whoever was listening.
|
||||||
recorded, byID, err := recordsOfAsked(ctx, inv, p, tier)
|
recorded := map[string][]inventory.Build{}
|
||||||
|
byID := map[string]inventory.Build{}
|
||||||
|
for _, m := range tier {
|
||||||
|
if s := p.Modules[m]; s != nil && s.State == "asked" {
|
||||||
|
builds, err := inv.Builds(ctx, m, 5)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
|
recorded[m] = builds
|
||||||
|
// Its own ask's record, by id — found even when the outcome named no module (ADR 0219).
|
||||||
|
if s.Build != "" {
|
||||||
|
b, found, err := inv.BuildByID(ctx, s.Build)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if found {
|
||||||
|
byID[s.Build] = b
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
if settleFromRecords(p, tier, recorded, byID) {
|
if settleFromRecords(p, tier, recorded, byID) {
|
||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
@@ -855,12 +864,8 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
strings.Join(machines, ", "), p.Tier, err)
|
strings.Join(machines, ", "), p.Tier, err)
|
||||||
}
|
}
|
||||||
now := time.Now().UTC()
|
now := time.Now().UTC()
|
||||||
// What each machine of the rest was sent, kept for when it reports it applied (novox/hq ADR 0282 decision
|
|
||||||
// 6): measured, never acted on.
|
|
||||||
sentWhat := sentNow(ctx, open.inventory, sent)
|
|
||||||
for _, m := range rest {
|
for _, m := range rest {
|
||||||
p.Modules[m].SentAt = &now
|
p.Modules[m].SentAt = &now
|
||||||
p.Modules[m].Rest = restOf(restTo[m], sent, sentWhat)
|
|
||||||
}
|
}
|
||||||
fmt.Printf("%s: tier %d built; sent %s to %s, one send each\n", p.ID, p.Tier, strings.Join(rest, ", "),
|
fmt.Printf("%s: tier %d built; sent %s to %s, one send each\n", p.ID, p.Tier, strings.Join(rest, ", "),
|
||||||
strings.Join(sent, ", "))
|
strings.Join(sent, ", "))
|
||||||
@@ -941,20 +946,6 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// restOf is, for one module, every machine of its rest that the send reached and what it carried there.
|
|
||||||
func restOf(to, sent []string, what map[string]inventory.SentDeclaration) map[string]inventory.SentDeclaration {
|
|
||||||
out := map[string]inventory.SentDeclaration{}
|
|
||||||
for _, n := range to {
|
|
||||||
if slices.Contains(sent, n) {
|
|
||||||
out[n] = what[n]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(out) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// firstSend sends one machine, in one send, every module of the plan's tier whose first machine it is
|
// firstSend sends one machine, in one send, every module of the plan's tier whose first machine it is
|
||||||
// (novox/hq issue 281), and records the send on each: what that machine ran of it before — read once,
|
// (novox/hq issue 281), and records the send on each: what that machine ran of it before — read once,
|
||||||
// before the send, so a module the same send carries is never read as already moved — the machines it
|
// before the send, so a module the same send carries is never read as already moved — the machines it
|
||||||
@@ -971,7 +962,7 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
|||||||
}
|
}
|
||||||
// A gated send (ADR 0236): everything waiting on the machine goes with the tier, and the gate judges
|
// A gated send (ADR 0236): everything waiting on the machine goes with the tier, and the gate judges
|
||||||
// all of it there.
|
// all of it there.
|
||||||
carried, sent, grants, err := gatedSend(ctx, open, node, owns)
|
carried, sent, err := gatedSend(ctx, open, node, owns)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -992,7 +983,6 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
|||||||
s.GatedBy = ""
|
s.GatedBy = ""
|
||||||
if m == lead {
|
if m == lead {
|
||||||
s.Gate.Carried = carried
|
s.Gate.Carried = carried
|
||||||
s.Gate.Grants = grants
|
|
||||||
} else {
|
} else {
|
||||||
s.GatedBy = lead
|
s.GatedBy = lead
|
||||||
}
|
}
|
||||||
@@ -1002,10 +992,6 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
|||||||
strings.Join(sent, ", "))
|
strings.Join(sent, ", "))
|
||||||
fmt.Printf("%s: tier %d built; sent %d module(s) to %s first in one send (%s), the rest once its gate passes\n",
|
fmt.Printf("%s: tier %d built; sent %d module(s) to %s first in one send (%s), the rest once its gate passes\n",
|
||||||
p.ID, p.Tier, len(modules), strings.Join(sent, ", "), strings.Join(modules, ", "))
|
p.ID, p.Tier, len(modules), strings.Join(sent, ", "), strings.Join(modules, ", "))
|
||||||
// The grants step taken before it (novox/hq issue 490), said with it.
|
|
||||||
if said := grantsSaid(grants); said != "" {
|
|
||||||
p.Note += "; " + said
|
|
||||||
}
|
|
||||||
// What the send recreates, said with it (novox/hq ADR 0245).
|
// What the send recreates, said with it (novox/hq ADR 0245).
|
||||||
if said := recreationsSaid(carried); said != "" {
|
if said := recreationsSaid(carried); said != "" {
|
||||||
p.Note += "; " + said
|
p.Note += "; " + said
|
||||||
@@ -1207,7 +1193,6 @@ func sayUnsent(p *inventory.Plan, rollsOut func(string) bool) {
|
|||||||
// planTicker advances open plans on a timer, for the steps outcomes alone cannot take.
|
// planTicker advances open plans on a timer, for the steps outcomes alone cannot take.
|
||||||
func planTicker(ctx context.Context, open *stores) {
|
func planTicker(ctx context.Context, open *stores) {
|
||||||
advancePlans(ctx, open)
|
advancePlans(ctx, open)
|
||||||
keepWalkPhases(ctx, open)
|
|
||||||
tick := time.NewTicker(30 * time.Second)
|
tick := time.NewTicker(30 * time.Second)
|
||||||
defer tick.Stop()
|
defer tick.Stop()
|
||||||
for {
|
for {
|
||||||
@@ -1216,7 +1201,6 @@ func planTicker(ctx context.Context, open *stores) {
|
|||||||
return
|
return
|
||||||
case <-tick.C:
|
case <-tick.C:
|
||||||
advancePlans(ctx, open)
|
advancePlans(ctx, open)
|
||||||
keepWalkPhases(ctx, open)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1764,34 +1748,6 @@ func splitList(s string) []string {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// recordsOfAsked is what settleFromRecords reads: for every module of the tier still `asked`, its last
|
|
||||||
// builds and the record of its own ask, by id.
|
|
||||||
func recordsOfAsked(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan, tier []string) (
|
|
||||||
map[string][]inventory.Build, map[string]inventory.Build, error) {
|
|
||||||
recorded := map[string][]inventory.Build{}
|
|
||||||
byID := map[string]inventory.Build{}
|
|
||||||
for _, m := range tier {
|
|
||||||
if s := p.Modules[m]; s != nil && s.State == "asked" {
|
|
||||||
builds, err := inv.Builds(ctx, m, 5)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
recorded[m] = builds
|
|
||||||
// Its own ask's record, by id — found even when the outcome named no module (ADR 0219).
|
|
||||||
if s.Build != "" {
|
|
||||||
b, found, err := inv.BuildByID(ctx, s.Build)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
if found {
|
|
||||||
byID[s.Build] = b
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return recorded, byID, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// settleFromRecords marks every module of the tier still `asked` built — or failed — from a build
|
// settleFromRecords marks every module of the tier still `asked` built — or failed — from a build
|
||||||
// recorded after it was asked, and says whether it changed anything (novox/hq 04-ISSUES/214).
|
// recorded after it was asked, and says whether it changed anything (novox/hq 04-ISSUES/214).
|
||||||
// Newest first, as Builds answers: the first record after the ask is the outcome of that ask.
|
// Newest first, as Builds answers: the first record after the ask is the outcome of that ask.
|
||||||
|
|||||||
@@ -160,7 +160,6 @@ func (b *backlogMesh) release(t *testing.T) inventory.Plan {
|
|||||||
// The backlog goes out one machine at a time: the first judged before the second is sent, each build's
|
// The backlog goes out one machine at a time: the first judged before the second is sent, each build's
|
||||||
// pass kept; a rebuild with the same bytes is no move at all; and a send outside a gate is refused.
|
// pass kept; a rebuild with the same bytes is no move at all; and a send outside a gate is refused.
|
||||||
func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) {
|
func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) {
|
||||||
judgeLeavesAsTheCore(t)
|
|
||||||
b := aBacklog(t)
|
b := aBacklog(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
inv := b.open.inventory
|
inv := b.open.inventory
|
||||||
|
|||||||
@@ -77,7 +77,6 @@ func testAWalkStoppedAfterItsGatePassedKeepsThePass(t *testing.T) {
|
|||||||
// the first send is older than the wait for a first machine's report. Read before the pass is taken over, the
|
// the first send is older than the wait for a first machine's report. Read before the pass is taken over, the
|
||||||
// carried module's step said the first machine never applied it, and the passed build was put back.
|
// carried module's step said the first machine never applied it, and the passed build was put back.
|
||||||
func TestACarriedModuleTakesItsLeadsPassBeforeItsStepIsRead(t *testing.T) {
|
func TestACarriedModuleTakesItsLeadsPassBeforeItsStepIsRead(t *testing.T) {
|
||||||
judgeLeavesAsTheCore(t)
|
|
||||||
tm := aTierMesh(t, "m01", "m02")
|
tm := aTierMesh(t, "m01", "m02")
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
inv := tm.open.inventory
|
inv := tm.open.inventory
|
||||||
|
|||||||
@@ -55,15 +55,6 @@ func rotateCommand(ctx context.Context, args []string) error {
|
|||||||
defer open.Close()
|
defer open.Close()
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
|
|
||||||
// A credential the provider's software issued is not the mesh's to remake (novox/hq ADR 0305 §9).
|
|
||||||
shelf, err := inv.Catalogue(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := issuedRefusesRotation(shelf, provision); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
holders, err := inv.HoldersOf(ctx, provision, *only)
|
holders, err := inv.HoldersOf(ctx, provision, *only)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -162,7 +162,6 @@ func TestABusRebuiltFromAnUnchangedSourceDemandsNoBusStep(t *testing.T) {
|
|||||||
// A plan's build made from the source every machine running the module runs is not sent and not
|
// A plan's build made from the source every machine running the module runs is not sent and not
|
||||||
// judged: nothing moves. A build of a changed source is sent to its first machine, gated, as before.
|
// judged: nothing moves. A build of a changed source is sent to its first machine, gated, as before.
|
||||||
func TestAPlanSendsNothingForAnUnchangedSource(t *testing.T) {
|
func TestAPlanSendsNothingForAnUnchangedSource(t *testing.T) {
|
||||||
judgeLeavesAsTheCore(t)
|
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name string
|
name string
|
||||||
source string
|
source string
|
||||||
|
|||||||
@@ -410,10 +410,6 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
|||||||
if b := str("by"); b != "" {
|
if b := str("by"); b != "" {
|
||||||
argv = append(argv, "--by", catalogue.DeliverySeat+" for "+b)
|
argv = append(argv, "--by", catalogue.DeliverySeat+" for "+b)
|
||||||
}
|
}
|
||||||
// The merges to leave out: the walk's other merges are walked again without them (novox/hq ADR 0299).
|
|
||||||
if w := str("without"); w != "" {
|
|
||||||
argv = append(argv, "--without", w)
|
|
||||||
}
|
|
||||||
return argv, nil
|
return argv, nil
|
||||||
case "delivery-walks":
|
case "delivery-walks":
|
||||||
argv := []string{"delivery", "walks"}
|
argv := []string{"delivery", "walks"}
|
||||||
@@ -777,13 +773,6 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return []string{"secret", "ask", str("node"), str("module"), str("secret"), "--at", str("at")}, nil
|
return []string{"secret", "ask", str("node"), str("module"), str("secret"), "--at", str("at")}, nil
|
||||||
case "issued":
|
|
||||||
// The credential a provider's software issued, taken from the provider (novox/hq ADR 0305): no value in
|
|
||||||
// the arguments, the provision alone.
|
|
||||||
if err := need("provision"); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return []string{"secret", "issued", str("provision")}, nil
|
|
||||||
case "secret-ask":
|
case "secret-ask":
|
||||||
// A module's own secret asked for, typed by the operator at the desk (novox/hq ADR 0277): never a value
|
// A module's own secret asked for, typed by the operator at the desk (novox/hq ADR 0277): never a value
|
||||||
// in the arguments. The desk is the module's machine unless at names another.
|
// in the arguments. The desk is the module's machine unless at names another.
|
||||||
@@ -1377,7 +1366,7 @@ func splitCommandLine(line string) ([]string, error) {
|
|||||||
var words []string
|
var words []string
|
||||||
var cur strings.Builder
|
var cur strings.Builder
|
||||||
inWord := false
|
inWord := false
|
||||||
quote, opened := rune(0), 0
|
quote := rune(0)
|
||||||
runes := []rune(line)
|
runes := []rune(line)
|
||||||
for i := 0; i < len(runes); i++ {
|
for i := 0; i < len(runes); i++ {
|
||||||
r := runes[i]
|
r := runes[i]
|
||||||
@@ -1398,7 +1387,7 @@ func splitCommandLine(line string) ([]string, error) {
|
|||||||
cur.WriteRune(r)
|
cur.WriteRune(r)
|
||||||
}
|
}
|
||||||
case r == '\'' || r == '"':
|
case r == '\'' || r == '"':
|
||||||
quote, opened = r, i
|
quote = r
|
||||||
inWord = true
|
inWord = true
|
||||||
case r == '\\' && i+1 < len(runes):
|
case r == '\\' && i+1 < len(runes):
|
||||||
i++
|
i++
|
||||||
@@ -1416,7 +1405,7 @@ func splitCommandLine(line string) ([]string, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if quote != 0 {
|
if quote != 0 {
|
||||||
return nil, unclosedQuote(quote, opened)
|
return nil, fmt.Errorf("command has an unclosed %c quote", quote)
|
||||||
}
|
}
|
||||||
if inWord {
|
if inWord {
|
||||||
words = append(words, cur.String())
|
words = append(words, cur.String())
|
||||||
@@ -1424,17 +1413,6 @@ func splitCommandLine(line string) ([]string, error) {
|
|||||||
return words, nil
|
return words, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// unclosedQuote is the refusal of a line whose quote is never closed. Most often an apostrophe inside
|
|
||||||
// a single-quoted value ended that quote early and a later quote was left open, so the refusal says
|
|
||||||
// where the open quote is and how a quote is written inside a quoted value — the shell's own two
|
|
||||||
// ways, which this splitter already reads (novox/hq issue 294). It quotes none of the line: a refusal
|
|
||||||
// is kept on the bus as the call's answer, and the line may carry a setting's value.
|
|
||||||
func unclosedQuote(quote rune, opened int) error {
|
|
||||||
return fmt.Errorf("command has an unclosed %c quote, opened at character %d — an apostrophe "+
|
|
||||||
"inside a single-quoted value ends it; write a ' inside single quotes as '\\'' (it'\\''s), or use "+
|
|
||||||
"double quotes and write \\\" for a \" and \\\\ for a \\ inside them", quote, opened+1)
|
|
||||||
}
|
|
||||||
|
|
||||||
// seatAnnouncement is what the controller says it serves on the bus (novox/hq ADR 0197): the
|
// seatAnnouncement is what the controller says it serves on the bus (novox/hq ADR 0197): the
|
||||||
// mesh-controller seat, one endpoint per verb it answers, each with the seat's own description and
|
// mesh-controller seat, one endpoint per verb it answers, each with the seat's own description and
|
||||||
// argument schema — the same facts `tools` answers from the records, as NATS's services format.
|
// argument schema — the same facts `tools` answers from the records, as NATS's services format.
|
||||||
@@ -1590,14 +1568,6 @@ func givenAtTheDesk(argv []string) bool {
|
|||||||
return argv[5] == "--at" && argv[6] != "" && !strings.HasPrefix(argv[6], "-")
|
return argv[5] == "--at" && argv[6] != "" && !strings.HasPrefix(argv[6], "-")
|
||||||
}
|
}
|
||||||
|
|
||||||
// takenFromTheProvider is exactly the line the `issued` verb composes, and nothing beside it: `secret issued
|
|
||||||
// <provision>` (novox/hq ADR 0305 §10). It carries no value — the value comes from the provider's own tool, sealed to
|
|
||||||
// the call — and answers none.
|
|
||||||
func takenFromTheProvider(argv []string) bool {
|
|
||||||
return len(argv) == 3 && argv[0] == "secret" && argv[1] == "issued" && argv[2] != "" &&
|
|
||||||
!strings.HasPrefix(argv[2], "-")
|
|
||||||
}
|
|
||||||
|
|
||||||
// terminalOnly refuses, through any verb, a command that is the operator's at the controller's terminal
|
// terminalOnly refuses, through any verb, a command that is the operator's at the controller's terminal
|
||||||
// alone (novox/hq ADR 0266). **Every `node` subcommand that is not a read**: `node account` and
|
// alone (novox/hq ADR 0266). **Every `node` subcommand that is not a read**: `node account` and
|
||||||
// `node agent-account` above all. Whoever may call a verb includes agents, and an agent that named itself
|
// `node agent-account` above all. Whoever may call a verb includes agents, and an agent that named itself
|
||||||
@@ -1614,7 +1584,7 @@ func terminalOnly(argv []string) error {
|
|||||||
// Of a secret's commands only rotation, which seals the new value to the machine that uses it, and the
|
// Of a secret's commands only rotation, which seals the new value to the machine that uses it, and the
|
||||||
// `give` verb's own line: an own secret typed by the operator into the desk's hidden prompt, sealed to this
|
// `give` verb's own line: an own secret typed by the operator into the desk's hidden prompt, sealed to this
|
||||||
// call and then to the module's machine, so no value travels in the verb or its answer (hq ADR 0259 §10).
|
// call and then to the module's machine, so no value travels in the verb or its answer (hq ADR 0259 §10).
|
||||||
if argv[0] == "secret" && (len(argv) < 2 || argv[1] != "rotate") && !givenAtTheDesk(argv) && !takenFromTheProvider(argv) {
|
if argv[0] == "secret" && (len(argv) < 2 || argv[1] != "rotate") && !givenAtTheDesk(argv) {
|
||||||
return terminalRefusal("secret %s is run at the controller's terminal only, never through a verb: accepting, "+
|
return terminalRefusal("secret %s is run at the controller's terminal only, never through a verb: accepting, "+
|
||||||
"recovering or exporting a secret hands it to whoever asks, and that includes agents (novox/hq ADR "+
|
"recovering or exporting a secret hands it to whoever asks, and that includes agents (novox/hq ADR "+
|
||||||
"0266). Nothing was done", strings.Join(argv[1:], " "))
|
"0266). Nothing was done", strings.Join(argv[1:], " "))
|
||||||
|
|||||||
@@ -415,48 +415,3 @@ func TestNoVerbSetsTheOperatorsKeyOrRevealsASecret(t *testing.T) {
|
|||||||
t.Fatalf("behind %v: %v", behind, err)
|
t.Fatalf("behind %v: %v", behind, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A value with a quote in it can be said on a `command` line, as in a shell, and a line whose quote
|
|
||||||
// is left open is refused naming where it opened and how a quote is written, quoting none of it (novox/hq issue 294: an
|
|
||||||
// apostrophe inside a single-quoted JSON value cut the line, and the refusal named no cause).
|
|
||||||
func TestAQuotedValueCanHoldAQuote(t *testing.T) {
|
|
||||||
for _, c := range []struct{ line, want string }{
|
|
||||||
{`settings set claude-code '{"role":"the operator'\''s laptop"}'`, `{"role":"the operator's laptop"}`},
|
|
||||||
{`settings set claude-code "{\"role\":\"the operator's laptop\"}"`, `{"role":"the operator's laptop"}`},
|
|
||||||
{"x \"a \\\\ b\nc\"", "a \\ b\nc"},
|
|
||||||
{`x 'a\b'`, `a\b`},
|
|
||||||
} {
|
|
||||||
argv, err := splitCommandLine(c.line)
|
|
||||||
if err != nil || argv[len(argv)-1] != c.want {
|
|
||||||
t.Errorf("%s: read back %q %v, want %q", c.line, argv, err, c.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := splitCommandLine(`settings set claude-code '{"role":"the operator's laptop"}' --node g14`)
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("an apostrophe that leaves a quote open was accepted")
|
|
||||||
}
|
|
||||||
for _, want := range []string{"character 57", `'\''`, `\"`} {
|
|
||||||
if !strings.Contains(err.Error(), want) {
|
|
||||||
t.Errorf("the refusal does not say %q: %v", want, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if strings.Contains(err.Error(), "laptop") || strings.Contains(err.Error(), "g14") {
|
|
||||||
t.Errorf("the refusal quotes the line, which may carry a setting's value: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Every value reads back as it was when written the way the refusal says: single-quoted with '\”
|
|
||||||
// for each quote, or double-quoted with \ before each " and \ — newlines and backslashes included.
|
|
||||||
func TestAQuotedValueRoundTrips(t *testing.T) {
|
|
||||||
for _, v := range []string{`plain`, `it's`, `say "hi"`, `back\slash\`, "two\nlines", `'"\'\"`, `''`, ``} {
|
|
||||||
single := "x '" + strings.ReplaceAll(v, "'", `'\''`) + "'"
|
|
||||||
double := `x "` + strings.NewReplacer(`\`, `\\`, `"`, `\"`).Replace(v) + `"`
|
|
||||||
for _, line := range []string{single, double} {
|
|
||||||
argv, err := splitCommandLine(line)
|
|
||||||
if err != nil || len(argv) != 2 || argv[1] != v {
|
|
||||||
t.Errorf("%s: read back %q %v, want %q", line, argv, err, v)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -41,9 +41,6 @@ func secretCommand(ctx context.Context, args []string) error {
|
|||||||
case "accept":
|
case "accept":
|
||||||
case "ask":
|
case "ask":
|
||||||
return secretAsk(ctx, args[1:])
|
return secretAsk(ctx, args[1:])
|
||||||
case "issued":
|
|
||||||
// The credential a provider's software issued, taken from the provider (novox/hq ADR 0305).
|
|
||||||
return secretIssued(ctx, args[1:])
|
|
||||||
case "rotate":
|
case "rotate":
|
||||||
return secretRotate(ctx, args[1:])
|
return secretRotate(ctx, args[1:])
|
||||||
case "recover":
|
case "recover":
|
||||||
@@ -171,7 +168,6 @@ func secretAsk(ctx context.Context, args []string) error {
|
|||||||
|
|
||||||
const secretUsage = "secret rotate <node> <module> <name> [--why <text> [--cause <word>]]\n" +
|
const secretUsage = "secret rotate <node> <module> <name> [--why <text> [--cause <word>]]\n" +
|
||||||
"secret ask <node> <module> <name> [--at <machine>]\n" +
|
"secret ask <node> <module> <name> [--at <machine>]\n" +
|
||||||
"secret issued <provision>\n" +
|
|
||||||
"secret accept <node> <module> <name> [--from <file> | --at-desk <machine>] [--provider <node> [--local <name>]]\n" +
|
"secret accept <node> <module> <name> [--from <file> | --at-desk <machine>] [--provider <node> [--local <name>]]\n" +
|
||||||
"secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>] [--provider <node>]\n" +
|
"secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>] [--provider <node>]\n" +
|
||||||
"secret export [--out <file>]"
|
"secret export [--out <file>]"
|
||||||
|
|||||||
@@ -27,19 +27,6 @@ type sendable struct {
|
|||||||
// said it reads one is sent none, because an older node-engine refuses a key it does not know, whole
|
// said it reads one is sent none, because an older node-engine refuses a key it does not know, whole
|
||||||
// (link/order.go, the contract).
|
// (link/order.go, the contract).
|
||||||
Epoch uint64
|
Epoch uint64
|
||||||
// Generation is the assignment generation it was composed from (novox/hq issue 234): a machine that
|
|
||||||
// applied a later one refuses it, so a send composed from a view of the assignments the mesh has moved
|
|
||||||
// past cannot undeclare what is still assigned. Zero is not sent at all — every machine whose
|
|
||||||
// node-engine has not said it reads one is sent none, for the reason the epoch is not (an older
|
|
||||||
// node-engine refuses a key it does not know, whole).
|
|
||||||
Generation int64
|
|
||||||
// composedFrom is the generation read before this declaration was composed, and actingEpoch the
|
|
||||||
// lease epoch its sender acted under — whether or not the machine is sent either — for the record of
|
|
||||||
// the send; never on the wire.
|
|
||||||
composedFrom int64
|
|
||||||
actingEpoch uint64
|
|
||||||
// modules are the modules this declaration names, for the record of the send; never on the wire.
|
|
||||||
modules []string
|
|
||||||
// Adoption is nil for a converged node, and then the body is byte for byte what it was before
|
// Adoption is nil for a converged node, and then the body is byte for byte what it was before
|
||||||
// adoption existed: an older host parses the envelope strictly and would refuse the key.
|
// adoption existed: an older host parses the envelope strictly and would refuse the key.
|
||||||
Adoption *adoptionEnvelope
|
Adoption *adoptionEnvelope
|
||||||
@@ -107,9 +94,6 @@ func (s sendable) Body() ([]byte, error) {
|
|||||||
if len(s.LeftOut) > 0 {
|
if len(s.LeftOut) > 0 {
|
||||||
envelope["left_out"] = s.LeftOut
|
envelope["left_out"] = s.LeftOut
|
||||||
}
|
}
|
||||||
if s.Generation > 0 {
|
|
||||||
envelope["generation"] = s.Generation
|
|
||||||
}
|
|
||||||
// An empty declaration is deliberate here — the node owns nothing the mesh put there
|
// An empty declaration is deliberate here — the node owns nothing the mesh put there
|
||||||
// (novox/hq issue 127) — and the host refuses an empty body unless it is told the emptiness
|
// (novox/hq issue 127) — and the host refuses an empty body unless it is told the emptiness
|
||||||
// is meant, so a truncated or mis-composed body is never mistaken for "own nothing".
|
// is meant, so a truncated or mis-composed body is never mistaken for "own nothing".
|
||||||
|
|||||||
@@ -226,20 +226,6 @@ var signalsTable = []signalRow{
|
|||||||
newest: func(f *signalFacts) time.Time {
|
newest: func(f *signalFacts) time.Time {
|
||||||
return newestOf(f.batches, func(b batchFacts) time.Time { return b.closed })
|
return newestOf(f.batches, func(b batchFacts) time.Time { return b.closed })
|
||||||
}},
|
}},
|
||||||
{Row: "S20", Signal: "a send refused for the assignment generation it was composed from",
|
|
||||||
Emitter: "node-engine", Trigger: "each refusal (novox/hq issue 234)",
|
|
||||||
Bound: "none: raised at the first refusal, naming its sender, the generation it came from and the one the " +
|
|
||||||
"machine applied; cleared an hour after the last",
|
|
||||||
Kind: kindOlderGeneration, Severity: conditions.Warning, Phase: 2,
|
|
||||||
needs: func(f *signalFacts) error { return f.refusedSendsErr }, watch: watchGenerationRefusals,
|
|
||||||
newest: func(f *signalFacts) time.Time {
|
|
||||||
return newestOf(f.refusedSends, func(s inventory.Send) time.Time {
|
|
||||||
if s.RefusedAt == nil {
|
|
||||||
return time.Time{}
|
|
||||||
}
|
|
||||||
return *s.RefusedAt
|
|
||||||
})
|
|
||||||
}},
|
|
||||||
{Row: "S17", Signal: "a send held for the bus's planned step is told to a person", Emitter: "controller's plan",
|
{Row: "S17", Signal: "a send held for the bus's planned step is told to a person", Emitter: "controller's plan",
|
||||||
Trigger: "each send refused because it would replace the bus outside its step (novox/hq issue 336)",
|
Trigger: "each send refused because it would replace the bus outside its step (novox/hq issue 336)",
|
||||||
Bound: "none: raised at the first refusal, for the operator, naming what waits, the bus build from and to, " +
|
Bound: "none: raised at the first refusal, for the operator, naming what waits, the bus build from and to, " +
|
||||||
@@ -537,14 +523,6 @@ func watchAdvisories(f *signalFacts) []conditions.Observation {
|
|||||||
if a.Kind == link.AdvisoryConsumerLost && !f.lostConsumers[a.Stream+"."+a.Consumer] {
|
if a.Kind == link.AdvisoryConsumerLost && !f.lostConsumers[a.Stream+"."+a.Consumer] {
|
||||||
continue // it exists again, or the mesh no longer expects it: a removal, not a loss
|
continue // it exists again, or the mesh no longer expects it: a removal, not a loss
|
||||||
}
|
}
|
||||||
if a.Kind == link.AdvisoryMaxDeliveries && a.Token == "" {
|
|
||||||
// A consumer's max-deliveries key is the dead-letter row's (novox/hq issue 330): said while
|
|
||||||
// DEAD_LETTERS holds what it gave up on, cleared when that is delivered again or dropped. The
|
|
||||||
// listener records no such advisory today; one read here as well added a look to the count and
|
|
||||||
// overwrote the row's words on every look (novox/hq issue 440). Only one that could not be kept
|
|
||||||
// (AdvisoryNotKept), which DEAD_LETTERS cannot say, is said from here.
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
severity := conditions.Warning
|
severity := conditions.Warning
|
||||||
times := ""
|
times := ""
|
||||||
if a.Count > 1 {
|
if a.Count > 1 {
|
||||||
@@ -555,10 +533,7 @@ func watchAdvisories(f *signalFacts) []conditions.Observation {
|
|||||||
machine = f.host
|
machine = f.host
|
||||||
}
|
}
|
||||||
o := conditions.Observation{Scope: conditions.ScopeBus, ID: a.ID, Kind: a.Kind, Token: a.Token,
|
o := conditions.Observation{Scope: conditions.ScopeBus, ID: a.ID, Kind: a.Kind, Token: a.Token,
|
||||||
Machine: machine, Severity: severity, Summary: a.Said + times, Said: a.Said,
|
Machine: machine, Severity: severity, Summary: a.Said + times, Said: a.Said}
|
||||||
// The advisory is remembered and read again on every look for an hour: the condition counts
|
|
||||||
// what the bus said and when, not the looks (novox/hq issue 402).
|
|
||||||
Happened: a.Last, Times: a.Count}
|
|
||||||
if a.Kind == link.AdvisoryMaxDeliveries && a.Token == link.AdvisoryNotKept {
|
if a.Kind == link.AdvisoryMaxDeliveries && a.Token == link.AdvisoryNotKept {
|
||||||
o.Machine = consumerMachine(a.Stream, a.Consumer)
|
o.Machine = consumerMachine(a.Stream, a.Consumer)
|
||||||
o.Headline = clip(conditions.Capital(fmt.Sprintf("%s gave up on a message, not kept",
|
o.Headline = clip(conditions.Capital(fmt.Sprintf("%s gave up on a message, not kept",
|
||||||
@@ -590,17 +565,7 @@ func watchDeadLetters(f *signalFacts) []conditions.Observation {
|
|||||||
messages, them = fmt.Sprintf("%d messages", n), "them"
|
messages, them = fmt.Sprintf("%d messages", n), "them"
|
||||||
}
|
}
|
||||||
who := consumerWho(stream, consumer)
|
who := consumerWho(stream, consumer)
|
||||||
// DEAD_LETTERS is a record of what was given up on: the condition says when the newest held message
|
|
||||||
// was kept, and its count is a running total of the messages given up on while it is open, never
|
|
||||||
// how often the controller looked (novox/hq issues 402 and 440). It is at least the number held now,
|
|
||||||
// and does not go down when one is delivered again or dropped. Without that time it counts its
|
|
||||||
// looks, as a source of the present does.
|
|
||||||
happened, times := f.deadLettersNewest[key], 0
|
|
||||||
if !happened.IsZero() {
|
|
||||||
times = n
|
|
||||||
}
|
|
||||||
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: key, Kind: link.AdvisoryMaxDeliveries,
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: key, Kind: link.AdvisoryMaxDeliveries,
|
||||||
Happened: happened, Times: times,
|
|
||||||
Machine: consumerMachine(stream, consumer), Severity: conditions.Warning,
|
Machine: consumerMachine(stream, consumer), Severity: conditions.Warning,
|
||||||
Summary: fmt.Sprintf("%s gave up on %s; %s kept in %s until delivered again or dropped, with why, "+
|
Summary: fmt.Sprintf("%s gave up on %s; %s kept in %s until delivered again or dropped, with why, "+
|
||||||
"through the controller's dead-letters verb", link.ConsumerInWords(stream, consumer), messages,
|
"through the controller's dead-letters verb", link.ConsumerInWords(stream, consumer), messages,
|
||||||
|
|||||||
@@ -100,15 +100,14 @@ var suppressions = map[string]suppression{
|
|||||||
inside: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-29 * time.Minute))} },
|
inside: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-29 * time.Minute))} },
|
||||||
past: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-31 * time.Minute))} },
|
past: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-31 * time.Minute))} },
|
||||||
},
|
},
|
||||||
// A message given up on and not kept: the one max-deliveries advisory S9 says itself (issue 440).
|
|
||||||
"S9": {
|
"S9": {
|
||||||
inside: func(f *signalFacts) {
|
inside: func(f *signalFacts) {
|
||||||
f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop",
|
f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop", Said: "gave up",
|
||||||
Token: link.AdvisoryNotKept, Said: "gave up, not kept", First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-61 * time.Minute), Count: 1}}
|
First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-61 * time.Minute), Count: 1}}
|
||||||
},
|
},
|
||||||
past: func(f *signalFacts) {
|
past: func(f *signalFacts) {
|
||||||
f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop",
|
f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop", Said: "gave up",
|
||||||
Token: link.AdvisoryNotKept, Said: "gave up, not kept", First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-59 * time.Minute), Count: 1}}
|
First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-59 * time.Minute), Count: 1}}
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"S10": {
|
"S10": {
|
||||||
@@ -178,14 +177,6 @@ var suppressions = map[string]suppression{
|
|||||||
commit: "c0ffee001122", modules: []string{"app"}, since: f.now.Add(-time.Second)}}}
|
commit: "c0ffee001122", modules: []string{"app"}, since: f.now.Add(-time.Second)}}}
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
// A send refused by its machine for the generation it was composed from (novox/hq issue 234): said at
|
|
||||||
// once, naming its sender, for an hour after. Inside: the last such refusal was more than an hour ago.
|
|
||||||
"S20": {
|
|
||||||
inside: func(f *signalFacts) {
|
|
||||||
f.refusedSends = []inventory.Send{refusedSend(f.now.Add(-61 * time.Minute))}
|
|
||||||
},
|
|
||||||
past: func(f *signalFacts) { f.refusedSends = []inventory.Send{refusedSend(f.now.Add(-time.Second))} },
|
|
||||||
},
|
|
||||||
// Twice by hand within a fortnight is a healer wanted; once, or the first of two a day too old, is not.
|
// Twice by hand within a fortnight is a healer wanted; once, or the first of two a day too old, is not.
|
||||||
"S15": {
|
"S15": {
|
||||||
inside: func(f *signalFacts) {
|
inside: func(f *signalFacts) {
|
||||||
|
|||||||
@@ -66,7 +66,7 @@ func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
|
|||||||
}
|
}
|
||||||
// Both machines' `node show` name it: where the provider runs, and where the consumer is.
|
// Both machines' `node show` name it: where the provider runs, and where the consumer is.
|
||||||
for _, node := range []string{"anchor", "laptop"} {
|
for _, node := range []string{"anchor", "laptop"} {
|
||||||
shown := printed(t, func() error { return showNode(ctx, open, node) })
|
shown := printed(t, func() error { return showNode(ctx, open.inventory, node) })
|
||||||
if !strings.Contains(shown, "open condition(s) about this machine") || !strings.Contains(shown, "mesh_laptop_dashboard") {
|
if !strings.Contains(shown, "open condition(s) about this machine") || !strings.Contains(shown, "mesh_laptop_dashboard") {
|
||||||
t.Fatalf("node show %s does not name it:\n%s", node, shown)
|
t.Fatalf("node show %s does not name it:\n%s", node, shown)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -289,10 +289,6 @@ func printStatus(asked answers) error {
|
|||||||
fmt.Printf("\n `take <node> <module>` compares what runs against what it declares, and runs it\n\n")
|
fmt.Printf("\n `take <node> <module>` compares what runs against what it declares, and runs it\n\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Assigned and not applied (novox/hq issue 380): before what is merely reported, because it reads like work
|
|
||||||
// finished and is none.
|
|
||||||
printNotApplied(asked.leftOut)
|
|
||||||
|
|
||||||
if len(asked.unheld) > 0 {
|
if len(asked.unheld) > 0 {
|
||||||
// **Reported, and not refused yet** (novox/hq ADR 0207 §4). Each machine still resolves and
|
// **Reported, and not refused yet** (novox/hq ADR 0207 §4). Each machine still resolves and
|
||||||
// is sent what it would be; this says which of its modules depend on a seat nothing there
|
// is sent what it would be; this says which of its modules depend on a seat nothing there
|
||||||
@@ -341,10 +337,6 @@ func printStatus(asked answers) error {
|
|||||||
"lists them, and each is in its condition's tried\n\n", asked.heals.Acts, asked.heals.Escalated)
|
"lists them, and each is in its condition's tried\n\n", asked.heals.Acts, asked.heals.Escalated)
|
||||||
}
|
}
|
||||||
|
|
||||||
// What each machine was last sent and by whom, one line each (novox/hq issue 485). Not a fault, so it does not
|
|
||||||
// break "all well"; a record that could not be read does.
|
|
||||||
printLastSends(nodes, asked.lastSent, asked.lastSentUnread)
|
|
||||||
|
|
||||||
if adopted := adoptedNodes(nodes); len(adopted) > 0 {
|
if adopted := adoptedNodes(nodes); len(adopted) > 0 {
|
||||||
// Said, because nothing forces the flip: a node left adopted is visible here rather than
|
// Said, because nothing forces the flip: a node left adopted is visible here rather than
|
||||||
// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
|
// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
|
||||||
@@ -464,13 +456,6 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
|||||||
return answers{}, err
|
return answers{}, err
|
||||||
}
|
}
|
||||||
plans[n.Name] = planned{plan, settings}
|
plans[n.Name] = planned{plan, settings}
|
||||||
// And which of its modules a push leaves out (novox/hq issue 380), judged as the push judges it.
|
|
||||||
if left := leftOutOf(plan, settings, n.Adopted); len(left) > 0 {
|
|
||||||
if out.leftOut == nil {
|
|
||||||
out.leftOut = map[string][]leftOutModule{}
|
|
||||||
}
|
|
||||||
out.leftOut[n.Name] = left
|
|
||||||
}
|
|
||||||
out.unheld = append(out.unheld, plan.Unheld...)
|
out.unheld = append(out.unheld, plan.Unheld...)
|
||||||
// And which of its modules a provider leaves out of its grants, for an identity too long
|
// And which of its modules a provider leaves out of its grants, for an identity too long
|
||||||
// for what the provision keeps (novox/hq ADR 0225) — judged from the consumer's own
|
// for what the provision keeps (novox/hq ADR 0225) — judged from the consumer's own
|
||||||
@@ -489,12 +474,6 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
|||||||
out.pendingUnread = err.Error()
|
out.pendingUnread = err.Error()
|
||||||
err = nil
|
err = nil
|
||||||
}
|
}
|
||||||
// And what each machine was last sent, by whom (novox/hq issue 485): the record of sends is read through
|
|
||||||
// the controller's verbs alone, the store's reader being shut out of its tables. Unreadable is said.
|
|
||||||
if out.lastSent, err = inv.LastSends(ctx); err != nil {
|
|
||||||
out.lastSentUnread = err.Error()
|
|
||||||
err = nil
|
|
||||||
}
|
|
||||||
out.plans, err = inv.RecentPlans(ctx, 5)
|
out.plans, err = inv.RecentPlans(ctx, 5)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return answers{}, err
|
return answers{}, err
|
||||||
@@ -625,10 +604,9 @@ func untakenModules(ctx context.Context, inv *inventory.Inventory, nodes []inven
|
|||||||
// read as success for the whole of the edge cut-over outage (novox/hq 04-ISSUES/125).
|
// read as success for the whole of the edge cut-over outage (novox/hq 04-ISSUES/125).
|
||||||
func (a answers) well() bool {
|
func (a answers) well() bool {
|
||||||
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
|
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
|
||||||
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 && len(a.leftOut) == 0 &&
|
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 &&
|
||||||
len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.overflowing) == 0 &&
|
len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.overflowing) == 0 &&
|
||||||
len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending) &&
|
len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending)
|
||||||
a.lastSentUnread == ""
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// pendingOpen is whether any pending assignment is still to be made. One that ended without being made is
|
// pendingOpen is whether any pending assignment is still to be made. One that ended without being made is
|
||||||
|
|||||||
@@ -196,11 +196,6 @@ func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, e
|
|||||||
if report.Ordered() {
|
if report.Ordered() {
|
||||||
link.StaleRefusals.Lifetime(report.Node, report.RefusedOlder, now)
|
link.StaleRefusals.Lifetime(report.Node, report.RefusedOlder, now)
|
||||||
}
|
}
|
||||||
// A send refused for the generation it was composed from is kept on the send it refused, so S20 names
|
|
||||||
// its sender (novox/hq issue 234).
|
|
||||||
if report.OlderGeneration != nil {
|
|
||||||
heardGenerationRefusal(ctx, l.Enrolment.Inventory, report)
|
|
||||||
}
|
|
||||||
// What the machine's witnesses put back and stand by (novox/hq ADR 0236): read by the gate and its
|
// What the machine's witnesses put back and stand by (novox/hq ADR 0236): read by the gate and its
|
||||||
// probe. Only from an account of the machine — not a word that a declaration was set aside, nor a rekey.
|
// probe. Only from an account of the machine — not a word that a declaration was set aside, nor a rekey.
|
||||||
if report.Superseded == "" && report.Rekey == nil && report.Node != "" {
|
if report.Superseded == "" && report.Rekey == nil && report.Node != "" {
|
||||||
|
|||||||
@@ -1,142 +0,0 @@
|
|||||||
{
|
|
||||||
"module": "nfs-server",
|
|
||||||
"version": "1",
|
|
||||||
"upgrade": {
|
|
||||||
"policy": "record",
|
|
||||||
"why": "the folders other machines mount: a build that breaks the exports leaves every client's mount hanging or refused, and the gate on this one machine does not see the clients (hq ADR 0236, ADR 0263)"
|
|
||||||
},
|
|
||||||
"capabilities": [
|
|
||||||
"package-manager",
|
|
||||||
"service-manager"
|
|
||||||
],
|
|
||||||
"provides": [
|
|
||||||
{
|
|
||||||
"name": "nfs-share",
|
|
||||||
"scope": "mesh",
|
|
||||||
"identity": false
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"data": {
|
|
||||||
"consumers": {
|
|
||||||
"nfs-share": {
|
|
||||||
"class": "none",
|
|
||||||
"why": "the shared folders are the operator's data (hq ADR 0051): what a client writes lands in them, and they are protected where the operator declares them, never by this module, which keeps nothing of a consumer's"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"claims": [
|
|
||||||
{
|
|
||||||
"name": "node-nfs-server",
|
|
||||||
"scope": "node",
|
|
||||||
"serves": [
|
|
||||||
"exports",
|
|
||||||
"clients",
|
|
||||||
"test",
|
|
||||||
"reload",
|
|
||||||
"adopt"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"state": [
|
|
||||||
{
|
|
||||||
"name": "exports",
|
|
||||||
"ttl-seconds": 120,
|
|
||||||
"per-machine": true
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"tools": [
|
|
||||||
"nfs_health"
|
|
||||||
],
|
|
||||||
"listens": [
|
|
||||||
{
|
|
||||||
"name": "nfs",
|
|
||||||
"port": 2049,
|
|
||||||
"protocol": "tcp",
|
|
||||||
"from": "mesh",
|
|
||||||
"fixed": true,
|
|
||||||
"why": "the shares, to the mesh's machines only (hq ADR 0263): NFS version 4 alone, which needs no other port, and never the home network, where a device that is not a node could claim any user id"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"resources": [
|
|
||||||
{
|
|
||||||
"id": "package",
|
|
||||||
"type": "package",
|
|
||||||
"package": "nfs-utils"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "nfs-conf",
|
|
||||||
"type": "file",
|
|
||||||
"path": "/etc/nfs.conf.d/50-mesh.conf",
|
|
||||||
"mode": "0644",
|
|
||||||
"content": "# Written by the mesh (module nfs-server, novox/hq ADR 0263). Replaced on every push; a drop-in of\n# the operator's that sorts after this one overrides it, and is theirs.\n#\n# NFS version 4 only: a client needs port 2049 and nothing else, so the module opens nothing more\n# than that, to the private network. Version 3 needs rpcbind and mountd, on ports the mesh does not open.\n[nfsd]\nvers2=n\nvers3=n\nvers4=y\nvers4.0=n\nvers4.1=y\nvers4.2=y\n"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "config-dir",
|
|
||||||
"type": "directory",
|
|
||||||
"path": "/etc/nfs-server",
|
|
||||||
"mode": "0755"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "config",
|
|
||||||
"type": "file",
|
|
||||||
"path": "/etc/nfs-server/shares.conf",
|
|
||||||
"mode": "0644",
|
|
||||||
"content": "# Written by the mesh (module nfs-server, novox/hq ADR 0263) from this machine's assignment.\n# Replaced on every push; change the `shares` setting, never this file.\n#\n# The shares: name=folder, or name=folder:ro, one share per folder. The module's process exports each\n# to the private network's range below, every client mapped to the folder's owner.\nshares=${setting:shares}\nrange=${machine:mesh-range}\n"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "run-dir",
|
|
||||||
"type": "directory",
|
|
||||||
"path": "/run/nfs-server",
|
|
||||||
"mode": "0755"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "server",
|
|
||||||
"type": "service",
|
|
||||||
"unit": "nfs-server.service",
|
|
||||||
"state": "running",
|
|
||||||
"boot": "enabled",
|
|
||||||
"restart-on": [
|
|
||||||
"nfs-conf"
|
|
||||||
],
|
|
||||||
"health": {
|
|
||||||
"kind": "unit"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "exports",
|
|
||||||
"type": "process",
|
|
||||||
"name": "nfs-server-exports",
|
|
||||||
"artifact": "tools",
|
|
||||||
"run": [
|
|
||||||
"./nfs-server",
|
|
||||||
"exports"
|
|
||||||
],
|
|
||||||
"restart-on": [
|
|
||||||
"config"
|
|
||||||
],
|
|
||||||
"health": {
|
|
||||||
"kind": "tool",
|
|
||||||
"tool": "nfs_health",
|
|
||||||
"interval": "60s",
|
|
||||||
"timeout": "10s",
|
|
||||||
"looks": 2,
|
|
||||||
"grace": "90s"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"build": {
|
|
||||||
"artifacts": [
|
|
||||||
{
|
|
||||||
"name": "tools",
|
|
||||||
"kind": "bundle",
|
|
||||||
"language": "go",
|
|
||||||
"system": "arch",
|
|
||||||
"from": "cmd/nfs-server",
|
|
||||||
"binary": "nfs-server",
|
|
||||||
"loads": [
|
|
||||||
"nfs-server"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -167,7 +167,6 @@ func (tm *tierMesh) raise(machine, kind, summary string) {
|
|||||||
// The incident's tier, small: a dozen modules whose first machine is the same are sent there in one
|
// The incident's tier, small: a dozen modules whose first machine is the same are sent there in one
|
||||||
// send, judged by one gate, and the rest of their machines are sent once — two sends, not two dozen.
|
// send, judged by one gate, and the rest of their machines are sent once — two sends, not two dozen.
|
||||||
func TestATierOfManyModulesIsSentToEachMachineOnce(t *testing.T) {
|
func TestATierOfManyModulesIsSentToEachMachineOnce(t *testing.T) {
|
||||||
judgeLeavesAsTheCore(t)
|
|
||||||
var modules []string
|
var modules []string
|
||||||
for i := 1; i <= 12; i++ {
|
for i := 1; i <= 12; i++ {
|
||||||
modules = append(modules, fmt.Sprintf("m%02d", i))
|
modules = append(modules, fmt.Sprintf("m%02d", i))
|
||||||
@@ -269,7 +268,6 @@ func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) {
|
|||||||
// A machine-level condition about anything else holds the machine back as a whole: everything the send
|
// A machine-level condition about anything else holds the machine back as a whole: everything the send
|
||||||
// moved there fails together, once, with that reason — not one module for it.
|
// moved there fails together, once, with that reason — not one module for it.
|
||||||
func TestAMachineUnhealthyAsAWholeFailsTheWholeSendTogether(t *testing.T) {
|
func TestAMachineUnhealthyAsAWholeFailsTheWholeSendTogether(t *testing.T) {
|
||||||
judgeLeavesAsTheCore(t)
|
|
||||||
tm := aTierMesh(t, "app1", "app2", "app3")
|
tm := aTierMesh(t, "app1", "app2", "app3")
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
inv := tm.open.inventory
|
inv := tm.open.inventory
|
||||||
@@ -332,7 +330,6 @@ func TestTheConditionsAboutAMachineItself(t *testing.T) {
|
|||||||
// rollback that failed; and `plans retry` asks it again rather than refusing a build that was never
|
// rollback that failed; and `plans retry` asks it again rather than refusing a build that was never
|
||||||
// judged (the incident's "NOT put back: no build kept").
|
// judged (the incident's "NOT put back: no build kept").
|
||||||
func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) {
|
func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) {
|
||||||
judgeLeavesAsTheCore(t)
|
|
||||||
g := aGateMesh(t)
|
g := aGateMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
inv := g.open.inventory
|
inv := g.open.inventory
|
||||||
@@ -387,7 +384,6 @@ func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) {
|
|||||||
// them, they are registered back and marked in the judging that finds them broken, the plan goes on judging
|
// them, they are registered back and marked in the judging that finds them broken, the plan goes on judging
|
||||||
// the other module to its own pass, and only then fails.
|
// the other module to its own pass, and only then fails.
|
||||||
func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) {
|
func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) {
|
||||||
judgeLeavesAsTheCore(t)
|
|
||||||
for _, order := range [][]string{{broker.RuntimeModule, "app1"}, {"app1", broker.RuntimeModule}} {
|
for _, order := range [][]string{{broker.RuntimeModule, "app1"}, {"app1", broker.RuntimeModule}} {
|
||||||
t.Run("gate kept on "+order[0], func(t *testing.T) {
|
t.Run("gate kept on "+order[0], func(t *testing.T) {
|
||||||
tm := aTierMesh(t, order...)
|
tm := aTierMesh(t, order...)
|
||||||
@@ -439,38 +435,3 @@ func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// **A send carrying a core module waits for the core's readings** (issue 524 review): the leaf beside the node
|
|
||||||
// tools has its one reading at once, but the send is judged as the core: not before the settle time, and three
|
|
||||||
// readings.
|
|
||||||
func TestASendCarryingTheCoreIsJudgedAsTheCore(t *testing.T) {
|
|
||||||
tm := aTierMesh(t, broker.RuntimeModule, "app1")
|
|
||||||
ctx := t.Context()
|
|
||||||
// One reading counted per advance, as every test with a gate left open does: a gate judged with no time
|
|
||||||
// between readings would count a pass at every look and never let an advance rest.
|
|
||||||
gateEvery, gateSettle = time.Hour, time.Hour
|
|
||||||
for i := 0; i < 6; i++ {
|
|
||||||
advancePlans(ctx, tm.open)
|
|
||||||
}
|
|
||||||
p := tm.plan(t)
|
|
||||||
lead := p.Modules[broker.RuntimeModule].Gate
|
|
||||||
if lead == nil {
|
|
||||||
lead = p.Modules["app1"].Gate
|
|
||||||
}
|
|
||||||
if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) || !p.Open() || lead == nil || lead.Verdict != "" || lead.Wants != gatePasses {
|
|
||||||
t.Fatalf("passed before the core's settle time: sent %v, plan %s (%s), gate %+v", tm.sent, p.State, p.Note, lead)
|
|
||||||
}
|
|
||||||
gateEvery, gateSettle = 0, 0
|
|
||||||
for i := 0; i < 6; i++ {
|
|
||||||
advancePlans(ctx, tm.open)
|
|
||||||
}
|
|
||||||
p = tm.plan(t)
|
|
||||||
if p.State != inventory.PlanDone || !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"laptop"}}) {
|
|
||||||
t.Fatalf("the plan is %s (%s), sent %v", p.State, p.Note, tm.sent)
|
|
||||||
}
|
|
||||||
for _, m := range []string{broker.RuntimeModule, "app1"} {
|
|
||||||
if g := p.Modules[m].Gate; g != nil && (g.Verdict != inventory.GatePassed || g.Passes < gatePasses) {
|
|
||||||
t.Fatalf("%s's gate: %+v; want passed on %d readings", m, g, gatePasses)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,384 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A provider waiting for the operator holds its consumers, and a wait beside another wait is said (novox/hq
|
|
||||||
// issue 405, found in the review of ADR 0283's controller change).
|
|
||||||
//
|
|
||||||
// The shapes are those of issue 386 (mounts waiting for smb-password-games: waitingResource, passwordWait) and of
|
|
||||||
// the openrazer wait on the workstation of 2026-10-11 (relogin, userUnit): an account in its group whose session
|
|
||||||
// began before it was, and its unit failed in that account's own service manager.
|
|
||||||
|
|
||||||
// waitingDatabase is a provider whose only part not healthy waits for the operator's secret: a database's
|
|
||||||
// process stated waiting, as the node-engine states a tool check answering waits (ADR 0283 decision 2). Its wait
|
|
||||||
// names the part that waits by the provision it gives.
|
|
||||||
func waitingDatabase() inventory.ResourceHealth {
|
|
||||||
return waitingDatabaseFor(inventory.Wait{Part: "postgres-database", Secret: "licence",
|
|
||||||
What: "the licence key of the database"})
|
|
||||||
}
|
|
||||||
|
|
||||||
func waitingDatabaseFor(waits ...inventory.Wait) inventory.ResourceHealth {
|
|
||||||
return inventory.ResourceHealth{Module: "db", Resource: "db.server", Kind: "process", Target: "db.service",
|
|
||||||
State: link.StateWaiting, Check: "tool", Reason: "the database waits for its licence", Waits: waits}
|
|
||||||
}
|
|
||||||
|
|
||||||
// backupsWait is a wait of the same provider for another part than the one its consumers need.
|
|
||||||
var backupsWait = inventory.Wait{Part: "the nightly backups", Secret: "backup-key", What: "the key of the backups"}
|
|
||||||
|
|
||||||
// failingConsumer is a consumer whose check that needs the database fails.
|
|
||||||
func failingConsumer(module string) inventory.ResourceHealth {
|
|
||||||
return inventory.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module,
|
|
||||||
State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"}
|
|
||||||
}
|
|
||||||
|
|
||||||
// dbSecrets is what the database's waits name: own secrets issued outside the mesh, so a wait for one checks out
|
|
||||||
// while nobody gave it (ADR 0283 decision 3, novox/hq issue 450).
|
|
||||||
var dbSecrets = catalogue.OwnSecrets{
|
|
||||||
"licence": {Path: "/s/licence", IssuedBy: catalogue.IssuedOutside},
|
|
||||||
"backup-key": {Path: "/s/backup-key", IssuedBy: catalogue.IssuedOutside},
|
|
||||||
}
|
|
||||||
|
|
||||||
// holdingOf is one reading of the record without a store: the newest statements, the open conditions, the
|
|
||||||
// catalogue, what was given on the provider's machine (nothing), and each consumer's provider already looked up,
|
|
||||||
// as providerFor memoises it.
|
|
||||||
func holdingOf(open []conditions.Condition, healths map[string]inventory.NodeHealth, bound map[[3]string]catalogue.Chosen) *holding {
|
|
||||||
h := &holding{ctx: context.Background(), healths: healths, open: open, providers: map[string]providerLookup{},
|
|
||||||
shelf: map[string]catalogue.Manifest{"db": {Module: "db", Version: "1", OwnSecrets: dbSecrets,
|
|
||||||
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}}},
|
|
||||||
waits: operatorWaitFacts{given: map[string]map[string]time.Time{"db@anchor": {}}}}
|
|
||||||
for k, p := range bound {
|
|
||||||
h.providers[k[0]+"\x00"+k[1]+"\x00"+k[2]] = providerLookup{p, true}
|
|
||||||
}
|
|
||||||
return h
|
|
||||||
}
|
|
||||||
|
|
||||||
var theDatabase = catalogue.Chosen{Node: "anchor", Module: "db"}
|
|
||||||
|
|
||||||
var shopOnTheDatabase = map[[3]string]catalogue.Chosen{{"laptop", "shop", "postgres-database"}: theDatabase}
|
|
||||||
|
|
||||||
func shopFailingBeside(provider ...inventory.ResourceHealth) map[string]inventory.NodeHealth {
|
|
||||||
return map[string]inventory.NodeHealth{
|
|
||||||
"anchor": {Node: "anchor", Resources: provider},
|
|
||||||
"laptop": {Node: "laptop", Resources: []inventory.ResourceHealth{failingConsumer("shop")}},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// (1) A provider whose only part not healthy waits for the operator holds the findings of the consumers of the
|
|
||||||
// waiting part under it (ADR 0240 rule 5); a consumer of another part, or one whose part cannot be matched, is
|
|
||||||
// its own (ADR 0283 decision 1: a module that waits says nothing else of it is wrong).
|
|
||||||
func TestAProviderWaitingForTheOperatorHoldsOnlyTheConsumersOfTheWaitingPart(t *testing.T) {
|
|
||||||
shop := []inventory.ResourceHealth{failingConsumer("shop")}
|
|
||||||
unhealthyDB := waitingDatabase()
|
|
||||||
unhealthyDB.State, unhealthyDB.Waits, unhealthyDB.Reason = link.StateUnhealthy, nil, "its tool check: refused"
|
|
||||||
healthyDB := waitingDatabase()
|
|
||||||
healthyDB.State, healthyDB.Waits = link.StateHealthy, nil
|
|
||||||
byCredential := holdingOf(nil, shopFailingBeside(waitingDatabaseFor(inventory.Wait{Part: "the server",
|
|
||||||
Secret: "licence", What: "the licence key"})), shopOnTheDatabase)
|
|
||||||
byCredential.shelf["db"] = catalogue.Manifest{Module: "db", Version: "1", OwnSecrets: dbSecrets, Provides: []catalogue.Offer{{
|
|
||||||
Name: "postgres-database", Credential: &catalogue.OfferCredential{Own: "licence"}}}}
|
|
||||||
for _, c := range []struct {
|
|
||||||
name string
|
|
||||||
hold *holding
|
|
||||||
held bool
|
|
||||||
onlyWaits bool
|
|
||||||
uncovered bool
|
|
||||||
}{
|
|
||||||
{"the waiting part is the provision", holdingOf(nil, shopFailingBeside(waitingDatabase()), shopOnTheDatabase), true, true, false},
|
|
||||||
{"the wait is for the provision's shared credential", byCredential, true, true, false},
|
|
||||||
{"the wait is for another part", holdingOf(nil, shopFailingBeside(waitingDatabaseFor(backupsWait)), shopOnTheDatabase), false, false, true},
|
|
||||||
{"only the needs-operator condition, its parts not said", holdingOf(
|
|
||||||
[]conditions.Condition{{Key: needsOperatorKey("db", "anchor"), Kind: kindNeedsOperator}},
|
|
||||||
shopFailingBeside(), shopOnTheDatabase), false, false, true},
|
|
||||||
{"an unhealthy provider holds as before", holdingOf(nil, shopFailingBeside(unhealthyDB), shopOnTheDatabase), true, false, false},
|
|
||||||
{"a healthy provider holds nothing", holdingOf(nil, shopFailingBeside(healthyDB), shopOnTheDatabase), false, false, false},
|
|
||||||
} {
|
|
||||||
by, held := c.hold.heldWith("laptop", "shop", shop)
|
|
||||||
if held != c.held || (held && by.provider != theDatabase) || (len(by.waits) > 0) != c.onlyWaits {
|
|
||||||
t.Errorf("%s: held %v under %v with waits %v; want held %v, only waits %v", c.name, held, by.provider,
|
|
||||||
by.waits, c.held, c.onlyWaits)
|
|
||||||
}
|
|
||||||
if _, uncovered := c.hold.waitingUncovered("laptop", "shop", shop); uncovered != c.uncovered {
|
|
||||||
t.Errorf("%s: said as a provider waiting for another part %v; want %v", c.name, uncovered, c.uncovered)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// (1) The consumer's first-node gate under a provider that only waits for the operator passes as a wait for a
|
|
||||||
// person (ADR 0254), carrying the wait (ADR 0283 decision 4); under an unhealthy provider it waits, as before.
|
|
||||||
func TestAConsumersGateUnderAWaitingProviderPassesCarryingTheWait(t *testing.T) {
|
|
||||||
now := time.Now()
|
|
||||||
since := now.Add(-time.Minute)
|
|
||||||
for _, c := range []struct {
|
|
||||||
name string
|
|
||||||
provider inventory.ResourceHealth
|
|
||||||
want health
|
|
||||||
says []string
|
|
||||||
}{
|
|
||||||
{"a provider that only waits", waitingDatabase(), healthPerson,
|
|
||||||
[]string{"waits on db on anchor, which waits for you", "the licence key of the database", "nox secret ask anchor db licence"}},
|
|
||||||
{"an unhealthy provider", func() inventory.ResourceHealth {
|
|
||||||
r := waitingDatabase()
|
|
||||||
r.State, r.Waits, r.Reason = link.StateUnhealthy, nil, "its tool check: refused"
|
|
||||||
return r
|
|
||||||
}(), healthWaiting, []string{"waits on db on anchor, which is unhealthy"}},
|
|
||||||
} {
|
|
||||||
healths := shopFailingBeside(c.provider)
|
|
||||||
for m, h := range healths {
|
|
||||||
h.HeardAt = now
|
|
||||||
healths[m] = h
|
|
||||||
}
|
|
||||||
f := gateFacts{now: now, health: healths, heldOn: heldReadings(holdingOf(nil, healths, shopOnTheDatabase))}
|
|
||||||
h, why := moduleHealthWord("shop", "laptop", since, f)
|
|
||||||
if h != c.want {
|
|
||||||
t.Errorf("%s: the consumer's gate reads %v %q; want %v", c.name, h, why, c.want)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
for _, s := range c.says {
|
|
||||||
if !strings.Contains(why, s) {
|
|
||||||
t.Errorf("%s: the gate's reading %q does not say %q", c.name, why, s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// judgeBoth judges the provider's statement and then the consumer's, two looks each, over a record that changes
|
|
||||||
// as the statements do.
|
|
||||||
func judgeBoth(t *testing.T, k *conditions.Keeper, provider []inventory.ResourceHealth,
|
|
||||||
byProvider, byConsumer map[string]inventory.NodeHealth) []conditions.Condition {
|
|
||||||
t.Helper()
|
|
||||||
ctx := t.Context()
|
|
||||||
was := readHoldingFor
|
|
||||||
t.Cleanup(func() { readHoldingFor = was })
|
|
||||||
healths := byProvider
|
|
||||||
readHoldingFor = func(_ context.Context, _ *inventory.Inventory, open []conditions.Condition) (*holding, error) {
|
|
||||||
return holdingOf(open, healths, shopOnTheDatabase), nil
|
|
||||||
}
|
|
||||||
for look := 1; look <= 2; look++ {
|
|
||||||
healths = byProvider
|
|
||||||
if err := judgeModuleHealth(ctx, nil, k, "anchor", map[string][]inventory.ResourceHealth{"db": provider},
|
|
||||||
map[string]int{"db": look}, time.Now()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for look := 1; look <= 2; look++ {
|
|
||||||
healths = byConsumer
|
|
||||||
if err := judgeModuleHealth(ctx, nil, k, "laptop", map[string][]inventory.ResourceHealth{"shop": {failingConsumer("shop")}},
|
|
||||||
map[string]int{"shop": look}, time.Now()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
open, err := k.Open(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return open
|
|
||||||
}
|
|
||||||
|
|
||||||
func conditionOf(open []conditions.Condition, key string) *conditions.Condition {
|
|
||||||
for i, c := range open {
|
|
||||||
if c.Key == key {
|
|
||||||
return &open[i]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// (1) The consumer raises nothing of its own; the provider's needs-operator condition lists who waits on it and
|
|
||||||
// stays a warning (ADR 0283 decision 5: never escalated). The consumer's statement arrives after the provider's,
|
|
||||||
// so it is the consumer's judging (sayWaiters) that lists it at the provider — no store involved.
|
|
||||||
func TestAWaitingProvidersConditionListsWhoWaitsOnItAndStaysAWarning(t *testing.T) {
|
|
||||||
k, _ := withConditionsInMemory(t)
|
|
||||||
open := judgeBoth(t, k, []inventory.ResourceHealth{waitingDatabase()},
|
|
||||||
map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Resources: []inventory.ResourceHealth{waitingDatabase()}}}, shopFailingBeside(waitingDatabase()))
|
|
||||||
provider := conditionOf(open, needsOperatorKey("db", "anchor"))
|
|
||||||
if len(open) != 1 || provider == nil {
|
|
||||||
t.Fatalf("a provider waiting for the operator and a consumer of its waiting part raised %v; want the "+
|
|
||||||
"provider's needs-operator alone", openKeysOf(open))
|
|
||||||
}
|
|
||||||
if said := provider.Evidence[0].Said; !strings.Contains(said, "shop on laptop") {
|
|
||||||
t.Fatalf("the provider's needs-operator does not list shop on laptop as waiting on it: %s", said)
|
|
||||||
}
|
|
||||||
if provider.Severity != conditions.Warning {
|
|
||||||
t.Fatalf("the provider's needs-operator became %s with a consumer waiting; it stays a warning", provider.Severity)
|
|
||||||
}
|
|
||||||
if provider.Resolver != conditions.ResolverOperator || !strings.Contains(provider.Needs, "desk prompt") {
|
|
||||||
t.Fatalf("the provider's condition: %+v", provider)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// (1) A consumer of another part than the one waiting is raised on its own, and says its provider waits.
|
|
||||||
func TestAConsumerOfAnotherPartIsRaisedOnItsOwn(t *testing.T) {
|
|
||||||
k, _ := withConditionsInMemory(t)
|
|
||||||
backups := waitingDatabaseFor(backupsWait)
|
|
||||||
open := judgeBoth(t, k, []inventory.ResourceHealth{backups}, shopFailingBeside(backups), shopFailingBeside(backups))
|
|
||||||
consumer := conditionOf(open, moduleUnhealthyKey("shop", "laptop"))
|
|
||||||
if consumer == nil || conditionOf(open, needsOperatorKey("db", "anchor")) == nil {
|
|
||||||
t.Fatalf("raised %v; want shop's own unhealthy beside db's needs-operator", openKeysOf(open))
|
|
||||||
}
|
|
||||||
if said := consumer.Evidence[0].Said; !strings.Contains(said, "db on anchor waits for you, but not for anything shop is known to need, so shop's fault is said on its own") {
|
|
||||||
t.Fatalf("the consumer's condition does not say its provider waits: %s", said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// relogin and userUnit are person_wait_test.go's; mounts is said here with the same account and unit beside its
|
|
||||||
// wait for the password.
|
|
||||||
func reloginBesideAWait() []inventory.ResourceHealth {
|
|
||||||
return []inventory.ResourceHealth{relogin("mounts", "operator"), userUnit("mounts", "operator"),
|
|
||||||
waitingResource(passwordWait)}
|
|
||||||
}
|
|
||||||
|
|
||||||
// (2) A module with a checked wait beside a relogin-needed account says both: the new login, and the act the
|
|
||||||
// operator owes it.
|
|
||||||
func TestAWaitBesideAReloginIsSaid(t *testing.T) {
|
|
||||||
k, _ := withConditionsInMemory(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
for look := 1; look <= 2; look++ {
|
|
||||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": reloginBesideAWait()},
|
|
||||||
map[string]int{"mounts": look}, time.Now()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
got, open := needsOperatorOpen(t, k)
|
|
||||||
if got == nil {
|
|
||||||
t.Fatalf("a wait for the operator beside a relogin is not said: %v", openKeysOf(open))
|
|
||||||
}
|
|
||||||
if !strings.Contains(got.Summary, "smb-password-games") || got.Severity != conditions.Warning {
|
|
||||||
t.Fatalf("the needs-operator beside the relogin: %+v", got)
|
|
||||||
}
|
|
||||||
relogged := false
|
|
||||||
for _, c := range open {
|
|
||||||
relogged = relogged || c.Key == reloginKey("mounts", "workstation")
|
|
||||||
}
|
|
||||||
if !relogged {
|
|
||||||
t.Fatalf("the relogin is no longer said beside the wait: %v", openKeysOf(open))
|
|
||||||
}
|
|
||||||
// The login done, the wait stays said and the relogin clears.
|
|
||||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}},
|
|
||||||
map[string]int{"mounts": 3}, time.Now()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
got, open = needsOperatorOpen(t, k)
|
|
||||||
if got == nil || len(open) != 1 {
|
|
||||||
t.Fatalf("after the new login: %v", openKeysOf(open))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// (2) The same beside a directory used as found.
|
|
||||||
func TestAWaitBesideADirectoryUsedAsFoundIsSaid(t *testing.T) {
|
|
||||||
k, _ := withConditionsInMemory(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
found := foundDirectory("mounts", time.Now().Add(-time.Hour))
|
|
||||||
for look := 1; look <= 2; look++ {
|
|
||||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{
|
|
||||||
"mounts": {found, waitingResource(passwordWait)}}, map[string]int{"mounts": look}, time.Now()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
got, open := needsOperatorOpen(t, k)
|
|
||||||
if got == nil {
|
|
||||||
t.Fatalf("a wait for the operator beside a directory used as found is not said: %v", openKeysOf(open))
|
|
||||||
}
|
|
||||||
asFound := false
|
|
||||||
for _, c := range open {
|
|
||||||
asFound = asFound || c.Key == usedAsFoundKey("mounts", "workstation")
|
|
||||||
}
|
|
||||||
if !asFound {
|
|
||||||
t.Fatalf("the directory used as found is no longer said beside the wait: %v", openKeysOf(open))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// (2) Beside a fault of its own, the wait is said too, and the fault's words do not count the waiting part among
|
|
||||||
// what fails.
|
|
||||||
func TestAWaitBesideAFaultIsSaidAndTheFaultIsTheFaultAlone(t *testing.T) {
|
|
||||||
k, _ := withConditionsInMemory(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
down := inventory.ResourceHealth{Module: "mounts", Resource: "mounts.apply", Kind: "process",
|
|
||||||
Target: "mesh-mounts-apply.service", State: link.StateUnhealthy, Reason: "down"}
|
|
||||||
for look := 1; look <= 2; look++ {
|
|
||||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{
|
|
||||||
"mounts": {down, waitingResource(passwordWait)}}, map[string]int{"mounts": look}, time.Now()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
got, open := needsOperatorOpen(t, k)
|
|
||||||
if got == nil {
|
|
||||||
t.Fatalf("a wait for the operator beside a fault is not said: %v", openKeysOf(open))
|
|
||||||
}
|
|
||||||
var fault *conditions.Condition
|
|
||||||
for i, c := range open {
|
|
||||||
if c.Key == moduleUnhealthyKey("mounts", "workstation") {
|
|
||||||
fault = &open[i]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if fault == nil {
|
|
||||||
t.Fatalf("the fault is not said: %v", openKeysOf(open))
|
|
||||||
}
|
|
||||||
if strings.Contains(fault.Summary, "mounts.watch") {
|
|
||||||
t.Fatalf("the fault's summary counts the waiting part as failing: %q", fault.Summary)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func openKeysOf(cs []conditions.Condition) []string {
|
|
||||||
var out []string
|
|
||||||
for _, c := range cs {
|
|
||||||
out = append(out, c.Key)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// A consumer raised on its own, whose provider then waits for the operator for the part it needs, is cleared saying
|
|
||||||
// which the provider is: it waits for you, not that it is unhealthy (novox/hq issue 405 review).
|
|
||||||
func TestAConsumerHeldOnceItsProviderWaitsSaysWhichItIs(t *testing.T) {
|
|
||||||
k, _ := withConditionsInMemory(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
start := time.Now().Add(-time.Second)
|
|
||||||
healthy := waitingDatabase()
|
|
||||||
healthy.State, healthy.Waits = link.StateHealthy, nil
|
|
||||||
healths := shopFailingBeside(healthy)
|
|
||||||
was := readHoldingFor
|
|
||||||
t.Cleanup(func() { readHoldingFor = was })
|
|
||||||
readHoldingFor = func(_ context.Context, _ *inventory.Inventory, open []conditions.Condition) (*holding, error) {
|
|
||||||
return holdingOf(open, healths, shopOnTheDatabase), nil
|
|
||||||
}
|
|
||||||
shop := map[string][]inventory.ResourceHealth{"shop": {failingConsumer("shop")}}
|
|
||||||
for look := 1; look <= 2; look++ {
|
|
||||||
if err := judgeModuleHealth(ctx, nil, k, "laptop", shop, map[string]int{"shop": look}, time.Now()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if open, _ := k.Open(ctx); conditionOf(open, moduleUnhealthyKey("shop", "laptop")) == nil {
|
|
||||||
t.Fatalf("shop beside a healthy provider is not raised: %v", openKeysOf(open))
|
|
||||||
}
|
|
||||||
healths = shopFailingBeside(waitingDatabase())
|
|
||||||
if err := judgeModuleHealth(ctx, nil, k, "laptop", shop, map[string]int{"shop": 3}, time.Now()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var why string
|
|
||||||
for deadline := time.Now().Add(2 * time.Second); why == "" && time.Now().Before(deadline); {
|
|
||||||
events, err := k.HistorySince(ctx, start)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, e := range events {
|
|
||||||
if e.Key == moduleUnhealthyKey("shop", "laptop") && e.Change == conditions.ChangeCleared {
|
|
||||||
why = e.Why
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if why == "" {
|
|
||||||
time.Sleep(10 * time.Millisecond)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !strings.Contains(why, "waits on db on anchor, which waits for you") {
|
|
||||||
t.Fatalf("shop's clearing says %q; want it to say db on anchor waits for you", why)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -55,10 +55,6 @@ type signalFacts struct {
|
|||||||
waits []waitFacts
|
waits []waitFacts
|
||||||
// batches are the batches not yet cut (S18, S19, novox/hq ADR 0276).
|
// batches are the batches not yet cut (S18, S19, novox/hq ADR 0276).
|
||||||
batches []batchFacts
|
batches []batchFacts
|
||||||
// refusedSends are the sends a machine refused within the hour for the generation they were composed
|
|
||||||
// from, each naming its sender (S20, novox/hq issue 234).
|
|
||||||
refusedSends []inventory.Send
|
|
||||||
refusedSendsErr error
|
|
||||||
|
|
||||||
loop loopFacts
|
loop loopFacts
|
||||||
loopErr error
|
loopErr error
|
||||||
@@ -84,9 +80,6 @@ type signalFacts struct {
|
|||||||
// deadLetters are how many messages DEAD_LETTERS holds per consumer, by `<stream>.<consumer>`
|
// deadLetters are how many messages DEAD_LETTERS holds per consumer, by `<stream>.<consumer>`
|
||||||
// (novox/hq issue 330): each consumer's max-deliveries condition is open while it holds any.
|
// (novox/hq issue 330): each consumer's max-deliveries condition is open while it holds any.
|
||||||
deadLetters map[string]int
|
deadLetters map[string]int
|
||||||
// deadLettersNewest is when DEAD_LETTERS kept the newest message it holds for each of those
|
|
||||||
// consumers: the condition counts the messages given up on, not the looks (novox/hq issue 440).
|
|
||||||
deadLettersNewest map[string]time.Time
|
|
||||||
advisoriesErr error
|
advisoriesErr error
|
||||||
|
|
||||||
selfCheck selfCheckFacts
|
selfCheck selfCheckFacts
|
||||||
@@ -357,18 +350,7 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
|
|||||||
if f.advisoriesErr == nil && w.js != nil {
|
if f.advisoriesErr == nil && w.js != nil {
|
||||||
f.deadLetters, f.advisoriesErr = link.HeldDeadLetters(w.js.Context())
|
f.deadLetters, f.advisoriesErr = link.HeldDeadLetters(w.js.Context())
|
||||||
}
|
}
|
||||||
if f.advisoriesErr == nil && len(f.deadLetters) > 0 {
|
|
||||||
f.deadLettersNewest, f.advisoriesErr = link.NewestDeadLetters(w.js.Context(), f.deadLetters)
|
|
||||||
for key := range f.deadLetters {
|
|
||||||
if _, ok := f.deadLettersNewest[key]; !ok && f.advisoriesErr == nil {
|
|
||||||
// Delivered again or dropped between the two reads: left out of this look, rather than
|
|
||||||
// observed without a time and counted as a look (novox/hq issue 440).
|
|
||||||
delete(f.deadLetters, key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
f.handActs, f.handActsErr = w.gatherHandActs(ctx, now)
|
f.handActs, f.handActsErr = w.gatherHandActs(ctx, now)
|
||||||
f.refusedSends, f.refusedSendsErr = inv.RefusedSendsSince(ctx, now.Add(-generationRefusalsSaid))
|
|
||||||
f.facts.taken, _, f.facts.began, f.facts.err = exportedFacts.last()
|
f.facts.taken, _, f.facts.began, f.facts.err = exportedFacts.last()
|
||||||
return f
|
return f
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ module github.com/novox/mesh-controller
|
|||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c
|
git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970
|
||||||
github.com/jackc/pgx/v5 v5.10.0
|
github.com/jackc/pgx/v5 v5.10.0
|
||||||
github.com/nats-io/nats-server/v2 v2.11.17
|
github.com/nats-io/nats-server/v2 v2.11.17
|
||||||
github.com/nats-io/nats.go v1.54.0
|
github.com/nats-io/nats.go v1.54.0
|
||||||
@@ -35,4 +35,4 @@ require (
|
|||||||
// committed. Every build (the build agent's `go build`, the Dockerfile) compiles from vendor/ and
|
// committed. Every build (the build agent's `go build`, the Dockerfile) compiles from vendor/ and
|
||||||
// fetches nothing; go refuses to build when vendor/ and this file disagree, so a pin moved without
|
// fetches nothing; go refuses to build when vendor/ and this file disagree, so a pin moved without
|
||||||
// `go mod vendor` fails loudly, at once, everywhere.
|
// `go mod vendor` fails loudly, at once, everywhere.
|
||||||
replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f
|
replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261009231844-b8c854611812
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f h1:8N5OW2mdTNIck2pe4EciTYX5NsrPsHrTLENGNIWYNTU=
|
git.novox.be/novox/mesh-host v0.0.0-20261009231844-b8c854611812 h1:pzVzwF5VMWaTECxu8+Pd1dNoOHNEm7upC5wPadQTkBw=
|
||||||
git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f/go.mod h1:tcTK4LMs1d6JpZUwy3hSHMFG/MIuDr/XFs7/nbeaW/c=
|
git.novox.be/novox/mesh-host v0.0.0-20261009231844-b8c854611812/go.mod h1:K3/xEzVgmrNKLMV2vv4M80MwmPnQNXqvQ4C5Jj0fJT4=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s=
|
git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970 h1:9tFDQsgmI+4X7/BpZGXIr+HemPKE7YddYGqWV0lINAI=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
|||||||
@@ -1,95 +0,0 @@
|
|||||||
// Package beside is where a test finds another repository of the mesh it reads: the catalogue's manifests,
|
|
||||||
// the node-engine's genesis template (novox/hq issue 432), the SDK's conformance fixtures (issue 449).
|
|
||||||
//
|
|
||||||
// A test used to read the checkout beside this one, `../../../mesh-catalog`, so its verdict depended on
|
|
||||||
// whatever sat on the machine running it: a stale or dirty checkout failed it on a desktop, and where none
|
|
||||||
// was beside it skipped and said nothing. Now there are two inputs, both named, and no third:
|
|
||||||
//
|
|
||||||
// - In a merge check, the build seat clones each core repository beside the one checked (the catalogue
|
|
||||||
// at its main, the node-engine at the commit the mesh runs) and says where in MESH_CHECK_BESIDE. A
|
|
||||||
// test judges against those clones, so agreement with the other repository is checked where
|
|
||||||
// `mesh/repo-check` runs; a repository missing there fails the test, never skips it. Which clones a
|
|
||||||
// check gets is chosen from the inventory: mesh-catalog by the source of the `nats` module, mesh-host
|
|
||||||
// by the source of `mesh-host`, and mesh-sdk as the controller's sibling at the commit the controller's
|
|
||||||
// go.mod pins. In a mesh where either module has no source repository nothing is
|
|
||||||
// cloned, and these tests fail loudly with "not beside this check": a cause in the setup, not in the
|
|
||||||
// change. And in a delivery group that holds a mesh-catalog pull request, these tests read the
|
|
||||||
// catalogue's main, not the group's head.
|
|
||||||
// - Anywhere else, the test judges against the copy captured in this repository's testdata/beside, at the
|
|
||||||
// commit testdata/beside/CAPTURED names. Never against a developer's own checkout: to judge one, set
|
|
||||||
// MESH_CHECK_BESIDE to the directory holding it, as the check does.
|
|
||||||
//
|
|
||||||
// The captured manifests are named module.json.captured, and put back as module.json in a directory of
|
|
||||||
// the test's own: a module.json anywhere in this repository is a module of it to the forge's announcer and
|
|
||||||
// the planner, and a merge would build and register a hundred copies of the catalogue's.
|
|
||||||
package beside
|
|
||||||
|
|
||||||
import (
|
|
||||||
"io/fs"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"runtime"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Env is where a merge check says the repositories cloned beside the one checked are (internal/builder's
|
|
||||||
// EnvBeside, repeated here so a test does not import the builder).
|
|
||||||
const Env = "MESH_CHECK_BESIDE"
|
|
||||||
|
|
||||||
// CapturedSuffix is what a captured file's name carries that the repository's own does not.
|
|
||||||
const CapturedSuffix = ".captured"
|
|
||||||
|
|
||||||
// Dir is the named repository a test reads — the clone beside a merge check, or the captured copy — and
|
|
||||||
// says which in the test's log.
|
|
||||||
func Dir(t testing.TB, repository string) string {
|
|
||||||
t.Helper()
|
|
||||||
if root := os.Getenv(Env); root != "" {
|
|
||||||
dir := filepath.Join(root, repository)
|
|
||||||
if _, err := os.Stat(dir); err != nil {
|
|
||||||
t.Fatalf("%s is not beside this check in %s=%s, so its agreement with this repository cannot be "+
|
|
||||||
"judged here: %v", repository, Env, root, err)
|
|
||||||
}
|
|
||||||
t.Logf("judged against %s as cloned beside this check", dir)
|
|
||||||
return dir
|
|
||||||
}
|
|
||||||
from := filepath.Join(Captured(), repository)
|
|
||||||
if _, err := os.Stat(from); err != nil {
|
|
||||||
t.Fatalf("no captured copy of %s at %s: %v", repository, from, err)
|
|
||||||
}
|
|
||||||
dir := filepath.Join(t.TempDir(), repository)
|
|
||||||
err := filepath.WalkDir(from, func(path string, d fs.DirEntry, err error) error {
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
rel, _ := filepath.Rel(from, path)
|
|
||||||
into := filepath.Join(dir, strings.TrimSuffix(rel, CapturedSuffix))
|
|
||||||
if d.IsDir() {
|
|
||||||
return os.MkdirAll(into, 0o755)
|
|
||||||
}
|
|
||||||
raw, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return os.WriteFile(into, raw, 0o644)
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("the captured copy of %s could not be laid out: %v", repository, err)
|
|
||||||
}
|
|
||||||
t.Logf("judged against the copy of %s captured in testdata/beside (see CAPTURED); a merge check "+
|
|
||||||
"judges it against the repository's own clone", repository)
|
|
||||||
return dir
|
|
||||||
}
|
|
||||||
|
|
||||||
// Catalogue is the catalogue's modules directory, as Dir finds the catalogue.
|
|
||||||
func Catalogue(t testing.TB) string {
|
|
||||||
t.Helper()
|
|
||||||
return filepath.Join(Dir(t, "mesh-catalog"), "modules")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Captured is this repository's testdata/beside, found from this file rather than from the working
|
|
||||||
// directory, so a test in any package reaches it.
|
|
||||||
func Captured() string {
|
|
||||||
_, file, _, _ := runtime.Caller(0)
|
|
||||||
return filepath.Join(filepath.Dir(file), "..", "..", "testdata", "beside")
|
|
||||||
}
|
|
||||||
@@ -8,7 +8,6 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/beside"
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -75,10 +74,10 @@ func TestTheAgreementCheckCatchesASubscriptionThatMatchesNothing(t *testing.T) {
|
|||||||
|
|
||||||
func theCataloguesEvents(t *testing.T) ([]AnEmitter, []AConsumer, []DeclaredSeat) {
|
func theCataloguesEvents(t *testing.T) ([]AnEmitter, []AConsumer, []DeclaredSeat) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
root := beside.Catalogue(t)
|
root := filepath.Join("..", "..", "..", "mesh-catalog", "modules")
|
||||||
entries, err := os.ReadDir(root)
|
entries, err := os.ReadDir(root)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Skipf("catalogue sibling not present: %v", err)
|
||||||
}
|
}
|
||||||
var emitters []AnEmitter
|
var emitters []AnEmitter
|
||||||
var consumers []AConsumer
|
var consumers []AConsumer
|
||||||
@@ -120,7 +119,7 @@ func theCataloguesEvents(t *testing.T) ([]AnEmitter, []AConsumer, []DeclaredSeat
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(emitters) == 0 {
|
if len(emitters) == 0 {
|
||||||
t.Fatalf("no module under %s emits anything, so this proved nothing", root)
|
t.Skip("no manifests found beside this checkout")
|
||||||
}
|
}
|
||||||
return emitters, consumers, seats
|
return emitters, consumers, seats
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,117 +0,0 @@
|
|||||||
package broker
|
|
||||||
|
|
||||||
import (
|
|
||||||
"regexp"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A tool call names its caller (novox/hq issue 365, by ADR 0259 §3's precedent for asks).
|
|
||||||
//
|
|
||||||
// mesh.mod.<module>.call.<tool>[.<node>].<caller>
|
|
||||||
// mesh.seat.<seat>.call.<verb>[.<node>].<caller>
|
|
||||||
//
|
|
||||||
// The last token is the bus user that published the call, and each user is granted these subjects with its own
|
|
||||||
// name there and no other — the caller is a fact the server enforces, and the tool runtime hands it to the
|
|
||||||
// module from the subject a call arrived on (mesh-tools node-tools internal/bus caller.go holds the same shape).
|
|
||||||
//
|
|
||||||
// **A kind of its own, `call`, not the `tool` subject with a token appended.** Every grant to call a tool is a
|
|
||||||
// wildcard over the `tool` kind — `.tool.<t>.*` for the instance on any machine, `mesh.mod.*.tool.>` for every
|
|
||||||
// tool — and either would match a `tool` subject with any caller appended, so a caller could name another.
|
|
||||||
// Under `call` nothing is granted but the caller-named subjects. No stream's filter covers it: a tool call is
|
|
||||||
// never persisted (design 25 §3).
|
|
||||||
//
|
|
||||||
// **Derived from the `tool` grants, in one place** (callerNamed): wherever a principal may call or answer a
|
|
||||||
// tool, it may call it in its own name, or answer it naming any caller — so no kind of principal is left
|
|
||||||
// unable to call in its own name, and a new grant to call is one in both shapes by construction.
|
|
||||||
//
|
|
||||||
// The `tool` grants stay beside these for one release, so every caller and every runtime moves without a gap:
|
|
||||||
// their retirement is hq issue 464.
|
|
||||||
const CallKind = "call"
|
|
||||||
|
|
||||||
var userName = regexp.MustCompile(`^[A-Za-z0-9_-]+(\.[A-Za-z0-9_-]+)*$`)
|
|
||||||
|
|
||||||
// CallerToken is a bus user's name as the last token of a call it publishes: each dot written `~`, which no part
|
|
||||||
// of a user's name may hold (safeSubject), so the token names that user and no other. "" for a name that is not
|
|
||||||
// a user's.
|
|
||||||
func CallerToken(user string) string {
|
|
||||||
if !userName.MatchString(user) {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return strings.ReplaceAll(user, ".", "~")
|
|
||||||
}
|
|
||||||
|
|
||||||
// toolGrant splits a grant on the `tool` kind — `mesh.mod.<m>.tool.<rest>` or `mesh.seat.<s>.tool.<rest>`, any
|
|
||||||
// part possibly a wildcard — at the kind; ok is false for any other subject.
|
|
||||||
func toolGrant(subject string) (head, rest string, ok bool) {
|
|
||||||
parts := strings.SplitN(subject, ".", 5)
|
|
||||||
if len(parts) != 5 || parts[0] != "mesh" || (parts[1] != "mod" && parts[1] != "seat") || parts[3] != "tool" ||
|
|
||||||
parts[2] == "" || parts[4] == "" {
|
|
||||||
return "", "", false
|
|
||||||
}
|
|
||||||
return parts[0] + "." + parts[1] + "." + parts[2], parts[4], true
|
|
||||||
}
|
|
||||||
|
|
||||||
// CalledSubject is where a call to a tool subject goes naming its caller; "" when the subject is not a tool's
|
|
||||||
// or the user is not a bus user.
|
|
||||||
func CalledSubject(subject, user string) string {
|
|
||||||
head, rest, ok := toolGrant(subject)
|
|
||||||
token := CallerToken(user)
|
|
||||||
if !ok || token == "" || strings.ContainsAny(rest, "*>") {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return head + "." + CallKind + "." + rest + "." + token
|
|
||||||
}
|
|
||||||
|
|
||||||
// CalledPattern is what a holder answering a tool subject also subscribes, to hear the calls that name their
|
|
||||||
// caller: the same address under the `call` kind, any caller last. "" for a subject that is not a tool's.
|
|
||||||
func CalledPattern(subject string) string {
|
|
||||||
head, rest, ok := toolGrant(subject)
|
|
||||||
if !ok || strings.ContainsAny(rest, "*>") {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return head + "." + CallKind + "." + rest + ".*"
|
|
||||||
}
|
|
||||||
|
|
||||||
// calledPublish is a grant to call on the `tool` kind, as the same grant in the caller's own name: its last
|
|
||||||
// token the caller's, and nothing that reaches past it. A `>` is every tail a call carries — the tool alone or
|
|
||||||
// the tool and the machine — so it becomes both, each ending in the caller.
|
|
||||||
func calledPublish(grant, token string) []string {
|
|
||||||
head, rest, ok := toolGrant(grant)
|
|
||||||
if !ok || token == "" {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
base := head + "." + CallKind + "."
|
|
||||||
switch {
|
|
||||||
case rest == ">":
|
|
||||||
return []string{base + "*." + token, base + "*.*." + token}
|
|
||||||
case strings.HasSuffix(rest, ".>"):
|
|
||||||
return []string{base + strings.TrimSuffix(rest, ">") + "*." + token}
|
|
||||||
}
|
|
||||||
return []string{base + rest + "." + token}
|
|
||||||
}
|
|
||||||
|
|
||||||
// calledSubscribe is a grant to answer on the `tool` kind, as the same grant for the calls naming any caller.
|
|
||||||
func calledSubscribe(grant string) []string {
|
|
||||||
head, rest, ok := toolGrant(grant)
|
|
||||||
if !ok {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
base := head + "." + CallKind + "."
|
|
||||||
if strings.HasSuffix(rest, ">") {
|
|
||||||
return []string{base + rest}
|
|
||||||
}
|
|
||||||
return []string{base + rest + ".*"}
|
|
||||||
}
|
|
||||||
|
|
||||||
// callerNamed adds, beside a principal's grants on the `tool` kind, the same grants under `call`: to publish in
|
|
||||||
// its own name, to subscribe naming anybody.
|
|
||||||
func callerNamed(user string, pub, sub []string) ([]string, []string) {
|
|
||||||
token := CallerToken(user)
|
|
||||||
for _, g := range pub {
|
|
||||||
pub = append(pub, calledPublish(g, token)...)
|
|
||||||
}
|
|
||||||
for _, g := range sub {
|
|
||||||
sub = append(sub, calledSubscribe(g)...)
|
|
||||||
}
|
|
||||||
return unique(pub), unique(sub)
|
|
||||||
}
|
|
||||||
@@ -1,100 +0,0 @@
|
|||||||
package broker
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats-server/v2/server"
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
"golang.org/x/crypto/bcrypt"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **On a real server, as composed** (novox/hq issue 365): a machine's runtime calls in its own name, and the
|
|
||||||
// server refuses it a call naming another — the grant, not the runtime, is what makes the caller a fact.
|
|
||||||
func TestAServerComposedFromTheGrantsRefusesACallNamingAnother(t *testing.T) {
|
|
||||||
hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
ledger := Seat{Name: "issue-tracker", Scope: "mesh", Serves: []string{"open"}}
|
|
||||||
accounts, err := ComposeAccounts([]Principal{
|
|
||||||
{Kind: KindNodeTools, Node: "novox", Module: RuntimeModule, PasswordHash: string(hash),
|
|
||||||
Carries: []Declared{{Module: "mesh-issues", Holds: []Seat{ledger}}}},
|
|
||||||
{Kind: KindNodeTools, Node: "shanks", Module: RuntimeModule, PasswordHash: string(hash)},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
conf := filepath.Join(t.TempDir(), "bus.conf")
|
|
||||||
if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+
|
|
||||||
`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
opts, err := server.ProcessConfigFile(conf)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("the composed accounts do not parse: %v", err)
|
|
||||||
}
|
|
||||||
opts.NoLog, opts.NoSigs = true, true
|
|
||||||
s, err := server.NewServer(opts)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
go s.Start()
|
|
||||||
if !s.ReadyForConnections(10 * time.Second) {
|
|
||||||
t.Fatal("the bus did not come up")
|
|
||||||
}
|
|
||||||
defer s.Shutdown()
|
|
||||||
|
|
||||||
holder, err := nats.Connect(s.ClientURL(), nats.UserInfo("novox.node-tools", "pw"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer holder.Close()
|
|
||||||
heard := make(chan string, 4)
|
|
||||||
sub, err := holder.Subscribe("mesh.seat.issue-tracker.call.open.*", func(m *nats.Msg) {
|
|
||||||
heard <- m.Subject
|
|
||||||
_ = m.Respond([]byte(`{"result":{}}`))
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
_ = holder.Flush()
|
|
||||||
if !sub.IsValid() {
|
|
||||||
t.Fatal("the holder may not hear the caller-named calls to its seat")
|
|
||||||
}
|
|
||||||
|
|
||||||
refusals := make(chan error, 4)
|
|
||||||
caller, err := nats.Connect(s.ClientURL(), nats.UserInfo("shanks.node-tools", "pw"),
|
|
||||||
nats.CustomInboxPrefix("_INBOX.shanks.node-tools"),
|
|
||||||
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refusals <- err }))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer caller.Close()
|
|
||||||
if _, err := caller.Request("mesh.seat.issue-tracker.call.open.shanks~node-tools", []byte(`{}`), 3*time.Second); err != nil {
|
|
||||||
t.Fatalf("a call in the caller's own name was not answered: %v", err)
|
|
||||||
}
|
|
||||||
if got := <-heard; got != "mesh.seat.issue-tracker.call.open.shanks~node-tools" {
|
|
||||||
t.Fatalf("heard %s", got)
|
|
||||||
}
|
|
||||||
if err := caller.Publish("mesh.seat.issue-tracker.call.open.novox~node-tools", []byte(`{}`)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
_ = caller.Flush()
|
|
||||||
select {
|
|
||||||
case err := <-refusals:
|
|
||||||
if !errors.Is(err, nats.ErrPermissionViolation) {
|
|
||||||
t.Errorf("the server said %v, want a permissions violation", err)
|
|
||||||
}
|
|
||||||
case <-time.After(3 * time.Second):
|
|
||||||
t.Error("the server did not refuse a call naming another caller")
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case got := <-heard:
|
|
||||||
t.Errorf("a call naming another reached the holder: %s", got)
|
|
||||||
case <-time.After(200 * time.Millisecond):
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,119 +0,0 @@
|
|||||||
package broker
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **A tool call names its caller, and the bus lets each user name itself alone** (novox/hq issue 365, by ADR
|
|
||||||
// 0259 §3's precedent for asks): wherever a principal may call a tool, it may call it on the caller-named
|
|
||||||
// subject — kind `call`, its own bus user last, dots written `~` — and on no subject naming anybody else.
|
|
||||||
func TestEachCredentialMayCallOnlyInItsOwnName(t *testing.T) {
|
|
||||||
ledger := Seat{Name: "node-desk", Scope: "node", Serves: []string{"who"}}
|
|
||||||
tracker := Seat{Name: "issue-tracker", Scope: "mesh", Serves: []string{"open"}}
|
|
||||||
for _, c := range []struct {
|
|
||||||
p Principal
|
|
||||||
may []string
|
|
||||||
mayNot []string
|
|
||||||
subject []string // what it subscribes, to answer calls naming any caller
|
|
||||||
}{
|
|
||||||
{p: Principal{Kind: KindPerson, Module: "jochen", Invokes: []string{"*"}},
|
|
||||||
may: []string{"mesh.mod.ledger.call.who.person~jochen", "mesh.mod.ledger.call.who.anchor.person~jochen",
|
|
||||||
"mesh.seat.issue-tracker.call.open.person~jochen", "mesh.seat.node-desk.call.who.anchor.person~jochen"},
|
|
||||||
mayNot: []string{"mesh.mod.ledger.call.who.shanks~node-tools", "mesh.mod.ledger.call.who.anchor.controller",
|
|
||||||
"mesh.seat.issue-tracker.call.open.person~somebody", "mesh.mod.ledger.call.who.person"}},
|
|
||||||
{p: Principal{Kind: KindNodeTools, Node: "shanks", Module: RuntimeModule,
|
|
||||||
Carries: []Declared{{Module: "ledger", Holds: []Seat{ledger, tracker}}}},
|
|
||||||
may: []string{"mesh.mod.ledger.call.who.shanks~node-tools", "mesh.seat.issue-tracker.call.open.shanks~node-tools"},
|
|
||||||
mayNot: []string{"mesh.mod.ledger.call.who.novox~node-tools", "mesh.seat.issue-tracker.call.open.controller"},
|
|
||||||
subject: []string{"mesh.mod.ledger.call.who.novox~node-tools", "mesh.seat.node-desk.call.who.shanks.person~jochen", "mesh.seat.issue-tracker.call.open.controller"}},
|
|
||||||
{p: Principal{Kind: KindModule, Node: "two", Module: "shop", Invokes: []string{"ledger.who", "seat:issue-tracker.open"}},
|
|
||||||
may: []string{"mesh.mod.ledger.call.who.two~shop", "mesh.mod.ledger.call.who.anchor.two~shop",
|
|
||||||
"mesh.seat.issue-tracker.call.open.two~shop"},
|
|
||||||
mayNot: []string{"mesh.mod.ledger.call.other.two~shop", "mesh.mod.ledger.call.who.one~shop",
|
|
||||||
"mesh.seat.issue-tracker.call.open.one~telegram"}},
|
|
||||||
{p: Principal{Kind: KindNode, Node: "one", Checks: []string{"ledger.health"}},
|
|
||||||
may: []string{"mesh.mod.ledger.call.health.one.node~one"},
|
|
||||||
mayNot: []string{"mesh.mod.ledger.call.health.two.node~one", "mesh.mod.ledger.call.health.one.node~two"}},
|
|
||||||
{p: Principal{Kind: KindModule, Node: "one", Module: "ledger", Holds: []Seat{ledger, tracker}},
|
|
||||||
subject: []string{"mesh.mod.ledger.call.who.person~jochen", "mesh.mod.ledger.call.who.one.controller",
|
|
||||||
"mesh.seat.node-desk.call.who.one.two~shop", "mesh.seat.issue-tracker.call.open.two~shop"}},
|
|
||||||
} {
|
|
||||||
perms, err := PermissionsFor(c.p)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%s: %v", c.p.Username(), err)
|
|
||||||
}
|
|
||||||
for _, s := range c.may {
|
|
||||||
if !MayPublish(perms, s) {
|
|
||||||
t.Errorf("%s may not call %s, in its own name", c.p.Username(), s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, s := range c.mayNot {
|
|
||||||
if MayPublish(perms, s) {
|
|
||||||
t.Errorf("%s may call %s, naming somebody else", c.p.Username(), s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, s := range c.subject {
|
|
||||||
if !MaySubscribe(perms, s) {
|
|
||||||
t.Errorf("%s does not hear %s, a call to what it serves", c.p.Username(), s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The subjects that name no caller stay granted beside the caller-named ones for one release, so a caller
|
|
||||||
// moves over without a gap (their retirement: hq issue 464).
|
|
||||||
func TestTheSubjectsThatNameNoCallerStayGrantedForOneRelease(t *testing.T) {
|
|
||||||
perms, err := PermissionsFor(Principal{Kind: KindModule, Node: "two", Module: "shop", Invokes: []string{"ledger.who"}})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, s := range []string{"mesh.mod.ledger.tool.who", "mesh.mod.ledger.tool.who.anchor"} {
|
|
||||||
if !MayPublish(perms, s) {
|
|
||||||
t.Errorf("the old subject %s is no longer granted", s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The desk's hidden prompt is the controller's alone on the caller-named subjects too (the review of
|
|
||||||
// 2026-10-09, M4): a grant of every tool does not reach it there either.
|
|
||||||
func TestTheDesksPromptIsTheControllersAloneUnderCallToo(t *testing.T) {
|
|
||||||
for _, p := range []Principal{{Kind: KindPerson, Module: "jochen", Invokes: []string{"*"}},
|
|
||||||
{Kind: KindNodeTools, Node: "shanks", Module: RuntimeModule}} {
|
|
||||||
perms, err := PermissionsFor(p)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
token := CallerToken(p.Username())
|
|
||||||
for _, s := range []string{"mesh.seat.node-launcher.call.secret.shanks." + token,
|
|
||||||
"mesh.mod.shell.call.node-launcher.secret.shanks." + token, "mesh.mod.shell.call.node-launcher.secret." + token} {
|
|
||||||
if MayPublish(perms, s) {
|
|
||||||
t.Errorf("%s may publish %s, the desk's hidden prompt", p.Username(), s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The controller's grants are the installer's first user list too, so they move with hq issue 464: this release
|
|
||||||
// it calls and serves on the subjects that name no caller alone, and nobody may call in its name.
|
|
||||||
func TestTheControllerKeepsTheSubjectsThatNameNoCallerThisRelease(t *testing.T) {
|
|
||||||
perms, err := PermissionsFor(Principal{Kind: KindController})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, s := range append(perms.Publish, perms.Subscribe...) {
|
|
||||||
if strings.Contains(s, "."+CallKind+".") {
|
|
||||||
t.Errorf("the controller is granted %s, which the installer's first user list does not carry", s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, p := range []Principal{{Kind: KindPerson, Module: "jochen", Invokes: []string{"*"}},
|
|
||||||
{Kind: KindNodeTools, Node: "shanks", Module: RuntimeModule}} {
|
|
||||||
perms, err := PermissionsFor(p)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if MayPublish(perms, "mesh.mod.ledger.call.who.controller") || MayPublish(perms, "mesh.seat.mesh-controller.call.status.controller") {
|
|
||||||
t.Errorf("%s may call in the controller's name", p.Username())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -10,8 +10,6 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"golang.org/x/crypto/bcrypt"
|
"golang.org/x/crypto/bcrypt"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/beside"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// **The first user list the installer carries must be the one the controller would compose.**
|
// **The first user list the installer carries must be the one the controller would compose.**
|
||||||
@@ -78,14 +76,13 @@ func theCarriedAccounts(t *testing.T) string {
|
|||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// theTemplate is the installer's bundle, as resources: the node-engine's, as a merge check clones it at
|
// theTemplate is the installer's bundle, as resources.
|
||||||
// the commit the mesh runs, or as captured in testdata/beside (internal/beside, novox/hq issue 432).
|
|
||||||
func theTemplate(t *testing.T) []map[string]any {
|
func theTemplate(t *testing.T) []map[string]any {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
path := filepath.Join(beside.Dir(t, "mesh-host"), "examples", "foundation-first-node-nats.lock")
|
path := filepath.Join("..", "..", "..", "mesh-host", "examples", "foundation-first-node-nats.lock")
|
||||||
raw, err := os.ReadFile(path)
|
raw, err := os.ReadFile(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Skipf("the host's checkout is not beside this one: %v", err)
|
||||||
}
|
}
|
||||||
// The template is JSON with line comments, which is how every one of them is written.
|
// The template is JSON with line comments, which is how every one of them is written.
|
||||||
var lines []string
|
var lines []string
|
||||||
|
|||||||
@@ -42,9 +42,8 @@ func TestInvokingGrantsNothingButTheCall(t *testing.T) {
|
|||||||
if strings.Contains(p, ".event.") {
|
if strings.Contains(p, ".event.") {
|
||||||
t.Errorf("a module that only invokes may publish %q, an event it never declared", p)
|
t.Errorf("a module that only invokes may publish %q, an event it never declared", p)
|
||||||
}
|
}
|
||||||
// A role's tools are tools (ADR 0132), named by their caller or not (novox/hq issue 365); a role's work
|
// A role's tools are tools (ADR 0132); a role's work queue and events are not.
|
||||||
// queue and events are not.
|
if strings.HasPrefix(p, "mesh.seat.") && !strings.Contains(p, ".tool.") {
|
||||||
if strings.HasPrefix(p, "mesh.seat.") && !strings.Contains(p, ".tool.") && !strings.Contains(p, ".call.") {
|
|
||||||
t.Errorf("a module that only invokes may publish %q, a seat it neither holds nor uses", p)
|
t.Errorf("a module that only invokes may publish %q, a seat it neither holds nor uses", p)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,53 +0,0 @@
|
|||||||
package broker
|
|
||||||
|
|
||||||
import "testing"
|
|
||||||
|
|
||||||
// The controller asks a tool that answers a software-issued credential in its own name, and nothing else under
|
|
||||||
// `call` (novox/hq ADR 0305 §2, issue 526): the grant is composed from the records, so the installer's first user
|
|
||||||
// list, a controller with no records, carries none.
|
|
||||||
func TestTheControllerAsksAnIssuedCredentialsToolInItsOwnNameAlone(t *testing.T) {
|
|
||||||
r := Records{Nodes: []string{"n", "ace"}, Assigned: map[string][]Declared{
|
|
||||||
"n": {{Module: "x", Issued: []string{"t"}}},
|
|
||||||
"ace": {{Module: "plex", Issued: []string{"plex_issued_credential"}}, {Module: "kometa"}},
|
|
||||||
}}
|
|
||||||
users, err := Users(r)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
controller := users[0]
|
|
||||||
if controller.Kind != KindController {
|
|
||||||
t.Fatalf("the first user is %s", controller.Username())
|
|
||||||
}
|
|
||||||
perms, err := PermissionsFor(controller)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, s := range []string{
|
|
||||||
"mesh.mod.x.call.t.n.controller",
|
|
||||||
"mesh.mod.plex.call.plex_issued_credential.ace.controller",
|
|
||||||
} {
|
|
||||||
if !MayPublish(perms, s) {
|
|
||||||
t.Errorf("the controller may not publish %s", s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, s := range []string{
|
|
||||||
"mesh.mod.x.call.t.n.ace.node-tools",
|
|
||||||
"mesh.mod.x.call.t.n." + CallerToken("ace.node-tools"),
|
|
||||||
"mesh.mod.plex.call.plex_issued_credential.ace." + CallerToken("ace.node-tools"),
|
|
||||||
"mesh.mod.plex.call.plex_status.ace.controller",
|
|
||||||
"mesh.mod.kometa.call.t.ace.controller",
|
|
||||||
"mesh.seat.mesh-controller.call.status.controller",
|
|
||||||
} {
|
|
||||||
if MayPublish(perms, s) {
|
|
||||||
t.Errorf("the controller may publish %s", s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
bare, err := PermissionsFor(Principal{Kind: KindController})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if MayPublish(bare, "mesh.mod.x.call.t.n.controller") {
|
|
||||||
t.Fatal("a controller composed with no records is granted a caller-named call")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -448,41 +448,3 @@ func (j *JetStream) BucketNames() ([]string, error) {
|
|||||||
}
|
}
|
||||||
return out, lister.Error()
|
return out, lister.Error()
|
||||||
}
|
}
|
||||||
|
|
||||||
// EnsureLog creates a module's log if it is absent and brings its configuration to match if it is
|
|
||||||
// present (novox/hq ADR 0297).
|
|
||||||
//
|
|
||||||
// **An update, never a delete and recreate**, for a bucket's reason: recreating discards what the log
|
|
||||||
// holds, and a log is a module's record. A configuration the server will not change in place (its
|
|
||||||
// storage, say, on a stream made by hand) is said as this assertion's error and the stream is left as
|
|
||||||
// it is — never removed to be made again.
|
|
||||||
func (j *JetStream) EnsureLog(l Log) error {
|
|
||||||
js, err := jetstream.New(j.conn)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
if _, err := js.CreateOrUpdateStream(ctx, l.Config()); err != nil {
|
|
||||||
return fmt.Errorf("asserting log %s: %w", l.Stream(), err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// LogStreams is every log stream on the server: every stream whose name starts with LogStreamPrefix.
|
|
||||||
func (j *JetStream) LogStreams() ([]string, error) {
|
|
||||||
js, err := jetstream.New(j.conn)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
lister := js.StreamNames(ctx)
|
|
||||||
var out []string
|
|
||||||
for name := range lister.Name() {
|
|
||||||
if strings.HasPrefix(name, LogStreamPrefix) {
|
|
||||||
out = append(out, name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out, lister.Err()
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,195 +0,0 @@
|
|||||||
package broker
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A module's log on the bus (novox/hq ADR 0297): its record of operations, each entry appended under
|
|
||||||
// a key and kept as long as the log is.
|
|
||||||
//
|
|
||||||
// A log follows a bucket in every respect (ADR 0201): the module names it locally, the mesh derives
|
|
||||||
// its stream and subjects, the controller creates it from the catalogue on every raise and from
|
|
||||||
// registration, never a module, and **no path of the mesh removes one** — a log whose declaration
|
|
||||||
// is gone is reported, as a bucket is. Unlike a bucket, a log is its owner's alone: no other module
|
|
||||||
// reads it, so there is no read of a log to grant or issue.
|
|
||||||
//
|
|
||||||
// Pure, but for the stream's configuration, which is the server's own type so that what is tested
|
|
||||||
// is exactly what is sent; jetstream.go is the part that asks a server.
|
|
||||||
|
|
||||||
// The mesh's caps on a log: what an entry's value may weigh, what a message on the log's stream may
|
|
||||||
// weigh — the value and its headers, which the server counts in a message's size, so a value of the
|
|
||||||
// full size still fits with the expected-last-sequence header an append carries — and what a log holds
|
|
||||||
// when its module says nothing and at most.
|
|
||||||
const (
|
|
||||||
LogMaxEntryBytes = 256 * 1024
|
|
||||||
LogMaxMessageBytes = LogMaxEntryBytes + 4*1024
|
|
||||||
LogDefaultMiB = 1024
|
|
||||||
LogMostMiB = 8192
|
|
||||||
)
|
|
||||||
|
|
||||||
// A Log is one module's declared log as the bus holds it.
|
|
||||||
type Log struct {
|
|
||||||
Module string
|
|
||||||
Name string
|
|
||||||
// MaxMiB is its cap in MiB; zero is LogDefaultMiB.
|
|
||||||
MaxMiB int
|
|
||||||
}
|
|
||||||
|
|
||||||
// LogStreamPrefix starts every log's stream name, which no other stream of the mesh's starts with.
|
|
||||||
const LogStreamPrefix = "LOG_"
|
|
||||||
|
|
||||||
// LogStreamName is the stream a module's log lives in: `LOG_<module>_<name>`. The module and the
|
|
||||||
// local name are each one token with no underscore, so two modules can never derive one stream.
|
|
||||||
func LogStreamName(module, name string) string { return LogStreamPrefix + module + "_" + name }
|
|
||||||
|
|
||||||
// LogSubject is the subject a log's entries are published under, without the key: an entry for key K
|
|
||||||
// is on `<LogSubject>.<K>`.
|
|
||||||
func LogSubject(module, name string) string { return "mesh.log." + module + "." + name }
|
|
||||||
|
|
||||||
// Stream is this log's stream name.
|
|
||||||
func (l Log) Stream() string { return LogStreamName(l.Module, l.Name) }
|
|
||||||
|
|
||||||
// Subject is this log's subject, without the key.
|
|
||||||
func (l Log) Subject() string { return LogSubject(l.Module, l.Name) }
|
|
||||||
|
|
||||||
// MaxBytes is this log's cap in bytes.
|
|
||||||
func (l Log) MaxBytes() int64 {
|
|
||||||
mib := l.MaxMiB
|
|
||||||
if mib <= 0 {
|
|
||||||
mib = LogDefaultMiB
|
|
||||||
}
|
|
||||||
return int64(mib) * 1024 * 1024
|
|
||||||
}
|
|
||||||
|
|
||||||
// Why is carried into the server's description of the stream, so somebody reading the server's own
|
|
||||||
// state finds whose it is and why it is kept.
|
|
||||||
func (l Log) Why() string {
|
|
||||||
return fmt.Sprintf("%s's log %q (novox/hq ADR 0297): its record of operations, one entry per operation "+
|
|
||||||
"under its key, appended by %s alone and kept as long as the log; never removed by the mesh, because "+
|
|
||||||
"it is data", l.Module, l.Name, l.Module)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Config is the stream a log is, field by field as novox/hq ADR 0297 fixes it: a file stream kept by
|
|
||||||
// limits, with no maximum age and no cap per key, whose message is an entry's value and 4 KiB of
|
|
||||||
// headers at most, that refuses a new entry when full rather than
|
|
||||||
// drop an old one, and that refuses deleting an entry or purging it. Direct gets are allowed, which
|
|
||||||
// is how the runtime reads it.
|
|
||||||
func (l Log) Config() jetstream.StreamConfig {
|
|
||||||
return jetstream.StreamConfig{
|
|
||||||
Name: l.Stream(),
|
|
||||||
Description: l.Why(),
|
|
||||||
Subjects: []string{l.Subject() + ".>"},
|
|
||||||
Storage: jetstream.FileStorage,
|
|
||||||
Retention: jetstream.LimitsPolicy,
|
|
||||||
MaxAge: 0,
|
|
||||||
MaxMsgs: -1,
|
|
||||||
MaxMsgsPerSubject: -1,
|
|
||||||
MaxBytes: l.MaxBytes(),
|
|
||||||
MaxMsgSize: LogMaxMessageBytes,
|
|
||||||
Discard: jetstream.DiscardNew,
|
|
||||||
AllowDirect: true,
|
|
||||||
DenyDelete: true,
|
|
||||||
DenyPurge: true,
|
|
||||||
Replicas: 1,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// LogIssued is one log an assignment may reach, by the name its module uses for it (novox/hq ADR
|
|
||||||
// 0297): its stream, the subject its entries go under, and whether it may append. Only the owner's
|
|
||||||
// instances are issued a log, so Writes is always true today; it is said so the runtime need not
|
|
||||||
// assume it.
|
|
||||||
type LogIssued struct {
|
|
||||||
Name string `json:"name"`
|
|
||||||
Stream string `json:"stream"`
|
|
||||||
Subject string `json:"subject"`
|
|
||||||
Writes bool `json:"writes"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// logsIssuedFor is every log a module's code may reach, as its membership lists them: its own.
|
|
||||||
func logsIssuedFor(d Declared) []LogIssued {
|
|
||||||
var out []LogIssued
|
|
||||||
for _, l := range d.Logs {
|
|
||||||
if !safeSubject.MatchString(d.Module) || !safeSubject.MatchString(l.Name) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
out = append(out, LogIssued{Name: l.Name, Stream: LogStreamName(d.Module, l.Name),
|
|
||||||
Subject: LogSubject(d.Module, l.Name), Writes: true})
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// logGrants is what a principal publishes to reach the logs its module keeps: for each, appending
|
|
||||||
// under the log's subjects, binding to its stream, and reading it directly. Nothing more — the
|
|
||||||
// runtime reads a log by direct gets alone and makes no consumer on it — and nothing of any other
|
|
||||||
// module's log, because a log is its owner's alone. Replies come on the principal's inbox, as for a
|
|
||||||
// bucket.
|
|
||||||
func logGrants(module string, names []string) []string {
|
|
||||||
if !safeSubject.MatchString(module) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
var out []string
|
|
||||||
for _, name := range names {
|
|
||||||
if !safeSubject.MatchString(name) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
stream := LogStreamName(module, name)
|
|
||||||
out = append(out,
|
|
||||||
LogSubject(module, name)+".>",
|
|
||||||
"$JS.API.STREAM.INFO."+stream,
|
|
||||||
"$JS.API.DIRECT.GET."+stream,
|
|
||||||
"$JS.API.DIRECT.GET."+stream+".>")
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// logNames is the local names of a module's logs.
|
|
||||||
func logNames(logs []Log) []string {
|
|
||||||
out := make([]string, 0, len(logs))
|
|
||||||
for _, l := range logs {
|
|
||||||
out = append(out, l.Name)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// A LogAsserter is the part of a JetStream connection log assertion needs. It has no way to remove a
|
|
||||||
// log, by design: nothing the mesh runs asks for one.
|
|
||||||
type LogAsserter interface {
|
|
||||||
// EnsureLog creates the log's stream if absent and brings its configuration to match if present,
|
|
||||||
// never deleting or recreating it.
|
|
||||||
EnsureLog(l Log) error
|
|
||||||
// LogStreams is every log stream on the server: every stream named with LogStreamPrefix.
|
|
||||||
LogStreams() ([]string, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// RaiseLogs asserts every declared log and answers the log streams on the server that nothing
|
|
||||||
// declares any more.
|
|
||||||
//
|
|
||||||
// **Those are reported, never removed** (novox/hq ADR 0297 §2, ADR 0201 §15–16): a log is a module's
|
|
||||||
// record, and a manifest edited, a module renamed or unassigned is an ordinary day's work that must
|
|
||||||
// not take a record with it. Removing one is a person's act, outside the mesh.
|
|
||||||
func RaiseLogs(a LogAsserter, logs []Log) (undeclared []string, err error) {
|
|
||||||
sorted := append([]Log(nil), logs...)
|
|
||||||
sort.Slice(sorted, func(i, j int) bool { return sorted[i].Stream() < sorted[j].Stream() })
|
|
||||||
declared := map[string]bool{}
|
|
||||||
for _, l := range sorted {
|
|
||||||
if err := a.EnsureLog(l); err != nil {
|
|
||||||
return nil, fmt.Errorf("asserting %s's log %q: %w", l.Module, l.Name, err)
|
|
||||||
}
|
|
||||||
declared[l.Stream()] = true
|
|
||||||
}
|
|
||||||
names, err := a.LogStreams()
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("listing the bus's logs: %w", err)
|
|
||||||
}
|
|
||||||
for _, n := range names {
|
|
||||||
if strings.HasPrefix(n, LogStreamPrefix) && !declared[n] {
|
|
||||||
undeclared = append(undeclared, n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.Strings(undeclared)
|
|
||||||
return undeclared, nil
|
|
||||||
}
|
|
||||||
@@ -1,294 +0,0 @@
|
|||||||
package broker
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"go/ast"
|
|
||||||
"go/parser"
|
|
||||||
"go/token"
|
|
||||||
"io/fs"
|
|
||||||
"path/filepath"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **The stream a log is, field by field** (novox/hq ADR 0297 §2, the shared contract): a file stream
|
|
||||||
// kept by limits, no maximum age, no cap per key, the declared cap, a message of at most 260 KiB (a
|
|
||||||
// value of 256 KiB and 4 KiB of headers),
|
|
||||||
// refusing what comes next when full, read directly, refusing a delete or a purge, one replica, and
|
|
||||||
// a description that says whose it is and why.
|
|
||||||
func TestALogsStreamIsAsTheDecisionFixesIt(t *testing.T) {
|
|
||||||
got := Log{Module: "mesh-issues", Name: "changes"}.Config()
|
|
||||||
want := jetstream.StreamConfig{
|
|
||||||
Name: "LOG_mesh-issues_changes",
|
|
||||||
Description: got.Description,
|
|
||||||
Subjects: []string{"mesh.log.mesh-issues.changes.>"},
|
|
||||||
Storage: jetstream.FileStorage,
|
|
||||||
Retention: jetstream.LimitsPolicy,
|
|
||||||
MaxAge: 0,
|
|
||||||
MaxMsgs: -1,
|
|
||||||
MaxMsgsPerSubject: -1,
|
|
||||||
MaxBytes: 1024 * 1024 * 1024,
|
|
||||||
MaxMsgSize: 260 * 1024,
|
|
||||||
Discard: jetstream.DiscardNew,
|
|
||||||
AllowDirect: true,
|
|
||||||
DenyDelete: true,
|
|
||||||
DenyPurge: true,
|
|
||||||
Replicas: 1,
|
|
||||||
}
|
|
||||||
a, _ := json.Marshal(got)
|
|
||||||
b, _ := json.Marshal(want)
|
|
||||||
if string(a) != string(b) {
|
|
||||||
t.Fatalf("the log's stream is\n %s\nwant\n %s", a, b)
|
|
||||||
}
|
|
||||||
if !strings.Contains(got.Description, "mesh-issues") || !strings.Contains(got.Description, "ADR 0297") {
|
|
||||||
t.Fatalf("the description does not say whose the log is and why: %q", got.Description)
|
|
||||||
}
|
|
||||||
if c := (Log{Module: "m", Name: "n", MaxMiB: 8192}).Config(); c.MaxBytes != 8192*1024*1024 {
|
|
||||||
t.Fatalf("a cap of 8192 MiB is %d bytes", c.MaxBytes)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **The membership names each of the owner's logs** with exactly the field names the runtime reads:
|
|
||||||
// `logs`, and in each `name`, `stream`, `subject`, `writes`. A module with no log is issued none, and
|
|
||||||
// the field is absent.
|
|
||||||
func TestAMembershipListsItsModulesLogs(t *testing.T) {
|
|
||||||
m := MembershipFor("one", Declared{Module: "mesh-issues",
|
|
||||||
Logs: []Log{{Module: "mesh-issues", Name: "changes"}}}, Placements{})
|
|
||||||
raw, err := json.Marshal(m)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var back map[string]json.RawMessage
|
|
||||||
if err := json.Unmarshal(raw, &back); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got := string(back["logs"]); got !=
|
|
||||||
`[{"name":"changes","stream":"LOG_mesh-issues_changes","subject":"mesh.log.mesh-issues.changes","writes":true}]` {
|
|
||||||
t.Fatalf("the membership's logs are %s", got)
|
|
||||||
}
|
|
||||||
none, _ := json.Marshal(MembershipFor("one", Declared{Module: "audit"}, Placements{}))
|
|
||||||
if strings.Contains(string(none), `"logs"`) {
|
|
||||||
t.Fatalf("a module with no log was issued logs: %s", none)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// logGrantsOf is the grants of a principal that are about logs.
|
|
||||||
func logGrantsOf(publish []string) []string {
|
|
||||||
var out []string
|
|
||||||
for _, s := range publish {
|
|
||||||
if strings.HasPrefix(s, "mesh.log.") || strings.Contains(s, ".LOG_") {
|
|
||||||
out = append(out, s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
slices.Sort(out)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// **The runtime is granted exactly the contract's subjects for each log it carries, and nothing else
|
|
||||||
// of any log**: appending under the log's subjects, binding to its stream, reading it directly. No
|
|
||||||
// consumer, no delete, no purge, and nothing of a log its modules do not keep.
|
|
||||||
func TestTheRuntimeIsGrantedItsModulesLogsAndNoMore(t *testing.T) {
|
|
||||||
perms, err := PermissionsFor(Principal{Kind: KindNodeTools, Node: "one", Module: RuntimeModule,
|
|
||||||
Carries: []Declared{
|
|
||||||
{Module: "mesh-issues", Logs: []Log{{Module: "mesh-issues", Name: "changes"}}},
|
|
||||||
{Module: "audit"},
|
|
||||||
}, PasswordHash: "x"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
want := []string{
|
|
||||||
"$JS.API.DIRECT.GET.LOG_mesh-issues_changes",
|
|
||||||
"$JS.API.DIRECT.GET.LOG_mesh-issues_changes.>",
|
|
||||||
"$JS.API.STREAM.INFO.LOG_mesh-issues_changes",
|
|
||||||
"mesh.log.mesh-issues.changes.>",
|
|
||||||
}
|
|
||||||
if got := logGrantsOf(perms.Publish); !slices.Equal(got, want) {
|
|
||||||
t.Fatalf("the runtime is granted\n %q\nwant\n %q", got, want)
|
|
||||||
}
|
|
||||||
for _, s := range perms.Subscribe {
|
|
||||||
if strings.HasPrefix(s, "mesh.log.") || strings.Contains(s, "LOG_") {
|
|
||||||
t.Fatalf("the runtime subscribes a log's subjects: %q", s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// A module running on its own account is granted the same for its own logs.
|
|
||||||
own, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "mesh-issues",
|
|
||||||
Logs: []string{"changes"}, PasswordHash: "x"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got := logGrantsOf(own.Publish); !slices.Equal(got, want) {
|
|
||||||
t.Fatalf("the module's own account is granted\n %q\nwant\n %q", got, want)
|
|
||||||
}
|
|
||||||
// And a module with no log, nothing of any.
|
|
||||||
none, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "audit", PasswordHash: "x"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got := logGrantsOf(none.Publish); len(got) != 0 {
|
|
||||||
t.Fatalf("a module with no log is granted %q", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A log name or module that is not one plain token is issued and granted nothing rather than a
|
|
||||||
// pattern that happens to parse.
|
|
||||||
func TestALogThatNamesNoStreamGrantsNothing(t *testing.T) {
|
|
||||||
if got := logGrants("a", []string{"x.y", "x>", "*", ""}); len(got) != 0 {
|
|
||||||
t.Fatalf("granted %v for logs that name no stream", got)
|
|
||||||
}
|
|
||||||
if got := logGrants("a.b", []string{"c"}); len(got) != 0 {
|
|
||||||
t.Fatalf("granted %v for a module that is not one token", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type logs struct {
|
|
||||||
ensured []string
|
|
||||||
on []string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l *logs) EnsureLog(x Log) error { l.ensured = append(l.ensured, x.Stream()); return nil }
|
|
||||||
func (l *logs) LogStreams() ([]string, error) { return l.on, nil }
|
|
||||||
|
|
||||||
// Every declared log is asserted; one on the server that nothing declares is said, not removed — and
|
|
||||||
// the asserter has no way to remove one.
|
|
||||||
func TestRaisingLogsReportsWhatNothingDeclares(t *testing.T) {
|
|
||||||
l := &logs{on: []string{"LOG_mesh-issues_changes", "LOG_gone_old", "KV_not_a_log"}}
|
|
||||||
undeclared, err := RaiseLogs(l, []Log{{Module: "mesh-issues", Name: "changes"}, {Module: "a", Name: "b"}})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !slices.Equal(l.ensured, []string{"LOG_a_b", "LOG_mesh-issues_changes"}) {
|
|
||||||
t.Fatalf("asserted %v", l.ensured)
|
|
||||||
}
|
|
||||||
if !slices.Equal(undeclared, []string{"LOG_gone_old"}) {
|
|
||||||
t.Fatalf("reported %v", undeclared)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **No path of the mesh deletes or purges a log or a bucket, or recreates one** (novox/hq ADR 0297 §2,
|
|
||||||
// ADR 0201 §15–16): no code of this repository outside its tests calls the client's stream or bucket
|
|
||||||
// delete, or purges a stream, but for the controller lease's own stream, which purges its own history
|
|
||||||
// below a sequence (internal/lease). A new call is a decision, not a refactor.
|
|
||||||
func TestNoPathDeletesALogOrABucket(t *testing.T) {
|
|
||||||
removers := map[string]bool{"DeleteStream": true, "DeleteKeyValue": true, "PurgeStream": true,
|
|
||||||
"DeleteObjectStore": true}
|
|
||||||
root := filepath.Join("..", "..")
|
|
||||||
var found []string
|
|
||||||
for _, dir := range []string{"cmd", "internal"} {
|
|
||||||
err := filepath.WalkDir(filepath.Join(root, dir), func(path string, e fs.DirEntry, err error) error {
|
|
||||||
if err != nil || e.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
f, err := parser.ParseFile(token.NewFileSet(), path, nil, 0)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
ast.Inspect(f, func(n ast.Node) bool {
|
|
||||||
call, ok := n.(*ast.CallExpr)
|
|
||||||
if !ok {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
sel, ok := call.Fun.(*ast.SelectorExpr)
|
|
||||||
if !ok {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
name := sel.Sel.Name
|
|
||||||
if removers[name] || (name == "Purge" && !strings.Contains(filepath.ToSlash(path), "internal/lease/")) {
|
|
||||||
found = append(found, path+": "+name)
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
})
|
|
||||||
for _, lit := range stringsIn(f) {
|
|
||||||
if strings.Contains(lit, "$JS.API.STREAM.DELETE") || strings.Contains(lit, "$JS.API.STREAM.PURGE") {
|
|
||||||
// Only the writers table names it, as a subject no one but the controller may publish.
|
|
||||||
if !strings.HasSuffix(filepath.ToSlash(path), "internal/broker/writers.go") {
|
|
||||||
found = append(found, path+": "+lit)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(found) > 0 {
|
|
||||||
t.Fatalf("a path of the mesh removes a stream, a bucket or what one holds:\n %s", strings.Join(found, "\n "))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// stringsIn is every string literal of a file.
|
|
||||||
func stringsIn(f *ast.File) []string {
|
|
||||||
var out []string
|
|
||||||
ast.Inspect(f, func(n ast.Node) bool {
|
|
||||||
if lit, ok := n.(*ast.BasicLit); ok && lit.Kind == token.STRING {
|
|
||||||
out = append(out, lit.Value)
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
})
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// Against a real server: a log is created as its configuration says, asserting it again keeps what
|
|
||||||
// it holds, a changed cap is brought to match in place, an entry cannot be deleted from it, and one
|
|
||||||
// nothing declares any more is reported and stays.
|
|
||||||
func TestALogIsAssertedInPlaceAndNeverRemoved(t *testing.T) {
|
|
||||||
bus := aLiveBus(t)
|
|
||||||
l := Log{Module: "logtest", Name: "changes", MaxMiB: 1}
|
|
||||||
if _, err := RaiseLogs(bus, []Log{l}); err != nil {
|
|
||||||
t.Fatalf("a real server refused a module's log: %v", err)
|
|
||||||
}
|
|
||||||
js, err := jetstream.New(bus.Conn())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
ack, err := js.Publish(ctx, l.Subject()+".7", []byte(`{"op":"opened"}`))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("an append to the log was refused: %v", err)
|
|
||||||
}
|
|
||||||
stream, err := js.Stream(ctx, l.Stream())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
have := stream.CachedInfo().Config
|
|
||||||
want := l.Config()
|
|
||||||
if have.Storage != want.Storage || have.Retention != want.Retention || have.MaxAge != 0 ||
|
|
||||||
have.MaxMsgsPerSubject != -1 || have.MaxBytes != want.MaxBytes || have.MaxMsgSize != want.MaxMsgSize ||
|
|
||||||
have.Discard != jetstream.DiscardNew || !have.AllowDirect || !have.DenyDelete || !have.DenyPurge ||
|
|
||||||
have.Replicas != 1 || !slices.Equal(have.Subjects, want.Subjects) {
|
|
||||||
t.Fatalf("the server holds the log as %+v", have)
|
|
||||||
}
|
|
||||||
if err := stream.DeleteMsg(ctx, ack.Sequence); err == nil {
|
|
||||||
t.Fatal("an entry was deleted from a log")
|
|
||||||
}
|
|
||||||
l.MaxMiB = 2
|
|
||||||
if _, err := RaiseLogs(bus, []Log{l}); err != nil {
|
|
||||||
t.Fatalf("asserting the log again failed, so a restart would: %v", err)
|
|
||||||
}
|
|
||||||
info, err := stream.Info(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if info.Config.MaxBytes != 2*1024*1024 {
|
|
||||||
t.Fatalf("the changed cap was not brought to match: %d", info.Config.MaxBytes)
|
|
||||||
}
|
|
||||||
if info.State.Msgs < 1 {
|
|
||||||
t.Fatal("asserting the log again lost what it held")
|
|
||||||
}
|
|
||||||
undeclared, err := RaiseLogs(bus, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !slices.Contains(undeclared, l.Stream()) {
|
|
||||||
t.Fatalf("a log nothing declares was not reported: %v", undeclared)
|
|
||||||
}
|
|
||||||
if _, err := js.Stream(ctx, l.Stream()); err != nil {
|
|
||||||
t.Fatalf("a log nothing declares is gone: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -51,10 +51,6 @@ type Membership struct {
|
|||||||
// and refuses, with the reason, what is not on it — the bus enforces only the union over every
|
// and refuses, with the reason, what is not on it — the bus enforces only the union over every
|
||||||
// module on the machine.
|
// module on the machine.
|
||||||
State []StateIssued `json:"state,omitempty"`
|
State []StateIssued `json:"state,omitempty"`
|
||||||
// Logs is every log this module's code may reach, by the name it uses for each (novox/hq ADR 0297):
|
|
||||||
// its own and no other's, since a log is its owner's alone. The runtime answers a bundle's log verbs
|
|
||||||
// from this list and refuses, with the reason, a log not on it.
|
|
||||||
Logs []LogIssued `json:"logs,omitempty"`
|
|
||||||
// SeatTraffic is what this module's code may submit, say, hear, take, ask, answer and read on seats
|
// SeatTraffic is what this module's code may submit, say, hear, take, ask, answer and read on seats
|
||||||
// that name their caller or their kind (novox/hq ADR 0259 §3). The runtime carrying the module
|
// that name their caller or their kind (novox/hq ADR 0259 §3). The runtime carrying the module
|
||||||
// publishes, takes and answers for it only what is listed here: the bus enforces only the union
|
// publishes, takes and answers for it only what is listed here: the bus enforces only the union
|
||||||
@@ -125,7 +121,6 @@ func MembershipFor(node string, d Declared, where Placements) Membership {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
m.State = stateIssuedFor(d, node)
|
m.State = stateIssuedFor(d, node)
|
||||||
m.Logs = logsIssuedFor(d)
|
|
||||||
t := SeatTrafficOf(d.Module, d.Holds, d.Uses, d.Watches)
|
t := SeatTrafficOf(d.Module, d.Holds, d.Uses, d.Watches)
|
||||||
for _, s := range append(append([]Seat{}, d.Uses...), d.Watches...) {
|
for _, s := range append(append([]Seat{}, d.Uses...), d.Watches...) {
|
||||||
if !s.Kinded {
|
if !s.Kinded {
|
||||||
|
|||||||
+68
-65
@@ -42,8 +42,64 @@ const (
|
|||||||
// Its authority is the union of what the modules it carries would each have had for their
|
// Its authority is the union of what the modules it carries would each have had for their
|
||||||
// tools — and nothing of what they consume, because tools are what it runs, not reactions.
|
// tools — and nothing of what they consume, because tools are what it runs, not reactions.
|
||||||
KindNodeTools Kind = "node-tools"
|
KindNodeTools Kind = "node-tools"
|
||||||
|
// KindView is the one read-only principal a view onto the bus connects as (novox/hq research 036,
|
||||||
|
// gap G1): a page in a browser, over the bus module's WebSocket listener, watching the issue tracker.
|
||||||
|
// Fixed, and derived from no declaration: what it hears is ViewHears, what it reads is ViewBucket,
|
||||||
|
// and it publishes nothing but direct reads of that one bucket (ViewReads), each answered in its own
|
||||||
|
// inbox. Composed like every other user, into the same
|
||||||
|
// file, once its credential is minted (`bus view-credential`); forgotten like every other user
|
||||||
|
// (`bus view-revoke`), at the next composition.
|
||||||
|
KindView Kind = "view"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// ViewUser is the view's one username: there is one view, and it is nobody's machine or module.
|
||||||
|
const ViewUser = "view"
|
||||||
|
|
||||||
|
// ViewBucket is the state the view reads: the issue tracker's issues, as the bus names the bucket
|
||||||
|
// (mesh-issues's state `issues`, novox/hq ADR 0201).
|
||||||
|
var ViewBucket = BucketName("mesh-issues", "issues")
|
||||||
|
|
||||||
|
// ViewHears are the events the view subscribes, each named: the issue tracker's own, the controller's
|
||||||
|
// walks and conditions, and the delivery owner's — what a page about issues shows beside them. Subscribe
|
||||||
|
// only, and no stream or consumer of its own: a page hears what happens while it is open, and reads the
|
||||||
|
// bucket for everything before.
|
||||||
|
var ViewHears = []string{
|
||||||
|
moduleEventSubject("mesh-issues", "opened"),
|
||||||
|
moduleEventSubject("mesh-issues", "moved"),
|
||||||
|
moduleEventSubject("mesh-issues", "noted"),
|
||||||
|
moduleEventSubject("mesh-issues", "linked"),
|
||||||
|
seatEventSubject(ControllerSeat, "plan-moved"),
|
||||||
|
seatEventSubject(ControllerSeat, "condition-raised"),
|
||||||
|
seatEventSubject(ControllerSeat, "condition-changed"),
|
||||||
|
seatEventSubject(ControllerSeat, "condition-cleared"),
|
||||||
|
moduleEventSubject("mesh-delivery", "transition"),
|
||||||
|
moduleEventSubject("mesh-delivery", "group"),
|
||||||
|
}
|
||||||
|
|
||||||
|
// ViewReads are the JetStream API requests the view makes, on ViewBucket's stream and no other: binding
|
||||||
|
// (STREAM.INFO), and direct reads — one key by its subject (`DIRECT.GET.<stream>.$KV.<bucket>.<key>`), and
|
||||||
|
// the batch form on the bare subject, which answers the newest value of every key (`multi_last`) into the
|
||||||
|
// asker's inbox. The page lists the bucket with the batch, and re-reads one key when the tracker's event
|
||||||
|
// names it (every event carries the issue's `number`).
|
||||||
|
//
|
||||||
|
// **No consumer, deliberately, and so no watch.** A KV watch is a push consumer, and a push consumer's
|
||||||
|
// deliver subject is the creator's choice, delivered by the server's own client — which the server does
|
||||||
|
// not hold to the creator's permissions. Measured on 2.11.17 (2026-10-10): the view, granted
|
||||||
|
// CONSUMER.CREATE on this stream, made a consumer delivering to `mesh.mod.mesh-issues.event.opened`, and
|
||||||
|
// a module subscribed there received the bucket's entry as the tracker's event. A grant of CONSUMER.CREATE
|
||||||
|
// is a publish to any subject in the account; the view publishes nothing, so it has none (nor
|
||||||
|
// CONSUMER.DELETE, which would let it delete a module's consumer). Nothing here is a write either: no
|
||||||
|
// `$KV.<bucket>.>`, which is what a put or a delete publishes to, and no STREAM.* that defines, purges or
|
||||||
|
// deletes.
|
||||||
|
func ViewReads() []string {
|
||||||
|
stream := "KV_" + ViewBucket
|
||||||
|
return []string{
|
||||||
|
"$JS.API.STREAM.INFO." + stream,
|
||||||
|
"$JS.API.DIRECT.GET." + stream,
|
||||||
|
"$JS.API.DIRECT.GET." + stream + ".>",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// RuntimeModule is the module that IS the node's tool runtime (novox/hq ADR 0175). Where it is
|
// RuntimeModule is the module that IS the node's tool runtime (novox/hq ADR 0175). Where it is
|
||||||
// assigned, the mesh composes one runtime principal for the machine in place of that module's own,
|
// assigned, the mesh composes one runtime principal for the machine in place of that module's own,
|
||||||
// and the per-module containers that served tools until then stop being the way tools reach a node.
|
// and the per-module containers that served tools until then stop being the way tools reach a node.
|
||||||
@@ -123,13 +179,6 @@ type Principal struct {
|
|||||||
// permission and nothing beside it.
|
// permission and nothing beside it.
|
||||||
Invokes []string
|
Invokes []string
|
||||||
|
|
||||||
// AsksIssued are the tools the controller asks in its own name for a credential a provider's software
|
|
||||||
// issued, each `<module>.<tool>` (novox/hq ADR 0305): on the subject that names its caller, so the tool
|
|
||||||
// knows the controller asks. Composed from the records, so the installer's first user list — a controller
|
|
||||||
// with no records — carries none, and moves with issue 464 like the rest of the controller's caller-named
|
|
||||||
// grants.
|
|
||||||
AsksIssued []string
|
|
||||||
|
|
||||||
// State is the local names of the state this principal's module keeps, and Reads the state of
|
// State is the local names of the state this principal's module keeps, and Reads the state of
|
||||||
// others it reads as `<module>.<name>` (novox/hq ADR 0201): a bucket each, kept by the owner's
|
// others it reads as `<module>.<name>` (novox/hq ADR 0201): a bucket each, kept by the owner's
|
||||||
// instances and read by whoever declares it.
|
// instances and read by whoever declares it.
|
||||||
@@ -139,8 +188,6 @@ type Principal struct {
|
|||||||
// no other; KeyedReads the keys of others' state it reads one key at a time (novox/hq ADR 0260).
|
// no other; KeyedReads the keys of others' state it reads one key at a time (novox/hq ADR 0260).
|
||||||
PerMachine []string
|
PerMachine []string
|
||||||
KeyedReads []KeyedRead
|
KeyedReads []KeyedRead
|
||||||
// Logs is the local names of the logs this principal's module keeps (novox/hq ADR 0297).
|
|
||||||
Logs []string
|
|
||||||
|
|
||||||
// SnapshotsTheBus is the bus's own module, the one holding mesh-broker (novox/hq ADR 0235). Its
|
// SnapshotsTheBus is the bus's own module, the one holding mesh-broker (novox/hq ADR 0235). Its
|
||||||
// whole authority is BusSnapshotGrants: it copies the streams for the night's backup and nothing
|
// whole authority is BusSnapshotGrants: it copies the streams for the night's backup and nothing
|
||||||
@@ -172,21 +219,6 @@ type Principal struct {
|
|||||||
// goes with the retired seat row.
|
// goes with the retired seat row.
|
||||||
var seatsTheControllerAsks = []string{"node-build-agent", "mesh-build-machine"}
|
var seatsTheControllerAsks = []string{"node-build-agent", "mesh-build-machine"}
|
||||||
|
|
||||||
// TheControllersAsk says whether a message of stream, published on subject, is an ask the controller
|
|
||||||
// itself makes: one on the accept subject of a seat in seatsTheControllerAsks, kept in that seat's own
|
|
||||||
// work queue. Such an ask, given up on by the seat's worker, can be delivered again with the authority
|
|
||||||
// the controller already holds — the publish on that seat's accepts and the stream API to remove the
|
|
||||||
// original — and no other can (novox/hq issue 334, ADR 0264's consequences: no grant over the seats'
|
|
||||||
// queues).
|
|
||||||
func TheControllersAsk(stream, subject string) bool {
|
|
||||||
for _, seat := range seatsTheControllerAsks {
|
|
||||||
if stream == seatStreamName(seat) && strings.HasPrefix(subject, "mesh.seat."+seat+".accept.") {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// SeatVerb is one verb of one seat, on every machine holding it.
|
// SeatVerb is one verb of one seat, on every machine holding it.
|
||||||
type SeatVerb struct{ Seat, Verb string }
|
type SeatVerb struct{ Seat, Verb string }
|
||||||
|
|
||||||
@@ -221,10 +253,6 @@ func ControllerOnly() []string {
|
|||||||
for _, base := range []string{"mesh.seat." + v.Seat + ".tool." + v.Verb, "mesh.mod.*.tool." + v.Seat + "." + v.Verb} {
|
for _, base := range []string{"mesh.seat." + v.Seat + ".tool." + v.Verb, "mesh.mod.*.tool." + v.Seat + "." + v.Verb} {
|
||||||
out = append(out, base, base+".*")
|
out = append(out, base, base+".*")
|
||||||
}
|
}
|
||||||
// And where a call names its caller (novox/hq issue 365): any machine, any caller.
|
|
||||||
for _, base := range []string{"mesh.seat." + v.Seat + "." + CallKind + "." + v.Verb, "mesh.mod.*." + CallKind + "." + v.Seat + "." + v.Verb} {
|
|
||||||
out = append(out, base+".>")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
@@ -260,11 +288,8 @@ func MaySubscribe(perms Permissions, subject string) bool {
|
|||||||
//
|
//
|
||||||
// And, since novox/hq ADR 0259, `release` and `stop`: the controller asks the operator for them about a
|
// And, since novox/hq ADR 0259, `release` and `stop`: the controller asks the operator for them about a
|
||||||
// delivery held past its bound, and calls them on the operator's warrant, with its why.
|
// delivery held past its bound, and calls them on the operator's warrant, with its why.
|
||||||
//
|
|
||||||
// And, since novox/hq ADR 0282, `times`: the self-check reads the delivery times to say a delivery over its budget.
|
|
||||||
var VerbsTheControllerAsksTheDeliveryOwner = []SeatVerb{{Seat: "mesh-delivery", Verb: "stalled"},
|
var VerbsTheControllerAsksTheDeliveryOwner = []SeatVerb{{Seat: "mesh-delivery", Verb: "stalled"},
|
||||||
{Seat: "mesh-delivery", Verb: "close"}, {Seat: "mesh-delivery", Verb: "release"}, {Seat: "mesh-delivery", Verb: "stop"},
|
{Seat: "mesh-delivery", Verb: "close"}, {Seat: "mesh-delivery", Verb: "release"}, {Seat: "mesh-delivery", Verb: "stop"}}
|
||||||
{Seat: "mesh-delivery", Verb: "times"}}
|
|
||||||
|
|
||||||
// VerbsTheControllerActsOnAWarrant are the other seat verbs the controller calls when the operator's warrant
|
// VerbsTheControllerActsOnAWarrant are the other seat verbs the controller calls when the operator's warrant
|
||||||
// chooses them (novox/hq ADR 0259): a machine's service restarted, and a walk started or stopped through the
|
// chooses them (novox/hq ADR 0259): a machine's service restarted, and a walk started or stopped through the
|
||||||
@@ -292,6 +317,8 @@ func (p Principal) Username() string {
|
|||||||
switch p.Kind {
|
switch p.Kind {
|
||||||
case KindPerson:
|
case KindPerson:
|
||||||
return "person." + p.Module
|
return "person." + p.Module
|
||||||
|
case KindView:
|
||||||
|
return ViewUser
|
||||||
case KindModule, KindNodeTools:
|
case KindModule, KindNodeTools:
|
||||||
// The runtime is named exactly as the module it stands for would have been: the mesh
|
// The runtime is named exactly as the module it stands for would have been: the mesh
|
||||||
// issues its credential through the same path a module's takes (`module issue`), and
|
// issues its credential through the same path a module's takes (`module issue`), and
|
||||||
@@ -564,6 +591,14 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
|||||||
// itself, its replies to the asker's own inbox.
|
// itself, its replies to the asker's own inbox.
|
||||||
pub = append(pub, discovering()...)
|
pub = append(pub, discovering()...)
|
||||||
|
|
||||||
|
case KindView:
|
||||||
|
// Hears what it is for and reads one bucket, and nothing else (ViewHears, ViewReads): no tool,
|
||||||
|
// no event of its own, no stream, no bucket written. Its requests are answered in its own
|
||||||
|
// inbox, granted below with the person's; a reply to anything is never permitted, because
|
||||||
|
// nothing is ever asked of it.
|
||||||
|
sub = append(sub, ViewHears...)
|
||||||
|
pub = append(pub, ViewReads()...)
|
||||||
|
|
||||||
case KindEnrolment:
|
case KindEnrolment:
|
||||||
// A leaked token is useless for anything but enrolling: it cannot read a declaration, hear
|
// A leaked token is useless for anything but enrolling: it cannot read a declaration, hear
|
||||||
// an event, or subscribe any inbox but the one its own token derives (design 25 §6).
|
// an event, or subscribe any inbox but the one its own token derives (design 25 §6).
|
||||||
@@ -755,8 +790,6 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
|||||||
// watched, its own written too.
|
// watched, its own written too.
|
||||||
pub = append(pub, stateGrants(stateAccess{Module: p.Module, Node: p.Node, Keeps: p.State,
|
pub = append(pub, stateGrants(stateAccess{Module: p.Module, Node: p.Node, Keeps: p.State,
|
||||||
PerMachine: p.PerMachine, Reads: p.Reads, KeyedReads: p.KeyedReads})...)
|
PerMachine: p.PerMachine, Reads: p.Reads, KeyedReads: p.KeyedReads})...)
|
||||||
// And its logs (novox/hq ADR 0297): appended to and read directly, its own alone.
|
|
||||||
pub = append(pub, logGrants(p.Module, p.Logs)...)
|
|
||||||
|
|
||||||
// 6. Its traffic on seats that name their caller or their kind, ask proofs or keep records
|
// 6. Its traffic on seats that name their caller or their kind, ask proofs or keep records
|
||||||
// (novox/hq ADR 0259 §3).
|
// (novox/hq ADR 0259 §3).
|
||||||
@@ -844,12 +877,6 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
|||||||
pub = append(pub, stateGrants(stateAccess{Module: d.Module, Node: p.Node, Keeps: stateNames(d.State),
|
pub = append(pub, stateGrants(stateAccess{Module: d.Module, Node: p.Node, Keeps: stateNames(d.State),
|
||||||
PerMachine: perMachineNames(d.State), Reads: d.Reads, KeyedReads: d.KeyedReads})...)
|
PerMachine: perMachineNames(d.State), Reads: d.Reads, KeyedReads: d.KeyedReads})...)
|
||||||
}
|
}
|
||||||
// **And it keeps the logs of the modules it carries** (novox/hq ADR 0297): each module's own, the
|
|
||||||
// union over them. That one module's code does not append to another's log through it is the
|
|
||||||
// runtime's to keep, from the logs each membership lists.
|
|
||||||
for _, d := range p.Carries {
|
|
||||||
pub = append(pub, logGrants(d.Module, logNames(d.Logs))...)
|
|
||||||
}
|
|
||||||
// **Never the traffic of a trusted holder** (novox/hq ADR 0259 §8): the machine's runtime runs as the
|
// **Never the traffic of a trusted holder** (novox/hq ADR 0259 §8): the machine's runtime runs as the
|
||||||
// operator's account, which every agent runs as, so a module saying warrants or speaking for a kind
|
// operator's account, which every agent runs as, so a module saying warrants or speaking for a kind
|
||||||
// that proves its sender is never composed into it — refused here, naming it, whatever registration
|
// that proves its sender is never composed into it — refused here, naming it, whatever registration
|
||||||
@@ -873,7 +900,7 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
|||||||
pub = unique(pub)
|
pub = unique(pub)
|
||||||
}
|
}
|
||||||
|
|
||||||
if p.Kind == KindPerson {
|
if p.Kind == KindPerson || p.Kind == KindView {
|
||||||
// An inbox to hear answers in, and nothing else. No ack subject: a person has no durable
|
// An inbox to hear answers in, and nothing else. No ack subject: a person has no durable
|
||||||
// consumer, because nothing is delivered to a person — they ask and are answered.
|
// consumer, because nothing is delivered to a person — they ask and are answered.
|
||||||
sub = append(sub, p.inbox())
|
sub = append(sub, p.inbox())
|
||||||
@@ -896,30 +923,6 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
|||||||
pub = append(pub, "$JS.ACK."+consumerStream(p)+"."+consumerDurable(p)+".>")
|
pub = append(pub, "$JS.ACK."+consumerStream(p)+"."+consumerDurable(p)+".>")
|
||||||
}
|
}
|
||||||
|
|
||||||
// **And every tool grant again, naming its caller** (novox/hq issue 365): a call in this principal's own
|
|
||||||
// name, and an answer to a call naming anybody. The `tool` grants above stay for one release beside these,
|
|
||||||
// so callers and runtimes move without a gap; their retirement is hq issue 464.
|
|
||||||
//
|
|
||||||
// **Not the controller's, this release.** Its grants are also the installer's first user list
|
|
||||||
// (TestTheInstallersFirstUserListIsWhatTheControllerWouldCompose), judged against the node-engine the mesh
|
|
||||||
// runs, so a change to them waits on a node-engine delivery. It calls and serves on the subjects that name
|
|
||||||
// no caller meanwhile — a caller-named call to its seat reaches nobody and is asked again on those at once —
|
|
||||||
// and moves with issue 464.
|
|
||||||
if p.Kind != KindController {
|
|
||||||
pub, sub = callerNamed(p.Username(), pub, sub)
|
|
||||||
} else {
|
|
||||||
// **But the tools that answer a software-issued credential** (novox/hq ADR 0305 §2): those answer the
|
|
||||||
// controller alone, and know it only from the subject's last token, so it asks them in its own name —
|
|
||||||
// those tools, on any machine, and nothing else under `call`.
|
|
||||||
for _, key := range p.AsksIssued {
|
|
||||||
module, tool, ok := strings.Cut(key, ".")
|
|
||||||
if !ok || module == "" || tool == "" || strings.ContainsAny(key, "*> ") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
pub = append(pub, "mesh.mod."+module+"."+CallKind+"."+tool+".*."+CallerToken(p.Username()))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
sort.Strings(pub)
|
sort.Strings(pub)
|
||||||
sort.Strings(sub)
|
sort.Strings(sub)
|
||||||
// One writer per piece of state (novox/hq to-be 45 §1): a grant that would make a second is
|
// One writer per piece of state (novox/hq to-be 45 §1): a grant that would make a second is
|
||||||
|
|||||||
@@ -31,6 +31,8 @@ func TestTheComposedConfigMatchesTheGolden(t *testing.T) {
|
|||||||
// The bus's own module: the snapshot API and its inbox, nothing else (novox/hq ADR 0235).
|
// The bus's own module: the snapshot API and its inbox, nothing else (novox/hq ADR 0235).
|
||||||
{Kind: KindModule, Node: "one", Module: "nats", SnapshotsTheBus: true,
|
{Kind: KindModule, Node: "one", Module: "nats", SnapshotsTheBus: true,
|
||||||
Serves: []string{"nats_streams"}, PasswordHash: "$2a$11$bbbbbbbbbbbbbbbbbbbbbb"},
|
Serves: []string{"nats_streams"}, PasswordHash: "$2a$11$bbbbbbbbbbbbbbbbbbbbbb"},
|
||||||
|
// The view: hears the issue tracker and reads its bucket, writes nothing (research 036).
|
||||||
|
{Kind: KindView, PasswordHash: "$2a$11$vvvvvvvvvvvvvvvvvvvvvv"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
@@ -240,9 +240,9 @@ func TestAPersonReachesNothingButTools(t *testing.T) {
|
|||||||
perms, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo",
|
perms, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo",
|
||||||
Invokes: []string{"*"}, PasswordHash: "x"})
|
Invokes: []string{"*"}, PasswordHash: "x"})
|
||||||
for _, p := range perms.Publish {
|
for _, p := range perms.Publish {
|
||||||
// A tool call — named by its caller or not (novox/hq issue 365) — or asking what answers (novox/hq
|
// A tool call, or asking what answers (novox/hq ADR 0197) — a question every service
|
||||||
// ADR 0197), a question every service answers about itself, which claims nothing and controls nothing.
|
// answers about itself, which claims nothing and controls nothing.
|
||||||
if !strings.Contains(p, ".tool.") && !strings.Contains(p, ".call.") && !strings.HasPrefix(p, "$SRV.") {
|
if !strings.Contains(p, ".tool.") && !strings.HasPrefix(p, "$SRV.") {
|
||||||
t.Errorf("a person may publish %q, which is not a tool call", p)
|
t.Errorf("a person may publish %q, which is not a tool call", p)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+8
-4
@@ -24,7 +24,7 @@ accounts {
|
|||||||
jetstream: enabled
|
jetstream: enabled
|
||||||
users = [
|
users = [
|
||||||
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
||||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*"] }
|
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*"] }
|
||||||
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] }
|
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] }
|
||||||
allow_responses: { max: 1, ttl: "1m" }
|
allow_responses: { max: 1, ttl: "1m" }
|
||||||
} }
|
} }
|
||||||
@@ -43,19 +43,23 @@ accounts {
|
|||||||
} }
|
} }
|
||||||
{ user: "one.telegram", password: "$2a$11$tttttttttttttttttttttt", permissions: {
|
{ user: "one.telegram", password: "$2a$11$tttttttttttttttttttttt", permissions: {
|
||||||
publish: { allow: ["$JS.ACK.EVENTS.one_telegram.>", "$JS.ACK.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker.>", "$JS.API.CONSUMER.INFO.EVENTS.one_telegram", "$JS.API.CONSUMER.INFO.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_telegram", "$JS.API.CONSUMER.MSG.NEXT.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.one.telegram", "mesh.seat.telegram-sender.event.delivered", "mesh.seat.telegram-sender.event.failed"] }
|
publish: { allow: ["$JS.ACK.EVENTS.one_telegram.>", "$JS.ACK.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker.>", "$JS.API.CONSUMER.INFO.EVENTS.one_telegram", "$JS.API.CONSUMER.INFO.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_telegram", "$JS.API.CONSUMER.MSG.NEXT.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.one.telegram", "mesh.seat.telegram-sender.event.delivered", "mesh.seat.telegram-sender.event.failed"] }
|
||||||
subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.telegram", "$SRV.INFO.telegram.>", "$SRV.PING", "$SRV.PING.telegram", "$SRV.PING.telegram.>", "$SRV.STATS", "$SRV.STATS.telegram", "$SRV.STATS.telegram.>", "_INBOX.one.telegram.>", "mesh.assignment.one.telegram", "mesh.mod.telegram.call.>", "mesh.mod.telegram.tool.>", "mesh.seat.telegram-sender.accept.send"] }
|
subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.telegram", "$SRV.INFO.telegram.>", "$SRV.PING", "$SRV.PING.telegram", "$SRV.PING.telegram.>", "$SRV.STATS", "$SRV.STATS.telegram", "$SRV.STATS.telegram.>", "_INBOX.one.telegram.>", "mesh.assignment.one.telegram", "mesh.mod.telegram.tool.>", "mesh.seat.telegram-sender.accept.send"] }
|
||||||
allow_responses: { max: 1, ttl: "1m" }
|
allow_responses: { max: 1, ttl: "1m" }
|
||||||
} }
|
} }
|
||||||
{ user: "two.audit", password: "$2a$11$aaaaaaaaaaaaaaaaaaaaaa", permissions: {
|
{ user: "two.audit", password: "$2a$11$aaaaaaaaaaaaaaaaaaaaaa", permissions: {
|
||||||
publish: { allow: ["$JS.ACK.EVENTS.two_audit.>", "$JS.API.CONSUMER.INFO.EVENTS.two_audit", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_audit", "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.two.audit"] }
|
publish: { allow: ["$JS.ACK.EVENTS.two_audit.>", "$JS.API.CONSUMER.INFO.EVENTS.two_audit", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_audit", "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.two.audit"] }
|
||||||
subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.audit", "$SRV.INFO.audit.>", "$SRV.PING", "$SRV.PING.audit", "$SRV.PING.audit.>", "$SRV.STATS", "$SRV.STATS.audit", "$SRV.STATS.audit.>", "_INBOX.two.audit.>", "mesh.assignment.two.audit", "mesh.mod.audit.call.>", "mesh.mod.audit.tool.>", "mesh.mod.shop.event.order.placed"] }
|
subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.audit", "$SRV.INFO.audit.>", "$SRV.PING", "$SRV.PING.audit", "$SRV.PING.audit.>", "$SRV.STATS", "$SRV.STATS.audit", "$SRV.STATS.audit.>", "_INBOX.two.audit.>", "mesh.assignment.two.audit", "mesh.mod.audit.tool.>", "mesh.mod.shop.event.order.placed"] }
|
||||||
allow_responses: { max: 1, ttl: "1m" }
|
allow_responses: { max: 1, ttl: "1m" }
|
||||||
} }
|
} }
|
||||||
{ user: "two.shop", password: "$2a$11$ssssssssssssssssssssss", permissions: {
|
{ user: "two.shop", password: "$2a$11$ssssssssssssssssssssss", permissions: {
|
||||||
publish: { allow: ["$JS.ACK.EVENTS.two_shop.>", "$JS.API.CONSUMER.INFO.EVENTS.two_shop", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_shop", "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.two.shop", "mesh.mod.shop.event.order.placed", "mesh.seat.telegram-sender.accept.send"] }
|
publish: { allow: ["$JS.ACK.EVENTS.two_shop.>", "$JS.API.CONSUMER.INFO.EVENTS.two_shop", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_shop", "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.two.shop", "mesh.mod.shop.event.order.placed", "mesh.seat.telegram-sender.accept.send"] }
|
||||||
subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.shop", "$SRV.INFO.shop.>", "$SRV.PING", "$SRV.PING.shop", "$SRV.PING.shop.>", "$SRV.STATS", "$SRV.STATS.shop", "$SRV.STATS.shop.>", "_INBOX.two.shop.>", "mesh.assignment.two.shop", "mesh.mod.shop.call.>", "mesh.mod.shop.tool.>"] }
|
subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.shop", "$SRV.INFO.shop.>", "$SRV.PING", "$SRV.PING.shop", "$SRV.PING.shop.>", "$SRV.STATS", "$SRV.STATS.shop", "$SRV.STATS.shop.>", "_INBOX.two.shop.>", "mesh.assignment.two.shop", "mesh.mod.shop.tool.>"] }
|
||||||
allow_responses: { max: 1, ttl: "1m" }
|
allow_responses: { max: 1, ttl: "1m" }
|
||||||
} }
|
} }
|
||||||
|
{ user: "view", password: "$2a$11$vvvvvvvvvvvvvvvvvvvvvv", permissions: {
|
||||||
|
publish: { allow: ["$JS.API.DIRECT.GET.KV_mesh-issues_issues", "$JS.API.DIRECT.GET.KV_mesh-issues_issues.>", "$JS.API.STREAM.INFO.KV_mesh-issues_issues"] }
|
||||||
|
subscribe: { allow: ["_INBOX.view.>", "mesh.mod.mesh-delivery.event.group", "mesh.mod.mesh-delivery.event.transition", "mesh.mod.mesh-issues.event.linked", "mesh.mod.mesh-issues.event.moved", "mesh.mod.mesh-issues.event.noted", "mesh.mod.mesh-issues.event.opened", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.plan-moved"] }
|
||||||
|
} }
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -29,17 +29,12 @@ type Declared struct {
|
|||||||
Holds []Seat
|
Holds []Seat
|
||||||
// Uses are the seats this module sends to.
|
// Uses are the seats this module sends to.
|
||||||
Uses []Seat
|
Uses []Seat
|
||||||
// Issued are the tools that answer a credential its software issued, one per offer that says so (novox/hq
|
|
||||||
// ADR 0305): the controller is granted to ask each in its own name, and no other tool so.
|
|
||||||
Issued []string
|
|
||||||
// Watches are the seats whose events it consumes.
|
// Watches are the seats whose events it consumes.
|
||||||
Watches []Seat
|
Watches []Seat
|
||||||
// Invokes are the tools it calls, `<module>.<tool>` or `*` (novox/hq ADR 0152).
|
// Invokes are the tools it calls, `<module>.<tool>` or `*` (novox/hq ADR 0152).
|
||||||
Invokes []string
|
Invokes []string
|
||||||
// State is the state it keeps, each a bucket its instances write (novox/hq ADR 0201).
|
// State is the state it keeps, each a bucket its instances write (novox/hq ADR 0201).
|
||||||
State []Bucket
|
State []Bucket
|
||||||
// Logs are the logs it keeps, each a stream its instances append to (novox/hq ADR 0297).
|
|
||||||
Logs []Log
|
|
||||||
// Reads are other modules' state it reads, each `<module>.<name>` (novox/hq ADR 0201).
|
// Reads are other modules' state it reads, each `<module>.<name>` (novox/hq ADR 0201).
|
||||||
Reads []string
|
Reads []string
|
||||||
// KeyedReads are keys of other modules' state it reads, each one key alone: what a seat's holder is
|
// KeyedReads are keys of other modules' state it reads, each one key alone: what a seat's holder is
|
||||||
@@ -72,6 +67,9 @@ type Records struct {
|
|||||||
Enrolling []string
|
Enrolling []string
|
||||||
// People is each person's name against the tools they may invoke, `*` for an administrator.
|
// People is each person's name against the tools they may invoke, `*` for an administrator.
|
||||||
People map[string][]string
|
People map[string][]string
|
||||||
|
// View says the mesh minted the view's credential (`bus view-credential`), so the one read-only
|
||||||
|
// view principal is composed (KindView); forgotten, it is left out, like a person.
|
||||||
|
View bool
|
||||||
// Interchangeable is each module whose definition says its instances are the same anywhere
|
// Interchangeable is each module whose definition says its instances are the same anywhere
|
||||||
// (ADR 0160), which decides whether the module's plain subject is issued to every instance.
|
// (ADR 0160), which decides whether the module's plain subject is issued to every instance.
|
||||||
Interchangeable map[string]bool
|
Interchangeable map[string]bool
|
||||||
@@ -87,7 +85,7 @@ type Records struct {
|
|||||||
// is a mesh that cannot be told anything, and there is no state of the records in which that is
|
// is a mesh that cannot be told anything, and there is no state of the records in which that is
|
||||||
// correct.
|
// correct.
|
||||||
func Users(r Records) ([]Principal, error) {
|
func Users(r Records) ([]Principal, error) {
|
||||||
out := []Principal{{Kind: KindController, Uses: asksSeatOf(r), AsksIssued: issuedOf(r)}}
|
out := []Principal{{Kind: KindController, Uses: asksSeatOf(r)}}
|
||||||
|
|
||||||
for _, node := range sortedCopy(r.Nodes) {
|
for _, node := range sortedCopy(r.Nodes) {
|
||||||
witness := false
|
witness := false
|
||||||
@@ -129,7 +127,6 @@ func Users(r Records) ([]Principal, error) {
|
|||||||
Holds: d.Holds, Uses: d.Uses, Watches: d.Watches, Invokes: d.Invokes,
|
Holds: d.Holds, Uses: d.Uses, Watches: d.Watches, Invokes: d.Invokes,
|
||||||
State: stateNames(d.State), Reads: d.Reads, SnapshotsTheBus: d.SnapshotsTheBus,
|
State: stateNames(d.State), Reads: d.Reads, SnapshotsTheBus: d.SnapshotsTheBus,
|
||||||
PerMachine: perMachineNames(d.State), KeyedReads: d.KeyedReads,
|
PerMachine: perMachineNames(d.State), KeyedReads: d.KeyedReads,
|
||||||
Logs: logNames(d.Logs),
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
if runtimeHere {
|
if runtimeHere {
|
||||||
@@ -148,6 +145,9 @@ func Users(r Records) ([]Principal, error) {
|
|||||||
for _, person := range sortedNames(r.People) {
|
for _, person := range sortedNames(r.People) {
|
||||||
out = append(out, Principal{Kind: KindPerson, Module: person, Invokes: r.People[person]})
|
out = append(out, Principal{Kind: KindPerson, Module: person, Invokes: r.People[person]})
|
||||||
}
|
}
|
||||||
|
if r.View {
|
||||||
|
out = append(out, Principal{Kind: KindView})
|
||||||
|
}
|
||||||
|
|
||||||
// Refused here rather than discovered by the server. Two users with one name is a file the
|
// Refused here rather than discovered by the server. Two users with one name is a file the
|
||||||
// server reads as one of them, and which one depends on the order — so a module assigned to a
|
// server reads as one of them, and which one depends on the order — so a module assigned to a
|
||||||
@@ -209,25 +209,6 @@ const controllerModule = "mesh-controller"
|
|||||||
// asksSeatOf is the seat an ask is made on, as its holder declares it (novox/hq ADR 0259): the controller
|
// asksSeatOf is the seat an ask is made on, as its holder declares it (novox/hq ADR 0259): the controller
|
||||||
// asks the operator through it like any other user, and is granted what its declaration names for a caller.
|
// asks the operator through it like any other user, and is granted what its declaration names for a caller.
|
||||||
// None while nothing holds it.
|
// None while nothing holds it.
|
||||||
// issuedOf is every tool of a module assigned anywhere that answers a credential its software issued, as
|
|
||||||
// `<module>.<tool>` (novox/hq ADR 0305): what the controller asks in its own name, and nothing beside.
|
|
||||||
func issuedOf(r Records) []string {
|
|
||||||
seen := map[string]bool{}
|
|
||||||
var out []string
|
|
||||||
for _, node := range sortedCopy(r.Nodes) {
|
|
||||||
for _, d := range r.Assigned[node] {
|
|
||||||
for _, t := range d.Issued {
|
|
||||||
if key := d.Module + "." + t; !seen[key] {
|
|
||||||
seen[key] = true
|
|
||||||
out = append(out, key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.Strings(out)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func asksSeatOf(r Records) []Seat {
|
func asksSeatOf(r Records) []Seat {
|
||||||
for _, node := range sortedCopy(r.Nodes) {
|
for _, node := range sortedCopy(r.Nodes) {
|
||||||
for _, d := range r.Assigned[node] {
|
for _, d := range r.Assigned[node] {
|
||||||
|
|||||||
@@ -0,0 +1,230 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats-server/v2/server"
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"golang.org/x/crypto/bcrypt"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The view against a real server, over WebSocket (novox/hq research 036): the composed user list is
|
||||||
|
// what the server reads, the listener is the shape the bus module declares (no TLS, compression on,
|
||||||
|
// reached across the overlay only), and the view with its credential binds the issue tracker's bucket,
|
||||||
|
// lists it with one batch read, follows a tracker event to re-read the key it names — and is refused
|
||||||
|
// every write: a put, a delete, an event, and a consumer delivering onto the tracker's event subject.
|
||||||
|
//
|
||||||
|
// go test ./internal/broker/ -run TestTheView
|
||||||
|
func TestTheViewReadsTheIssuesOverWebSocketAndWritesNothing(t *testing.T) {
|
||||||
|
hash := func(password string) string {
|
||||||
|
h, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.MinCost)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return string(h)
|
||||||
|
}
|
||||||
|
const node = "anchor"
|
||||||
|
tracker := Principal{Kind: KindModule, Node: node, Module: "mesh-issues", State: []string{"issues"},
|
||||||
|
Emits: []string{"opened", "moved"}, PasswordHash: hash("tracker")}
|
||||||
|
accounts, err := ComposeAccounts([]Principal{
|
||||||
|
{Kind: KindController, PasswordHash: hash("controller")},
|
||||||
|
tracker,
|
||||||
|
{Kind: KindView, PasswordHash: hash("view")},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
conf := filepath.Join(t.TempDir(), "accounts.conf")
|
||||||
|
if err := os.WriteFile(conf, []byte(accounts), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// The server reads the composed file as the bus does — through its own parser — and listens as the
|
||||||
|
// bus module's configuration says: a WebSocket listener without TLS and with compression, beside
|
||||||
|
// the client port. Ports chosen by the system, so this runs beside a live bus.
|
||||||
|
opts, err := server.ProcessConfigFile(conf)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("the server refused the composed user list: %v", err)
|
||||||
|
}
|
||||||
|
opts.Host, opts.Port = "127.0.0.1", server.RANDOM_PORT
|
||||||
|
opts.JetStream, opts.StoreDir = true, t.TempDir()
|
||||||
|
opts.NoLog, opts.NoSigs = true, true
|
||||||
|
opts.Websocket = server.WebsocketOpts{Host: "127.0.0.1", Port: server.RANDOM_PORT, NoTLS: true, Compression: true}
|
||||||
|
s, err := server.NewServer(opts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
go s.Start()
|
||||||
|
if !s.ReadyForConnections(30 * time.Second) {
|
||||||
|
s.Shutdown()
|
||||||
|
t.Fatal("the server did not come up")
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { s.Shutdown(); s.WaitForShutdown() })
|
||||||
|
|
||||||
|
dial := func(url, user, password string, refused chan<- string) *nats.Conn {
|
||||||
|
t.Helper()
|
||||||
|
nc, err := nats.Connect(url, nats.UserInfo(user, password), nats.CustomInboxPrefix("_INBOX."+user),
|
||||||
|
nats.Compression(true), nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {
|
||||||
|
if refused != nil && errors.Is(err, nats.ErrPermissionViolation) {
|
||||||
|
refused <- err.Error()
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s could not connect to %s: %v", user, url, err)
|
||||||
|
}
|
||||||
|
t.Cleanup(nc.Close)
|
||||||
|
return nc
|
||||||
|
}
|
||||||
|
|
||||||
|
// The controller defines the bucket, as it does for every module's state; the tracker writes it.
|
||||||
|
controller := dial(s.ClientURL(), "controller", "controller", nil)
|
||||||
|
cjs, _ := controller.JetStream()
|
||||||
|
if _, err := cjs.CreateKeyValue(&nats.KeyValueConfig{Bucket: ViewBucket, History: 8}); err != nil {
|
||||||
|
t.Fatalf("the controller could not define %s: %v", ViewBucket, err)
|
||||||
|
}
|
||||||
|
trackerConn := dial(s.ClientURL(), tracker.Username(), "tracker", nil)
|
||||||
|
tjs, _ := trackerConn.JetStream()
|
||||||
|
tkv, err := tjs.KeyValue(ViewBucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := tkv.Put("365", []byte(`{"number":365,"status":"open"}`)); err != nil {
|
||||||
|
t.Fatalf("the tracker could not write its own bucket: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The view, over WebSocket with its credential.
|
||||||
|
refused := make(chan string, 8)
|
||||||
|
view := dial(s.WebsocketURL(), ViewUser, "view", refused)
|
||||||
|
if !strings.HasPrefix(view.ConnectedUrl(), "ws://") {
|
||||||
|
t.Fatalf("the view is connected to %s, not over WebSocket", view.ConnectedUrl())
|
||||||
|
}
|
||||||
|
vjs, _ := view.JetStream(nats.MaxWait(3 * time.Second))
|
||||||
|
vkv, err := vjs.KeyValue(ViewBucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("the view could not bind %s: %v", ViewBucket, err)
|
||||||
|
}
|
||||||
|
if got, err := vkv.Get("365"); err != nil {
|
||||||
|
t.Fatalf("the view could not read a key: %v", err)
|
||||||
|
} else if !strings.Contains(string(got.Value()), `"number":365`) {
|
||||||
|
t.Fatalf("the view read %q", got.Value())
|
||||||
|
}
|
||||||
|
if _, err := tkv.Put("366", []byte(`{"number":366,"status":"open"}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The list: one batch read, the newest value of every key, into the view's own inbox, ended by the
|
||||||
|
// server's end-of-batch status (204).
|
||||||
|
listed := map[string]string{}
|
||||||
|
inbox := view.NewRespInbox()
|
||||||
|
batch, err := view.SubscribeSync(inbox)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := view.PublishRequest("$JS.API.DIRECT.GET.KV_"+ViewBucket, inbox,
|
||||||
|
[]byte(`{"multi_last":["$KV.`+ViewBucket+`.>"]}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
m, err := batch.NextMsg(5 * time.Second)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("the view's batch read ended without its end-of-batch (%d keys so far): %v", len(listed), err)
|
||||||
|
}
|
||||||
|
if m.Header.Get("Status") == "204" {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if status := m.Header.Get("Status"); status != "" {
|
||||||
|
t.Fatalf("the batch read answered %s %s", status, m.Header.Get("Description"))
|
||||||
|
}
|
||||||
|
listed[m.Header.Get("Nats-Subject")] = string(m.Data)
|
||||||
|
}
|
||||||
|
_ = batch.Unsubscribe()
|
||||||
|
for _, key := range []string{"365", "366"} {
|
||||||
|
if !strings.Contains(listed["$KV."+ViewBucket+"."+key], `"number":`+key) {
|
||||||
|
t.Errorf("the batch read did not list %s: %v", key, listed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A change followed: the tracker moves 365 and says so; the view hears the event and reads the key
|
||||||
|
// it names.
|
||||||
|
moved, err := view.SubscribeSync("mesh.mod.mesh-issues.event.moved")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = view.Flush()
|
||||||
|
if _, err := tkv.Put("365", []byte(`{"number":365,"status":"located"}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := trackerConn.Publish("mesh.mod.mesh-issues.event.moved", []byte(`{"number":365,"to":"located"}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := moved.NextMsg(5 * time.Second); err != nil {
|
||||||
|
t.Fatalf("the view did not hear the tracker's event: %v", err)
|
||||||
|
}
|
||||||
|
if got, err := vkv.Get("365"); err != nil || !strings.Contains(string(got.Value()), "located") {
|
||||||
|
t.Fatalf("after the event the view read %v (%v)", got, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The hole a watch would open, shut** (ViewReads): a consumer delivering onto the tracker's event
|
||||||
|
// subject would have the server republish the bucket there, as the tracker. Refused, and nothing
|
||||||
|
// reaches a module listening on that subject.
|
||||||
|
listener, err := trackerConn.SubscribeSync("mesh.mod.mesh-issues.event.opened")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = trackerConn.Flush()
|
||||||
|
if _, err := vjs.AddConsumer("KV_"+ViewBucket, &nats.ConsumerConfig{Name: "w", DeliverSubject: "mesh.mod.mesh-issues.event.opened",
|
||||||
|
AckPolicy: nats.AckNonePolicy, FilterSubject: "$KV." + ViewBucket + ".>"}); err == nil {
|
||||||
|
t.Error("the view made a consumer")
|
||||||
|
}
|
||||||
|
if _, err := vkv.WatchAll(); err == nil {
|
||||||
|
t.Error("the view made a watch, which is a consumer")
|
||||||
|
}
|
||||||
|
if m, err := listener.NextMsg(2 * time.Second); err == nil {
|
||||||
|
t.Errorf("a message reached the tracker's event subject from the view: %q", m.Data)
|
||||||
|
}
|
||||||
|
// The refusals of the consumer create land as permission violations too; drained before the writes.
|
||||||
|
drain := time.After(500 * time.Millisecond)
|
||||||
|
for draining := true; draining; {
|
||||||
|
select {
|
||||||
|
case <-refused:
|
||||||
|
case <-drain:
|
||||||
|
draining = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// And every write is refused: the server says so, and the bucket is unchanged.
|
||||||
|
if _, err := vkv.Put("367", []byte(`{"number":367}`)); err == nil {
|
||||||
|
t.Error("the view put a key")
|
||||||
|
}
|
||||||
|
if err := vkv.Delete("365"); err == nil {
|
||||||
|
t.Error("the view deleted a key")
|
||||||
|
}
|
||||||
|
if err := view.Publish("mesh.mod.mesh-issues.event.opened", []byte(`{"number":367}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = view.Flush()
|
||||||
|
violations := map[string]bool{}
|
||||||
|
deadline := time.After(10 * time.Second)
|
||||||
|
for len(violations) < 3 {
|
||||||
|
select {
|
||||||
|
case v := <-refused:
|
||||||
|
for _, subject := range []string{"$KV." + ViewBucket + ".367", "$KV." + ViewBucket + ".365", "mesh.mod.mesh-issues.event.opened"} {
|
||||||
|
if strings.Contains(v, subject) {
|
||||||
|
violations[subject] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case <-deadline:
|
||||||
|
t.Fatalf("the server refused %d of the view's 3 writes as permission violations", len(violations))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := tkv.Get("367"); !errors.Is(err, nats.ErrKeyNotFound) {
|
||||||
|
t.Errorf("after the view's put, 367 is %v", err)
|
||||||
|
}
|
||||||
|
if _, err := tkv.Get("365"); err != nil {
|
||||||
|
t.Errorf("after the view's delete, 365 is gone: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,144 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
"sort"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The view (novox/hq research 036): one read-only user, composed like every other, whose whole
|
||||||
|
// authority is a list here — so a grant that is not on the list fails a test, not a review.
|
||||||
|
|
||||||
|
// Exactly what it hears, exactly what it asks, and nothing it could write or answer. A mutation that
|
||||||
|
// adds a publish grant — `$KV.<bucket>.>`, an event, a tool — fails here.
|
||||||
|
func TestTheViewHearsAndReadsAndCanPublishNothingElse(t *testing.T) {
|
||||||
|
perms, err := PermissionsFor(Principal{Kind: KindView})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
wantSub := append(append([]string(nil), ViewHears...), "_INBOX.view.>")
|
||||||
|
sort.Strings(wantSub)
|
||||||
|
if !reflect.DeepEqual(perms.Subscribe, wantSub) {
|
||||||
|
t.Errorf("the view subscribes\n %v\nand should subscribe exactly\n %v", perms.Subscribe, wantSub)
|
||||||
|
}
|
||||||
|
wantPub := ViewReads()
|
||||||
|
sort.Strings(wantPub)
|
||||||
|
if !reflect.DeepEqual(perms.Publish, wantPub) {
|
||||||
|
t.Errorf("the view publishes\n %v\nand should publish exactly\n %v", perms.Publish, wantPub)
|
||||||
|
}
|
||||||
|
if len(perms.PublishDeny) != 0 {
|
||||||
|
t.Errorf("the view needs no deny, because nothing it may publish reaches the controller's own: %v", perms.PublishDeny)
|
||||||
|
}
|
||||||
|
if perms.AllowResponses {
|
||||||
|
t.Error("the view may answer, and nothing is ever asked of it")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every publish grant is binding the one bucket's stream or reading it directly. **The mutation this
|
||||||
|
// holds against**: a write grant of any shape, and a consumer of any shape — a consumer's deliver
|
||||||
|
// subject is the creator's choice, so creating one is publishing anywhere (ViewReads).
|
||||||
|
stream := "KV_" + ViewBucket
|
||||||
|
for _, p := range perms.Publish {
|
||||||
|
readOnly := p == "$JS.API.STREAM.INFO."+stream ||
|
||||||
|
p == "$JS.API.DIRECT.GET."+stream ||
|
||||||
|
p == "$JS.API.DIRECT.GET."+stream+".>"
|
||||||
|
if !readOnly {
|
||||||
|
t.Errorf("the view is granted a publish on %q, which is not a read of %s", p, ViewBucket)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, refused := range []string{
|
||||||
|
"$KV." + ViewBucket + ".365", // a put or a delete
|
||||||
|
"$KV.mesh-controller_conditions.x", // another bucket
|
||||||
|
"$JS.API.STREAM.CREATE." + stream, // defining the stream
|
||||||
|
"$JS.API.STREAM.PURGE." + stream, // emptying it
|
||||||
|
"$JS.API.STREAM.DELETE." + stream, // deleting it
|
||||||
|
"$JS.API.STREAM.MSG.DELETE." + stream, // deleting a message
|
||||||
|
"$JS.API.CONSUMER.CREATE.KV_mesh-controller_conditions.x", // reading another bucket
|
||||||
|
"$JS.API.CONSUMER.CREATE." + stream + ".w.$KV." + ViewBucket + ".>", // a watch: delivers anywhere
|
||||||
|
"$JS.API.CONSUMER.CREATE." + stream, // an unnamed consumer
|
||||||
|
"$JS.API.CONSUMER.DELETE." + stream + ".a_mesh-issues", // a module's consumer
|
||||||
|
"$JS.FC." + stream + ".x",
|
||||||
|
"$JS.API.DIRECT.GET.KV_mesh-controller_conditions", // another bucket, directly
|
||||||
|
"$JS.API.STREAM.INFO.EVENTS", // the events stream
|
||||||
|
"$JS.API.INFO", // the account
|
||||||
|
"mesh.mod.mesh-issues.event.opened", // claiming the tracker said something
|
||||||
|
"mesh.mod.mesh-issues.tool.open", // opening an issue
|
||||||
|
"mesh.seat.issue-tracker.tool.open", // through the seat
|
||||||
|
"mesh.seat.issue-tracker.tool.open.novox", // on one machine
|
||||||
|
"mesh.seat.mesh-controller.tool.status", // the controller's verbs
|
||||||
|
"mesh.seat.mesh-controller.event.plan-moved",
|
||||||
|
"$SRV.PING",
|
||||||
|
"_INBOX.controller.x",
|
||||||
|
} {
|
||||||
|
if MayPublish(perms, refused) {
|
||||||
|
t.Errorf("the view may publish %q", refused)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, refused := range []string{
|
||||||
|
"mesh.mod.mesh-issues.tool.open", // a tool asked of the tracker
|
||||||
|
"mesh.mod.telegram.event.received", // another module's events
|
||||||
|
"mesh.seat.mesh-controller.event.applied",
|
||||||
|
"mesh.control.novox.report",
|
||||||
|
"_INBOX.controller.x",
|
||||||
|
"_INBOX.person.jochen.x",
|
||||||
|
"_DELIVER.controller.EVENTS",
|
||||||
|
} {
|
||||||
|
if MaySubscribe(perms, refused) {
|
||||||
|
t.Errorf("the view may subscribe %q", refused)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, heard := range []string{
|
||||||
|
"mesh.mod.mesh-issues.event.opened",
|
||||||
|
"mesh.mod.mesh-issues.event.moved",
|
||||||
|
"mesh.mod.mesh-issues.event.noted",
|
||||||
|
"mesh.mod.mesh-issues.event.linked",
|
||||||
|
"mesh.seat.mesh-controller.event.plan-moved",
|
||||||
|
"mesh.seat.mesh-controller.event.condition-raised",
|
||||||
|
"mesh.seat.mesh-controller.event.condition-changed",
|
||||||
|
"mesh.seat.mesh-controller.event.condition-cleared",
|
||||||
|
"mesh.mod.mesh-delivery.event.transition",
|
||||||
|
"mesh.mod.mesh-delivery.event.group",
|
||||||
|
"_INBOX.view.abc",
|
||||||
|
} {
|
||||||
|
if !MaySubscribe(perms, heard) {
|
||||||
|
t.Errorf("the view cannot subscribe %q", heard)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Composed once the mesh minted its credential, and not before: its row is the whole record of it.
|
||||||
|
func TestTheViewIsComposedOnlyOnceItsCredentialIsMinted(t *testing.T) {
|
||||||
|
without, err := Users(Records{Nodes: []string{"anchor"}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, p := range without {
|
||||||
|
if p.Kind == KindView {
|
||||||
|
t.Fatal("the view is composed before its credential was minted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
with, err := Users(Records{Nodes: []string{"anchor"}, View: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
views := 0
|
||||||
|
for _, p := range with {
|
||||||
|
if p.Kind == KindView {
|
||||||
|
views++
|
||||||
|
if p.Username() != ViewUser {
|
||||||
|
t.Errorf("the view is called %q, and its row is %q", p.Username(), ViewUser)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if views != 1 {
|
||||||
|
t.Fatalf("%d view users composed; there is one view", views)
|
||||||
|
}
|
||||||
|
// And without its hash it is named as missing, like any user — never written as a user anybody is.
|
||||||
|
_, missing := WithPasswords(with, map[string]string{})
|
||||||
|
found := false
|
||||||
|
for _, m := range missing {
|
||||||
|
found = found || m == ViewUser
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Error("a view with no password was not named as missing one")
|
||||||
|
}
|
||||||
|
}
|
||||||
+5
-106
@@ -7,10 +7,8 @@ import (
|
|||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"io/fs"
|
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path"
|
"path"
|
||||||
@@ -161,18 +159,17 @@ func build(ctx context.Context, run Runner, publish Publisher,
|
|||||||
}
|
}
|
||||||
// The credential is a file git reads, never an argument: a URL carrying a password in argv
|
// The credential is a file git reads, never an argument: a URL carrying a password in argv
|
||||||
// would be readable by anything that can list processes for as long as a clone runs.
|
// would be readable by anything that can list processes for as long as a clone runs.
|
||||||
credentials, forget, err := storeCredential(workspace, forge)
|
credentials := ""
|
||||||
if err != nil {
|
if forge.URL != "" {
|
||||||
|
credentials = filepath.Join(workspace, "git-credentials")
|
||||||
|
if err := os.WriteFile(credentials, []byte(forge.URL+"\n"), 0o600); err != nil {
|
||||||
return Result{}, err
|
return Result{}, err
|
||||||
}
|
}
|
||||||
defer forget()
|
}
|
||||||
tree := filepath.Join(workspace, "source")
|
tree := filepath.Join(workspace, "source")
|
||||||
if err := os.RemoveAll(tree); err != nil {
|
if err := os.RemoveAll(tree); err != nil {
|
||||||
return Result{}, err
|
return Result{}, err
|
||||||
}
|
}
|
||||||
// Removed when the build ends, whatever happens: the tree holds the .npmrc a build may be handed, in the
|
|
||||||
// workspace a later check's container mounts as its HOME (novox/hq issue 462).
|
|
||||||
defer os.RemoveAll(tree)
|
|
||||||
// A fresh clone every time rather than a fetch into a tree that is already there. A build
|
// A fresh clone every time rather than a fetch into a tree that is already there. A build
|
||||||
// that reuses a working tree can succeed because of something a previous build left behind,
|
// that reuses a working tree can succeed because of something a previous build left behind,
|
||||||
// and that is a build nobody can reproduce.
|
// and that is a build nobody can reproduce.
|
||||||
@@ -180,9 +177,6 @@ func build(ctx context.Context, run Runner, publish Publisher,
|
|||||||
say("clone", "FAILED: %v", err)
|
say("clone", "FAILED: %v", err)
|
||||||
return Result{}, fmt.Errorf("cannot clone %s: %w", repository, err)
|
return Result{}, fmt.Errorf("cannot clone %s: %w", repository, err)
|
||||||
}
|
}
|
||||||
if err := recordedWithoutUserinfo(ctx, run, tree, repository); err != nil {
|
|
||||||
return Result{}, err
|
|
||||||
}
|
|
||||||
say("clone", "done")
|
say("clone", "done")
|
||||||
if ref != "" {
|
if ref != "" {
|
||||||
if _, err := run(ctx, tree, "git", "checkout", "--quiet", ref); err != nil {
|
if _, err := run(ctx, tree, "git", "checkout", "--quiet", ref); err != nil {
|
||||||
@@ -254,8 +248,6 @@ func build(ctx context.Context, run Runner, publish Publisher,
|
|||||||
if err := os.WriteFile(npmrcPath, []byte(content), 0o600); err != nil {
|
if err := os.WriteFile(npmrcPath, []byte(content), 0o600); err != nil {
|
||||||
return Result{}, fmt.Errorf("cannot write the package-registry credential for the build: %w", err)
|
return Result{}, fmt.Errorf("cannot write the package-registry credential for the build: %w", err)
|
||||||
}
|
}
|
||||||
// Only for as long as the build: a later check's container mounts this workspace (novox/hq issue 462).
|
|
||||||
defer os.Remove(npmrcPath)
|
|
||||||
say("packages", "resolving %s from the mesh's package registry", npmrc.Scope)
|
say("packages", "resolving %s from the mesh's package registry", npmrc.Scope)
|
||||||
src.notPinned("it resolves packages from the mesh's registry at build time")
|
src.notPinned("it resolves packages from the mesh's registry at build time")
|
||||||
}
|
}
|
||||||
@@ -449,9 +441,6 @@ func contextFrom(ctx context.Context, run Runner, workspace, artifact, credentia
|
|||||||
if _, err := run(ctx, workspace, "git", cloneWith(credentials, "clone", "--quiet", url, dir)...); err != nil {
|
if _, err := run(ctx, workspace, "git", cloneWith(credentials, "clone", "--quiet", url, dir)...); err != nil {
|
||||||
return "", fmt.Errorf("cannot clone %s: %w", url, err)
|
return "", fmt.Errorf("cannot clone %s: %w", url, err)
|
||||||
}
|
}
|
||||||
if err := recordedWithoutUserinfo(ctx, run, dir, url); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if from.Ref != "" {
|
if from.Ref != "" {
|
||||||
if _, err := run(ctx, dir, "git", "checkout", "--quiet", from.Ref); err != nil {
|
if _, err := run(ctx, dir, "git", "checkout", "--quiet", from.Ref); err != nil {
|
||||||
return "", fmt.Errorf("%s has no %s: %w", from.Repository, from.Ref, err)
|
return "", fmt.Errorf("%s has no %s: %w", from.Repository, from.Ref, err)
|
||||||
@@ -478,96 +467,6 @@ func contextURL(from catalogue.ArtifactContext, seats map[string]string) (string
|
|||||||
return strings.TrimRight(base, "/") + "/" + strings.TrimSuffix(strings.Trim(from.Repository, "/"), ".git") + ".git", nil
|
return strings.TrimRight(base, "/") + "/" + strings.TrimSuffix(strings.Trim(from.Repository, "/"), ".git") + ".git", nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// storeCredential writes the forge credential where git's credential store reads it, and the function that
|
|
||||||
// removes it again; "" and nothing to remove when the builder holds none.
|
|
||||||
//
|
|
||||||
// **Never inside the workspace** (novox/hq issue 462): a merge check's toolchain container mounts the
|
|
||||||
// workspace as its HOME, so a credential kept there — even one a build left behind — is readable by any pull
|
|
||||||
// request's merge-check.sh, and what it prints is kept on the bus. So it lives in a directory of its own
|
|
||||||
// outside the workspace, made private, and a credential an older builder left in the workspace is removed.
|
|
||||||
func storeCredential(workspace string, forge GitCredential) (string, func(), error) {
|
|
||||||
if err := os.Remove(filepath.Join(workspace, "git-credentials")); err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
||||||
return "", nil, fmt.Errorf("a credential left in the workspace cannot be removed: %w", err)
|
|
||||||
}
|
|
||||||
if forge.URL == "" {
|
|
||||||
return "", func() {}, nil
|
|
||||||
}
|
|
||||||
dir, err := os.MkdirTemp("", "mesh-forge-credential-")
|
|
||||||
if err != nil {
|
|
||||||
return "", nil, err
|
|
||||||
}
|
|
||||||
forget := func() { os.RemoveAll(dir) }
|
|
||||||
if withinDir(workspace, dir) {
|
|
||||||
forget()
|
|
||||||
return "", nil, fmt.Errorf("the temporary directory %s is inside the workspace %s, which a check's "+
|
|
||||||
"container mounts: the forge credential is not written where it could read it", dir, workspace)
|
|
||||||
}
|
|
||||||
path := filepath.Join(dir, "git-credentials")
|
|
||||||
if err := os.WriteFile(path, []byte(forge.URL+"\n"), 0o600); err != nil {
|
|
||||||
forget()
|
|
||||||
return "", nil, err
|
|
||||||
}
|
|
||||||
return path, forget, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// recordedWithoutUserinfo makes a clone record the URL it came from without userinfo: git keeps it as given in
|
|
||||||
// the clone's .git/config, inside the workspace a check's container mounts (novox/hq issue 462).
|
|
||||||
func recordedWithoutUserinfo(ctx context.Context, run Runner, clone, repository string) error {
|
|
||||||
bare, carried := withoutUserinfo(repository)
|
|
||||||
if !carried {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if _, err := run(ctx, clone, "git", "remote", "set-url", "origin", bare); err != nil {
|
|
||||||
return fmt.Errorf("the clone of %s keeps its credential: %w", bare, err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// forgetLeftCredentials removes what an earlier build or an older builder left in the workspace that holds
|
|
||||||
// a credential: the forge's git-credentials, and every .npmrc a build wrote into a tree it cloned. Run
|
|
||||||
// before a check, whose container mounts the workspace as its HOME (novox/hq issue 462). The Go caches are
|
|
||||||
// not walked: no build writes a credential there, and they hold more files than everything else.
|
|
||||||
func forgetLeftCredentials(workspace string) error {
|
|
||||||
if err := os.Remove(filepath.Join(workspace, "git-credentials")); err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return filepath.WalkDir(workspace, func(p string, d fs.DirEntry, err error) error {
|
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, os.ErrNotExist) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if d.IsDir() && p != workspace && (d.Name() == "go-cache" || d.Name() == "go-modules") &&
|
|
||||||
filepath.Dir(p) == workspace {
|
|
||||||
return filepath.SkipDir
|
|
||||||
}
|
|
||||||
if !d.IsDir() && d.Name() == ".npmrc" {
|
|
||||||
if err := os.Remove(p); err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
||||||
return fmt.Errorf("a package-registry credential left at %s cannot be removed: %w", p, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// withinDir is whether path is dir or under it, both made absolute and resolved.
|
|
||||||
func withinDir(dir, path string) bool {
|
|
||||||
d, err1 := filepath.Abs(dir)
|
|
||||||
p, err2 := filepath.Abs(path)
|
|
||||||
if err1 != nil || err2 != nil {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
if r, err := filepath.EvalSymlinks(d); err == nil {
|
|
||||||
d = r
|
|
||||||
}
|
|
||||||
if r, err := filepath.EvalSymlinks(p); err == nil {
|
|
||||||
p = r
|
|
||||||
}
|
|
||||||
rel, err := filepath.Rel(d, p)
|
|
||||||
return err != nil || rel == "." || filepath.IsLocal(rel)
|
|
||||||
}
|
|
||||||
|
|
||||||
// cloneWith is a git invocation that may offer a stored credential.
|
// cloneWith is a git invocation that may offer a stored credential.
|
||||||
//
|
//
|
||||||
// The first `-c credential.helper=` clears every helper the environment might carry, so exactly
|
// The first `-c credential.helper=` clears every helper the environment might carry, so exactly
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user