Merge remote-tracking branch 'origin/main' into adr-0297-module-logs
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery delivered

This commit is contained in:
jochen
2026-10-12 02:29:00 +02:00
75 changed files with 5286 additions and 165 deletions
+3
View File
@@ -3,3 +3,6 @@
# A built binary. The lab writes one here when pointed at the repository root by mistake; # A built binary. The lab writes one here when pointed at the repository root by mistake;
# built artifacts belong in build/, which is already ignored. # built artifacts belong in build/, which is already ignored.
/mesh-builder /mesh-builder
/mesh-controller
/postgres-provisioner
/redis-provisioner
+3
View File
@@ -408,6 +408,9 @@ func comparisonOf(module string, c comparison, opts takeOptions) (preview string
switch { switch {
case sec.Origin == inventory.OriginAccepted: case sec.Origin == inventory.OriginAccepted:
fmt.Fprintf(&b, " %s %s: accepted from a person, carried in as it is\n", what, name) fmt.Fprintf(&b, " %s %s: accepted from a person, carried in as it is\n", what, name)
case sec.Origin == inventory.OriginIssued:
// Taken from the provider's software (novox/hq ADR 0305): the value the running service uses.
fmt.Fprintf(&b, " %s %s: issued by the provider's software, carried in as it is\n", what, name)
case opts.Mint[sec.Name]: case opts.Mint[sec.Name]:
fmt.Fprintf(&b, " %s %s: minted by the mesh; the service takes the new value, as --mint said\n", what, name) fmt.Fprintf(&b, " %s %s: minted by the mesh; the service takes the new value, as --mint said\n", what, name)
case foundData: case foundData:
+53 -2
View File
@@ -367,7 +367,13 @@ func keepBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan,
} }
closes, latest := windowOf(merges, b.Created, window, atMost) closes, latest := windowOf(merges, b.Created, window, atMost)
b.Delivery.Merges = named b.Delivery.Merges = named
b.Delivery.Batch = &inventory.PlanBatch{ClosesAt: closes, AtMost: latest, Behind: behind, Own: b.OwnPath()} var closed bool
var withheld []string
if was := b.Delivery.Batch; was != nil {
closed, withheld = was.Closed, was.Withheld // a stop's, which no look takes back (novox/hq ADR 0299)
}
b.Delivery.Batch = &inventory.PlanBatch{ClosesAt: closes, AtMost: latest, Behind: behind, Own: b.OwnPath(),
Closed: closed, Withheld: withheld}
b.State = inventory.PlanAssembling b.State = inventory.PlanAssembling
if behind != "" && windowClosed(*b, now) { if behind != "" && windowClosed(*b, now) {
b.State = inventory.PlanQueued b.State = inventory.PlanQueued
@@ -402,7 +408,7 @@ func windowClosed(b inventory.Plan, now time.Time) bool {
return true return true
} }
w := b.Delivery.Batch w := b.Delivery.Batch
return !now.Before(w.ClosesAt) || !now.Before(w.AtMost) return w.Closed || !now.Before(w.ClosesAt) || !now.Before(w.AtMost)
} }
// carriedOf is a set of merges as a walk carries them: the latest merge of each repository's branch, and // carriedOf is a set of merges as a walk carries them: the latest merge of each repository's branch, and
@@ -762,7 +768,18 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
} }
sort.Strings(also) sort.Strings(also)
ordered := groupOrderEdges(members, entries, read, edges) ordered := groupOrderEdges(members, entries, read, edges)
var withheld []string
plan := planOfMoves(moved, append(append([]inventory.Edge{}, edges...), ordered...)) plan := planOfMoves(moved, append(append([]inventory.Edge{}, edges...), ordered...))
// **Withheld, a stopped merge's modules** (novox/hq ADR 0299): a module is built at its branch, which holds the
// merge a stop left out of the walk these merges were walked again from — built here only as a dependent, it
// would deliver that merge. One a merge of this batch moves is built: that merge carries the stopped change,
// and the stop named it.
if batch.Delivery != nil && batch.Delivery.Batch != nil {
if out := withhold(&plan, batch.Delivery.Batch.Withheld, moved); len(out) > 0 {
withheld = out
fmt.Printf(" withheld, a stopped merge's modules built only as dependents: %s\n", strings.Join(out, ", "))
}
}
plan.ID, plan.Revision, plan.Created, plan.Epoch = batch.ID, batch.Revision, now, batch.Epoch plan.ID, plan.Revision, plan.Created, plan.Epoch = batch.ID, batch.Revision, now, batch.Epoch
plan.Commits = commits plan.Commits = commits
newest := newestCommit(commits) newest := newestCommit(commits)
@@ -773,6 +790,7 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
} }
plan.Delivery.Merges = named plan.Delivery.Merges = named
plan.Delivery.Alone = batch.Delivery != nil && batch.Delivery.Alone plan.Delivery.Alone = batch.Delivery != nil && batch.Delivery.Alone
plan.Delivery.Withheld = withheld
// **Its moments and its class** (novox/hq ADR 0282 decision 6): measured, never acted on. // **Its moments and its class** (novox/hq ADR 0282 decision 6): measured, never acted on.
plan.Times = walkTimesAtCut(*batch, plan, entries, now) plan.Times = walkTimesAtCut(*batch, plan, entries, now)
if len(moved) == 0 { if len(moved) == 0 {
@@ -823,6 +841,39 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
return nil return nil
} }
// withhold takes from a plan every module named that its merges do not move, and the tiers left empty: the
// modules taken.
func withhold(plan *inventory.Plan, names, moved []string) []string {
var out []string
for _, n := range names {
if _, in := plan.Modules[n]; in && !slices.Contains(moved, n) {
delete(plan.Modules, n)
out = append(out, n)
}
}
if len(out) == 0 {
return nil
}
var tiers [][]string
for _, t := range plan.Tiers {
var kept []string
for _, n := range t {
if !slices.Contains(out, n) {
kept = append(kept, n)
}
}
if len(kept) > 0 {
tiers = append(tiers, kept)
}
}
if tiers == nil {
tiers = [][]string{}
}
plan.Tiers = tiers
sort.Strings(out)
return out
}
// walkTimesAtCut is a walk's own moments as it is cut (novox/hq ADR 0282 decision 6): when its batch's window // walkTimesAtCut is a walk's own moments as it is cut (novox/hq ADR 0282 decision 6): when its batch's window
// closed — no merge for the window's length, or its maximum, whichever came first — when it was cut, and its // closed — no merge for the window's length, or its maximum, whichever came first — when it was cut, and its
// class. A merge walked alone had no window. // class. A merge walked alone had no window.
+7
View File
@@ -835,6 +835,9 @@ type answers struct {
// public name on the machine went dark. The holds were correct; they were recorded only in the // public name on the machine went dark. The holds were correct; they were recorded only in the
// machine's own state file, and the one visible symptom was a count that did not add up. // machine's own state file, and the one visible symptom was a count that did not add up.
untaken map[string]map[string]int untaken map[string]map[string]int
// leftOut is, per machine, every module of its set its composition leaves out, and why (novox/hq issue
// 380): assigned and not applied, which every push said only in passing. Not well while there is any.
leftOut map[string][]leftOutModule
// unheld is every module on a machine whose resources are applied through a seat nothing on // unheld is every module on a machine whose resources are applied through a seat nothing on
// that machine holds (novox/hq ADR 0207), with the modules that could hold it. Reported, not // that machine holds (novox/hq ADR 0207), with the modules that could hold it. Reported, not
// refused, until the switch — and while there is any, the mesh is not all well: the order the // refused, until the switch — and while there is any, the mesh is not all well: the order the
@@ -863,6 +866,10 @@ type answers struct {
// applied, is not well: an assignment somebody made is not made yet, or will not be. // applied, is not well: an assignment somebody made is not made yet, or will not be.
pending []inventory.PendingAssignment pending []inventory.PendingAssignment
pendingUnread string pendingUnread string
// lastSent is every machine's last recorded send, by name, with what the machine answered of it (novox/hq
// issue 485); lastSentUnread why it could not be read.
lastSent map[string]inventory.Send
lastSentUnread string
} }
// heldBy is every artifact this mesh has built, for a build that may need one as its base. // heldBy is every artifact this mesh has built, for a build that may need one as its base.
+24
View File
@@ -2,6 +2,7 @@ package main
import ( import (
"fmt" "fmt"
"slices"
"sort" "sort"
"strings" "strings"
@@ -28,6 +29,14 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
for _, e := range entries { for _, e := range entries {
byName[e.Manifest.Module] = e byName[e.Manifest.Module] = e
} }
// The machine holding the bus, whose declaration carries every module's grants (novox/hq issue 490).
holder := ""
for _, e := range entries {
if e.Manifest.BusUsers != "" && e.Manifest.ClaimsSeat(catalogue.BrokerSeat) && len(e.On) > 0 {
holder = e.On[0]
}
}
var granting []string
machines := map[string]*link.MachinePlan{} machines := map[string]*link.MachinePlan{}
machine := func(name string) *link.MachinePlan { machine := func(name string) *link.MachinePlan {
if machines[name] == nil { if machines[name] == nil {
@@ -50,6 +59,10 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
machine(on).Receives = append(machine(on).Receives, name) machine(on).Receives = append(machine(on).Receives, name)
} }
} }
if !waits && holder != "" && !(len(e.On) == 1 && e.On[0] == holder) && len(e.On) > 0 &&
!slices.Contains(granting, name) {
granting = append(granting, name)
}
switch { switch {
case (name == "nats" || catalogue.ProvidesBus(e.Manifest)) && len(e.On) > 0: case (name == "nats" || catalogue.ProvidesBus(e.Manifest)) && len(e.On) > 0:
// Said before the merge (novox/hq issue 336): a new bus build holds every send to the bus's machine // Said before the merge (novox/hq issue 336): a new bus build holds every send to the bus's machine
@@ -76,6 +89,13 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
} }
} }
} }
if len(granting) > 0 {
// Said before the merge (novox/hq issue 490): what the walk does when the change alters the bus's
// user list. Whether it does is known only once the builds are registered, so it is said as a rule.
p.Steps = append(p.Steps, fmt.Sprintf("%s: if this changes what the bus's user list says (a new tool, "+
"subject or user), %s is sent that list alone, every build there kept, before %s is judged on its first "+
"machine", grantsStepWord, holder, strings.Join(granting, ", ")))
}
var names []string var names []string
for name := range machines { for name := range machines {
names = append(names, name) names = append(names, name)
@@ -88,6 +108,10 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
return p return p
} }
// grantsStepWord names the grants step (novox/hq issue 490): the bus's user list sent alone to the machine
// holding the bus ahead of a walk's gate. Not the bus step, which replaces the bus itself.
const grantsStepWord = "grants step"
// busUpgradeNeeded is how a change plan says that merging it holds the bus's machine for a person's step. // busUpgradeNeeded is how a change plan says that merging it holds the bus's machine for a person's step.
const busUpgradeNeeded = "merging this needs a person's bus upgrade" const busUpgradeNeeded = "merging this needs a person's bus upgrade"
+404 -13
View File
@@ -117,28 +117,411 @@ func letGo(ctx context.Context, inv *inventory.Inventory, id, by, why string) (i
// stopWalk ends a walk on its delivery's word: failed, said as stopped by whom, why. What it asked still // stopWalk ends a walk on its delivery's word: failed, said as stopped by whom, why. What it asked still
// builds and registers; nothing further is asked or sent. // builds and registers; nothing further is asked or sent.
func stopWalk(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, error) { func stopWalk(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, error) {
p, _, err := stopWalkWithout(ctx, inv, id, by, why, nil, time.Now().UTC())
return p, err
}
// stopAnswer is what a stop that leaves merges out says, as data (novox/hq ADR 0299): the record stopped, the
// batch that walks its other merges again with those merges, every merge left out, the modules the stopped walk
// already sent that stay on the machines named, the modules the batch's walk withholds, and every merge elsewhere
// that will deliver a stopped change when it is walked — a later merge of its repository, or one moving its modules.
type stopAnswer struct {
Stopped string `json:"stopped"`
WalkedAgainBy string `json:"walked_again_by,omitempty"`
WalkedAgain []string `json:"walked_again,omitempty"`
LeftOut []inventory.PlanLeftOut `json:"left_out,omitempty"`
Kept []inventory.PlanKept `json:"kept,omitempty"`
Withheld []string `json:"withheld,omitempty"`
StillCarriedBy []string `json:"still_carried_by,omitempty"`
}
// stopWalkWithout ends a walk on its delivery's word and, given merges to leave out, walks every other merge it
// answered again without them (novox/hq issue 459, ADR 0299): one walk carries every merge of its batch, so ending
// it for one group member ended every other delivery on it.
//
// - Left out are the merges named, each later merge of their repository's branch, which contains one, and each
// merge moving a module a left-out merge moves: a module is built at its branch, which holds the stopped merge.
// - The others join the open batch of their kind — merges heard while the walk ran wait there, and an older
// commit never walks after a newer one — or a new batch when none is open; its window is closed, and its walk
// withholds the left-out merges' modules where it would build them only as dependents.
// - Given a batch not yet cut, the merges left out leave it for a stopped record of their own, and the batch
// goes on without them; with nothing left in it, the batch itself is stopped.
// - The stopped record, the batch and every merge moved are written in one transaction (decision 7).
//
// With nothing left to walk, or no merge named, the walk is only ended. A merge named that the record does not
// answer refuses the stop whole.
func stopWalkWithout(ctx context.Context, inv *inventory.Inventory, id, by, why string, without []string,
now time.Time) (inventory.Plan, stopAnswer, error) {
said := stopAnswer{Stopped: id}
release, err := inv.HoldPlans(ctx, true) release, err := inv.HoldPlans(ctx, true)
if err != nil { if err != nil {
return inventory.Plan{}, err return inventory.Plan{}, said, err
} }
defer release() defer release()
p, err := inv.PlanByID(ctx, id) p, err := inv.PlanByID(ctx, id)
if err != nil {
return inventory.Plan{}, said, err
}
batch := p.Batch()
if !p.Open() && !(batch && len(without) > 0) {
return p, said, fmt.Errorf("%s is already %s", p.ID, p.State)
}
var merges, again []inventory.BatchedMerge
if len(without) > 0 {
if merges, err = inv.MergesOf(ctx, p.ID); err != nil {
return p, said, err
}
if again, said.LeftOut, err = leaveOut(merges, without); err != nil {
return p, said, fmt.Errorf("%s is not stopped: %w", p.ID, err)
}
}
stopped := func(r *inventory.Plan) {
if r.Delivery == nil {
r.Delivery = &inventory.PlanDelivery{}
}
r.Delivery.Stopped, r.Delivery.StoppedWhy, r.Delivery.LeftOut = by, why, said.LeftOut
r.State = inventory.PlanFailed
r.Note = fmt.Sprintf("stopped by %s at tier %d: %s", by, r.Tier, why)
}
withheld := movesOf(leftOutMerges(merges, said.LeftOut))
said.Withheld = withheld
if said.StillCarriedBy, err = stillCarried(ctx, inv, p.ID, merges, said.LeftOut, withheld); err != nil {
return p, said, err
}
if batch {
return p, said, stopInBatch(ctx, inv, &p, merges, again, &said, stopped, now)
}
entries, err := inv.Catalogued(ctx)
if err != nil {
return p, said, err
}
running := map[string][]string{}
for _, e := range entries {
running[e.Manifest.Module] = e.On
}
said.Kept = keptOf(p, movesOf(again), running)
stopped(&p)
p.Delivery.Kept = said.Kept
if len(again) == 0 {
if len(said.LeftOut) > 0 {
p.Note += fmt.Sprintf("; left out %s, nothing else to walk", mergesLeftOutWords(said.LeftOut))
}
return p, said, inv.SavePlan(ctx, &p)
}
target, err := batchToWalkAgain(ctx, inv, again, now)
if err != nil {
return p, said, fmt.Errorf("%s is not stopped: %w", p.ID, err)
}
target.Delivery.Batch.Closed = true
target.Delivery.Batch.Withheld = union(target.Delivery.Batch.Withheld, withheld)
sort.Strings(target.Delivery.Batch.Withheld)
said.WalkedAgainBy = target.ID
var moves []inventory.MergeMove
for _, m := range again {
said.WalkedAgain = append(said.WalkedAgain, m.Repository+"@"+m.Commit)
moves = append(moves, inventory.MergeMove{Repository: m.Repository, Commit: m.Commit, Plan: target.ID, Alone: m.Alone})
}
p.Delivery.WalkedAgainBy = target.ID
p.Note += fmt.Sprintf("; its other merges are walked again by %s: %s", target.ID, strings.Join(said.WalkedAgain, ", "))
if len(said.LeftOut) > 0 {
p.Note += "; left out " + mergesLeftOutWords(said.LeftOut)
}
if len(said.Kept) > 0 {
p.Note += "; " + keptWords(said.Kept)
}
// The batch first, so no merge names a plan the store does not hold; all of it, or nothing (decision 7).
if err := inv.SavePlansMoving(ctx, []*inventory.Plan{&target, &p}, moves); err != nil {
return p, said, fmt.Errorf("%s is not stopped, and none of its merges moved: %w", p.ID, err)
}
// What the batch holds and names now: written again by the cutter's next look should this fail.
if err := keepBatch(ctx, inv, &target, now, ""); err != nil {
fmt.Printf(" %s holds its merges; its record is written again at the next look: %v\n", target.ID, err)
}
fmt.Printf("%s stopped by %s; %s walks its other merges again: %s\n", p.ID, by, target.ID,
strings.Join(said.WalkedAgain, ", "))
return p, said, nil
}
// stopInBatch leaves merges out of a batch not yet cut (ADR 0299): they go to a stopped record of their own, which
// names the batch as walking the others, and the batch withholds their modules; a batch left with none is stopped
// itself.
func stopInBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan, merges, again []inventory.BatchedMerge,
said *stopAnswer, stopped func(*inventory.Plan), now time.Time) error {
if len(again) == 0 {
stopped(b)
b.Delivery.Batch = nil
b.Note = "a batch whose every merge was left out by a stop: " + b.Note
return inv.SavePlan(ctx, b)
}
left := leftOutMerges(merges, said.LeftOut)
first := left[0]
r := inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch,
Commit: first.Commit, Merged: first.Merged, Created: now, Tiers: [][]string{},
Modules: map[string]*inventory.PlanModule{}}
if r.ID == b.ID {
r.ID += "-left"
}
stopped(&r)
r.Delivery.WalkedAgainBy = b.ID
r.Note += fmt.Sprintf("; left out of %s before it was walked: %s; %s walks the others", b.ID,
mergesLeftOutWords(said.LeftOut), b.ID)
b.Delivery.Batch.Withheld = union(b.Delivery.Batch.Withheld, said.Withheld)
sort.Strings(b.Delivery.Batch.Withheld)
var moves []inventory.MergeMove
for _, m := range left {
moves = append(moves, inventory.MergeMove{Repository: m.Repository, Commit: m.Commit, Plan: r.ID, Alone: m.Alone})
}
for _, m := range again {
said.WalkedAgain = append(said.WalkedAgain, m.Repository+"@"+m.Commit)
}
said.Stopped, said.WalkedAgainBy = r.ID, b.ID
if err := inv.SavePlansMoving(ctx, []*inventory.Plan{&r, b}, moves); err != nil {
return fmt.Errorf("%s is not changed, and none of its merges moved: %w", b.ID, err)
}
if err := keepBatch(ctx, inv, b, now, ""); err != nil {
fmt.Printf(" %s holds its merges; its record is written again at the next look: %v\n", b.ID, err)
}
fmt.Printf("%s: %s left out by a stop, kept by %s\n", b.ID, mergesLeftOutWords(said.LeftOut), r.ID)
return nil
}
// batchToWalkAgain is the batch a stopped walk's merges join: the open batch of their kind, or a new one, not yet
// written. The kind is read from what the merges moved.
func batchToWalkAgain(ctx context.Context, inv *inventory.Inventory, again []inventory.BatchedMerge,
now time.Time) (inventory.Plan, error) {
own := false
for _, n := range movesOf(again) {
if _, on := onTheControllersPath[n]; on {
own = true
}
}
batches, err := inv.Batches(ctx)
if err != nil { if err != nil {
return inventory.Plan{}, err return inventory.Plan{}, err
} }
if !p.Open() { for _, b := range batches {
return p, fmt.Errorf("%s is already %s", p.ID, p.State) if b.OwnPath() == own {
if b.Delivery == nil {
b.Delivery = &inventory.PlanDelivery{}
}
if b.Delivery.Batch == nil {
b.Delivery.Batch = &inventory.PlanBatch{Own: own}
}
return b, nil
}
} }
if p.Delivery == nil { first := again[0]
p.Delivery = &inventory.PlanDelivery{} return inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch,
Commit: first.Commit, Merged: first.Merged, Created: now, State: inventory.PlanAssembling,
Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{},
Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: own}}}, nil
}
// leaveOut splits a walk's merges by the merges named to leave out, each `<repository>@<commit>` (a commit's
// first characters will do): the merges walked again, and those left out — each named one; each later merge of
// its repository's branch, which contains it; and each merge moving a module a left-out merge moves, which is
// built at its branch and so holds the stopped merge — until none more is. A name the walk does not answer is
// refused.
func leaveOut(merges []inventory.BatchedMerge, without []string) ([]inventory.BatchedMerge, []inventory.PlanLeftOut, error) {
var stopped []inventory.BatchedMerge
for _, w := range without {
repository, commit, ok := strings.Cut(strings.TrimSpace(w), "@")
if !ok || repository == "" || len(commit) < 7 {
return nil, nil, fmt.Errorf("%q names no merge: <repository>@<commit>", w)
}
i := slices.IndexFunc(merges, func(m inventory.BatchedMerge) bool {
return strings.EqualFold(m.Repository, repository) && strings.HasPrefix(m.Commit, commit)
})
if i < 0 {
return nil, nil, fmt.Errorf("it answers no merge %s@%s", repository, commit)
}
stopped = append(stopped, merges[i])
} }
p.Delivery.Stopped, p.Delivery.StoppedWhy = by, why same := func(a, b inventory.BatchedMerge) bool {
p.State = inventory.PlanFailed return strings.EqualFold(a.Repository, b.Repository) && a.Commit == b.Commit
p.Note = fmt.Sprintf("stopped by %s at tier %d: %s", by, p.Tier, why)
if err := inv.SavePlan(ctx, &p); err != nil {
return p, err
} }
return p, nil out := map[int]inventory.PlanLeftOut{}
for i, m := range merges {
if slices.ContainsFunc(stopped, func(s inventory.BatchedMerge) bool { return same(s, m) }) {
out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit}
}
}
for grew := true; grew; {
grew = false
for i, m := range merges {
if _, left := out[i]; left {
continue
}
for j, o := range out {
l := merges[j]
contains := o.Contains
if contains == "" {
contains = l.Commit
}
if strings.EqualFold(l.Repository, m.Repository) && l.Branch == m.Branch && laterMerge(m, l) {
out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit, Contains: contains}
} else if shared := sharedMoves(m, l); len(shared) > 0 {
out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit, Contains: contains,
Shares: shared}
} else {
continue
}
grew = true
break
}
}
}
var again []inventory.BatchedMerge
var left []inventory.PlanLeftOut
for i, m := range merges {
if o, is := out[i]; is {
left = append(left, o)
} else {
again = append(again, m)
}
}
sort.SliceStable(left, func(i, j int) bool { return left[i].Contains == "" && left[j].Contains != "" })
return again, left, nil
}
// sharedMoves are the modules two kept merges both moved when they were heard.
func sharedMoves(a, b inventory.BatchedMerge) []string {
var out []string
theirs := announcedOf(b).Moves
for _, n := range announcedOf(a).Moves {
if slices.Contains(theirs, n) && !slices.Contains(out, n) {
out = append(out, n)
}
}
sort.Strings(out)
return out
}
// leftOutMerges are the kept merges a stop left out.
func leftOutMerges(merges []inventory.BatchedMerge, out []inventory.PlanLeftOut) []inventory.BatchedMerge {
var left []inventory.BatchedMerge
for _, m := range merges {
if slices.ContainsFunc(out, func(o inventory.PlanLeftOut) bool {
return strings.EqualFold(o.Repository, m.Repository) && o.Commit == m.Commit
}) {
left = append(left, m)
}
}
return left
}
// movesOf is every module kept merges moved when they were heard, sorted.
func movesOf(merges []inventory.BatchedMerge) []string {
var out []string
for _, m := range merges {
out = union(out, announcedOf(m).Moves)
}
sort.Strings(out)
return out
}
// keptOf are the modules a stopped walk sent that no merge walked again moves, and the machines each reached:
// those keep running the stopped walk's build. A module sent to its first machines reached those, and the rest it
// was sent to after them; one sent with no first machine (its policy sends it to all together, or a later tier is
// built by it) reached every machine running it.
func keptOf(p inventory.Plan, walkedAgain []string, running map[string][]string) []inventory.PlanKept {
var out []inventory.PlanKept
for name, m := range p.Modules {
if m == nil || slices.Contains(walkedAgain, name) {
continue
}
var machines []string
if m.FirstAt != nil || m.SentAt != nil {
machines = union(machines, m.First)
}
if m.SentAt != nil && len(m.First) == 0 {
machines = union(machines, running[name])
}
if m.Gate != nil && m.Gate.Since != nil {
machines = union(machines, m.Gate.Machines)
}
for machine := range m.Rest {
machines = union(machines, []string{machine})
}
if len(machines) == 0 {
continue
}
sort.Strings(machines)
out = append(out, inventory.PlanKept{Module: name, Machines: machines})
}
sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module })
return out
}
// keptWords is what a stopped walk leaves on the machines, as a person reads it.
func keptWords(kept []inventory.PlanKept) string {
var words []string
for _, k := range kept {
words = append(words, k.Module+" on "+strings.Join(k.Machines, ", "))
}
return "left running the stopped walk's build, which nothing walks again: " + strings.Join(words, "; ")
}
// stillCarried are the merges outside the stopped record, in a batch or an open walk, that will deliver a stopped
// change when they are walked: a later merge of its repository's branch, which contains it, or one that moves a
// module it moves. Holding a merged change back for good is a revert's (ADR 0299).
func stillCarried(ctx context.Context, inv *inventory.Inventory, id string, merges []inventory.BatchedMerge,
out []inventory.PlanLeftOut, withheld []string) ([]string, error) {
var named []inventory.BatchedMerge
for _, m := range leftOutMerges(merges, out) {
if slices.ContainsFunc(out, func(o inventory.PlanLeftOut) bool { return o.Commit == m.Commit && o.Contains == "" }) {
named = append(named, m)
}
}
if len(named) == 0 {
return nil, nil
}
open, err := inv.OpenPlans(ctx)
if err != nil {
return nil, err
}
batches, err := inv.Batches(ctx)
if err != nil {
return nil, err
}
var carried []string
for _, p := range append(open, batches...) {
if p.ID == id {
continue
}
ms, err := inv.MergesOf(ctx, p.ID)
if err != nil {
return nil, err
}
for _, m := range ms {
later := slices.ContainsFunc(named, func(s inventory.BatchedMerge) bool {
return strings.EqualFold(s.Repository, m.Repository) && s.Branch == m.Branch && laterMerge(m, s)
})
shares := slices.ContainsFunc(announcedOf(m).Moves, func(n string) bool { return slices.Contains(withheld, n) })
if later || shares {
carried = append(carried, fmt.Sprintf("%s@%s (in %s)", m.Repository, m.Commit, p.ID))
}
}
}
sort.Strings(carried)
return carried, nil
}
// mergesLeftOutWords is the merges a stop left out, as a person reads them: each that contains a stopped one, or
// shares its modules, says so.
func mergesLeftOutWords(out []inventory.PlanLeftOut) string {
var words []string
for _, o := range out {
s := o.Repository + "@" + short(o.Commit)
switch {
case len(o.Shares) > 0:
s += " (which moves " + strings.Join(o.Shares, ", ") + " as " + short(o.Contains) + " does)"
case o.Contains != "":
s += " (which contains " + short(o.Contains) + ")"
}
words = append(words, s)
}
return strings.Join(words, ", ")
} }
// orderMember is one member of a group, as mesh-delivery says it. // orderMember is one member of a group, as mesh-delivery says it.
@@ -466,6 +849,8 @@ func deliveryCommand(ctx context.Context, args []string) error {
group := set.String("group", "", "a delivery group's id") group := set.String("group", "", "a delivery group's id")
by := set.String("by", catalogue.DeliverySeat, "who says it") by := set.String("by", catalogue.DeliverySeat, "who says it")
why := set.String("why", "", "why") why := set.String("why", "", "why")
without := set.String("without", "", "merges a stop leaves out, <repository>@<commit>, comma-separated: the walk's "+
"other merges are walked again without them (novox/hq ADR 0299)")
limit := set.Int("n", 50, "how many ended walks to answer beside the open ones") limit := set.Int("n", 50, "how many ended walks to answer beside the open ones")
planID := set.String("plan", "", "one walk") planID := set.String("plan", "", "one walk")
positionals, err := parseAround(set, rest) positionals, err := parseAround(set, rest)
@@ -552,15 +937,21 @@ func deliveryCommand(ctx context.Context, args []string) error {
return nil return nil
case "stop": case "stop":
if len(positionals) != 1 || strings.TrimSpace(*why) == "" { if len(positionals) != 1 || strings.TrimSpace(*why) == "" {
return errors.New("delivery stop <plan> --why <text> [--by <who>]") return errors.New("delivery stop <plan> --why <text> [--by <who>] [--without <repository>@<commit>,…]")
} }
var p inventory.Plan var p inventory.Plan
var said stopAnswer
if err := sayingOnTheBus(ctx, func() (err error) { if err := sayingOnTheBus(ctx, func() (err error) {
p, err = stopWalk(ctx, inv, positionals[0], *by, strings.TrimSpace(*why)) p, said, err = stopWalkWithout(ctx, inv, positionals[0], *by, strings.TrimSpace(*why), splitList(*without),
time.Now().UTC())
return err return err
}); err != nil { }); err != nil {
return err return err
} }
if *without != "" {
// What it left out and what it walks again, as data: the delivery's owner reads it (ADR 0299).
return answer(said)
}
fmt.Printf("%s stopped by %s at tier %d of %d; what was asked still builds and registers, nothing further "+ fmt.Printf("%s stopped by %s at tier %d of %d; what was asked still builds and registers, nothing further "+
"is asked or sent\n", p.ID, *by, p.Tier, len(p.Tiers)) "is asked or sent\n", p.ID, *by, p.Tier, len(p.Tiers))
return nil return nil
+8 -4
View File
@@ -74,8 +74,12 @@ type probe struct {
// probeRegistry is the registry, in to-be 45's order. **The registry is the design's live form**: a // probeRegistry is the registry, in to-be 45's order. **The registry is the design's live form**: a
// probe added to a design is a row added here. // probe added to a design is a row added here.
var probeRegistry = []probe{ var probeRegistry = []probe{
{ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation", // D1 also says every module assigned and left out of a machine's declaration (novox/hq issue 380), from the
From: "issues 236, 263, 275", Kind: "declaration-refused", Raises: []string{kindAwaitingPush}, Phase: 1, // resolution it already makes: needs-operator for a setting nobody gave, left-out for any other cause.
{ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation; no module " +
"assigned to a machine is left out of its declaration unsaid",
From: "issues 236, 263, 275, 380", Kind: "declaration-refused",
Raises: []string{kindAwaitingPush, kindLeftOut, kindNeedsOperator}, Phase: 1,
run: probeDeclarations}, run: probeDeclarations},
{ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6", {ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6",
From: "issue 262", Kind: "resolver-wrong", Phase: 1, run: probeResolvers}, From: "issue 262", Kind: "resolver-wrong", Phase: 1, run: probeResolvers},
@@ -118,8 +122,8 @@ var probeRegistry = []probe{
Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries}, Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries},
// The delivery budgets (novox/hq ADR 0282 decision 7): the newest delivery of each class within its budget, // The delivery budgets (novox/hq ADR 0282 decision 7): the newest delivery of each class within its budget,
// read from mesh-delivery's `times`; a measurement said, never a delivery held. // read from mesh-delivery's `times`; a measurement said, never a delivery held.
{ID: probeBudgetsID, Asserts: "the newest delivery of a leaf module ran on every machine within five minutes of " + {ID: probeBudgetsID, Asserts: "the newest delivery of a leaf module ran on every machine within three minutes of " +
"its merge, and of a core module within ten: mesh-delivery's `times`", From: "ADR 0282", "its merge, and of a core module within ten: mesh-delivery's `times`", From: "ADR 0282, ADR 0304",
Kind: kindOverBudget, Phase: 3, run: probeBudgets}, Kind: kindOverBudget, Phase: 3, run: probeBudgets},
// A client of the bus reconnecting in a loop (novox/hq issue 327), from the server's record of closed // A client of the bus reconnecting in a loop (novox/hq issue 327), from the server's record of closed
// connections, which the bus's own module reads. // connections, which the bus's own module reads.
+66 -9
View File
@@ -27,8 +27,10 @@ import (
// **"Reported applied" is not enough.** ADR 0218 sent a module to one machine first and the rest once // **"Reported applied" is not enough.** ADR 0218 sent a module to one machine first and the rest once
// that machine reported it applied. A build that applies and then does nothing, crashes, serves no // that machine reported it applied. A build that applies and then does nothing, crashes, serves no
// tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by // tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by
// the component's health — the core's health definitions, or a module's own — passing three times over // the component's health — the core's health definitions, or a module's own — within ten minutes of the
// at least two minutes, within ten minutes of the apply. Only then are the rest sent. // apply: a core module passing three times over at least two minutes, a leaf module once, as soon as the
// node-engine states it healthy (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it). Only then
// are the rest sent.
// //
// **A failing gate stops the plan and puts the previous build back there.** The build is marked failed // **A failing gate stops the plan and puts the previous build back there.** The build is marked failed
// at its gate — registration refuses it and nothing sends it again on its own — the module's registered // at its gate — registration refuses it and nothing sends it again on its own — the module's registered
@@ -43,11 +45,38 @@ var (
gateSettle = 2 * time.Minute gateSettle = 2 * time.Minute
// gateBound is how long after the apply the build may take to become healthy. // gateBound is how long after the apply the build may take to become healthy.
gateBound = 10 * time.Minute gateBound = 10 * time.Minute
// gatePasses is how many consecutive judgings must find it healthy, gateEvery apart at the least. // gatePasses is how many consecutive judgings must find a core module healthy, gateEvery apart at the
gatePasses = 3 // least, after gateSettle. gateLeafPasses is a leaf module's: one, taken once the node-engine states every
gateEvery = 40 * time.Second // resource of it healthy by its own check, with no settle (novox/hq ADR 0282 decision 4 as ADR 0304
// decision 3 changed it, issue 524).
gatePasses = 3
gateLeafPasses = 1
gateLeafSettles = false
gateEvery = 40 * time.Second
) )
// coreModule is whether a module is judged as the core (novox/hq ADR 0282 decision 1): one on the controller's
// own path, or one holding the mesh's resolver. Every other module is a leaf.
func coreModule(module string, m catalogue.Manifest) bool {
if _, own := onTheControllersPath[module]; own {
return true
}
for _, c := range m.Claims {
if c.Name == resolverSeat {
return true
}
}
return coreComponent(module) != ""
}
// gateWants is how many passes a module needs, and whether it must stay healthy gateSettle first.
func gateWants(module string, m catalogue.Manifest) (passes int, settles bool) {
if coreModule(module, m) {
return gatePasses, true
}
return gateLeafPasses, gateLeafSettles
}
// gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions // gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions
// are raised by a plan as it judges, and kept or cleared by the probe on every run. // are raised by a plan as it judges, and kept or cleared by the probe on every run.
const gateProbe = "DG" const gateProbe = "DG"
@@ -608,8 +637,24 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
} }
} }
settled := now.Sub(*g.Since) >= gateSettle settled := now.Sub(*g.Since) >= gateSettle
// What the send needs is what its most demanding module needs: a send carrying a core module is judged
// as the core.
wants, settles := 0, false
for _, m := range modules {
p, s := gateWants(m, shelf[m])
wants, settles = max(wants, p), settles || s
}
if len(modules) == 0 {
// A send judged on no machine of its modules (its first machine not running them) has no class to
// be judged by: it is judged as the core, never passed at once without a reading.
wants, settles = gatePasses, true
}
g.Wants = wants
// A module's own pass is its own class's: a leaf beside a core module that fails keeps its build after
// its one reading, and only what did not pass is put back (issue 318's rule, by class since issue 524).
passedAlone := func(m string) bool { passedAlone := func(m string) bool {
return settled && reading[m] <= healthPerson && g.Healthy[m] >= gatePasses p, s := gateWants(m, shelf[m])
return (settled || !s) && reading[m] <= healthPerson && g.Healthy[m] >= p
} }
// fail decides the send failed: what passed on its own keeps its pass, everything else is put back. // fail decides the send failed: what passed on its own keeps its pass, everything else is put back.
fail := func(why string) { fail := func(why string) {
@@ -657,7 +702,7 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
g.Passes++ g.Passes++
g.LastPass, g.Last, g.Failing = &now, "", nil g.LastPass, g.Last, g.Failing = &now, "", nil
g.Read(now, true, "") g.Read(now, true, "")
if g.Passes >= gatePasses && settled { if g.Passes >= wants && (settled || !settles) {
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes, decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now) now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now)
} }
@@ -1340,9 +1385,9 @@ func gateLine(g *inventory.PlanGate) string {
if g.Why != "" { if g.Why != "" {
line += ": " + g.Why line += ": " + g.Why
} else if g.Last != "" { } else if g.Last != "" {
line += fmt.Sprintf(" (%d of %d passes; wanting: %s)", g.Passes, gatePasses, g.Last) line += fmt.Sprintf(" (%d of %d passes; wanting: %s)", g.Passes, gateWanted(g), g.Last)
} else if g.Verdict == "" { } else if g.Verdict == "" {
line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gatePasses) line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gateWanted(g))
} }
if g.Verdict == "" && len(g.Waits) > 0 { if g.Verdict == "" && len(g.Waits) > 0 {
line += waitsSaid(g.Waits) line += waitsSaid(g.Waits)
@@ -1353,6 +1398,10 @@ func gateLine(g *inventory.PlanGate) string {
if g.Rollback != "" { if g.Rollback != "" {
line += "; " + g.Rollback line += "; " + g.Rollback
} }
// Before the send it judges (novox/hq issue 490).
if said := grantsSaid(g.Grants); said != "" {
line += "; " + said
}
return line return line
} }
@@ -1381,3 +1430,11 @@ func witnessObservation(node string, r lease.Rollback) conditions.Observation {
Summary: fmt.Sprintf("the witness on %s judged the %s %s not healthy %s at %s: %s", node, r.Component, Summary: fmt.Sprintf("the witness on %s judged the %s %s not healthy %s at %s: %s", node, r.Component,
short(r.From), what, r.At.UTC().Format(time.RFC3339), r.Why)} short(r.From), what, r.At.UTC().Format(time.RFC3339), r.Why)}
} }
// gateWanted is the passes a gate's send needs, three where the gate was judged before it was said.
func gateWanted(g *inventory.PlanGate) int {
if g.Wants > 0 {
return g.Wants
}
return gatePasses
}
@@ -53,6 +53,7 @@ func healthyContainer(module string) inventory.ResourceHealth {
// tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass, // tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass,
// counted over the gate's spacing — before it says its own. The broken one alone is put back. // counted over the gate's spacing — before it says its own. The broken one alone is put back.
func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) { func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) {
judgeLeavesAsTheCore(t)
open := aMesh(t) open := aMesh(t)
ctx := t.Context() ctx := t.Context()
withGateBounds(t, 150*time.Millisecond, 20*time.Millisecond, 10*time.Second) withGateBounds(t, 150*time.Millisecond, 20*time.Millisecond, 10*time.Second)
@@ -284,6 +285,7 @@ func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) {
// tools are marked and put back in the very judging that found them broken — their registered build is the // tools are marked and put back in the very judging that found them broken — their registered build is the
// one before, and the laptop is sent it — while `app` goes on being judged, recovers, and keeps its own pass. // one before, and the laptop is sent it — while `app` goes on being judged, recovers, and keeps its own pass.
func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testing.T) { func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testing.T) {
judgeLeavesAsTheCore(t)
open := aMesh(t) open := aMesh(t)
ctx := t.Context() ctx := t.Context()
inv := open.inventory inv := open.inventory
+56 -1
View File
@@ -153,6 +153,7 @@ func (g *gateMesh) plan(t *testing.T) inventory.Plan {
// registered and sent to it again — and never reaches the second machine; it is marked, said as a // registered and sent to it again — and never reaches the second machine; it is marked, said as a
// condition and an event, never registered or rolled back again. // condition and an event, never registered or rolled back again.
func TestABuildThatFailsItsGateIsRolledBackOnItsFirstMachineAndGoesNoFurther(t *testing.T) { func TestABuildThatFailsItsGateIsRolledBackOnItsFirstMachineAndGoesNoFurther(t *testing.T) {
judgeLeavesAsTheCore(t)
g := aGateMesh(t) g := aGateMesh(t)
ctx := t.Context() ctx := t.Context()
inv := g.open.inventory inv := g.open.inventory
@@ -255,7 +256,7 @@ func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
t.Fatalf("sent %v", g.sent) t.Fatalf("sent %v", g.sent)
} }
gate := p.Modules["app"].Gate gate := p.Modules["app"].Gate
if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gatePasses { if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gateWanted(gate) {
t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate) t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate)
} }
if v, found, err := g.open.inventory.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed { if v, found, err := g.open.inventory.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed {
@@ -266,8 +267,52 @@ func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
} }
} }
// A leaf module passes its gate on one reading, as soon as its first machine is healthy, without waiting out the
// core's settle time (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it, issue 524).
func TestALeafModulePassesOnOneHealthyReading(t *testing.T) {
g := aGateMesh(t)
ctx := t.Context()
gateSettle = time.Hour // a core module could not pass for an hour
advancePlans(ctx, g.open) // the first machine is sent the new build
advancePlans(ctx, g.open) // one healthy judging
p := g.plan(t)
gate := p.Modules["app"].Gate
if gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes != 1 || gate.Wants != 1 {
t.Fatalf("a leaf module's gate after one healthy judging: %+v (plan %s: %s)", gate, p.State, p.Note)
}
if !strings.Contains(gate.Why, "healthy 1 times") {
t.Errorf("the verdict says %q", gate.Why)
}
}
// Which modules are judged as the core: those on the controller's own path and the resolver's holder, three
// readings after the settle time; every other module one (novox/hq ADR 0282 decision 1, ADR 0304 decision 3).
func TestAModuleIsJudgedByItsClass(t *testing.T) {
for _, c := range []struct {
module string
claims []catalogue.Claim
passes int
settles bool
}{
{"app", nil, 1, false},
{"nzbget", nil, 1, false},
{"mesh-controller", nil, 3, true},
{"mesh-host", nil, 3, true},
{"node-tools", nil, 3, true},
{"nats", nil, 3, true},
{catalogue.DeliverySeat, nil, 3, true},
{"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, true},
} {
p, s := gateWants(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims})
if p != c.passes || s != c.settles {
t.Errorf("%s: %d passes, settles %v; want %d, %v", c.module, p, s, c.passes, c.settles)
}
}
}
// A first machine that never becomes healthy fails its gate at the bound, and is put back. // A first machine that never becomes healthy fails its gate at the bound, and is put back.
func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) { func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) {
judgeLeavesAsTheCore(t)
g := aGateMesh(t) g := aGateMesh(t)
ctx := t.Context() ctx := t.Context()
advancePlans(ctx, g.open) advancePlans(ctx, g.open)
@@ -593,3 +638,13 @@ func TestAWitnessesVerdictIsAConditionWhileItsReportsSayIt(t *testing.T) {
t.Fatalf("a verdict the reports no longer carry is still said: %+v %v", obs, err) t.Fatalf("a verdict the reports no longer carry is still said: %+v %v", obs, err)
} }
} }
// judgeLeavesAsTheCore makes a test's leaf modules judged as a core module is, three readings after the settle
// time, for a test of the gate's other rules (a fault, the bound, a supersession) that needs a judging still
// open after a healthy reading.
func judgeLeavesAsTheCore(t *testing.T) {
t.Helper()
wasPasses, wasSettles := gateLeafPasses, gateLeafSettles
gateLeafPasses, gateLeafSettles = gatePasses, true
t.Cleanup(func() { gateLeafPasses, gateLeafSettles = wasPasses, wasSettles })
}
+88 -10
View File
@@ -202,21 +202,99 @@ func modulesWords(modules []string) string {
return strings.Join(modules, ", ") return strings.Join(modules, ", ")
} }
// writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234): // writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234), and
// nothing when the mesh has not recorded a send to it. // what it answered (issue 485): nothing when the mesh has not recorded a send to it.
func writeLastSend(ctx context.Context, w io.Writer, inv *inventory.Inventory, node string) error { func writeLastSend(ctx context.Context, w io.Writer, inv *inventory.Inventory, node string) error {
s, found, err := inv.LastSend(ctx, node) s, found, err := inv.LastSend(ctx, node)
if err != nil || !found { if err != nil || !found {
return err return err
} }
generation := "no generation recorded" fmt.Fprintf(w, "%s:\n", node)
if s.Generation > 0 { for _, line := range lastSendLines(s) {
generation = fmt.Sprintf("assignment generation %d", s.Generation) fmt.Fprintln(w, line)
}
fmt.Fprintf(w, "%s was last sent sequence %d at %s by %s, composed from %s, naming %s\n", node, s.Sequence, s.SentAt.Local().Format("2006-01-02 15:04:05"), s.Sender, generation, modulesWords(s.Modules))
if s.RefusedAt != nil {
fmt.Fprintf(w, " and refused it at %s: it had applied generation %d\n",
s.RefusedAt.Local().Format("2006-01-02 15:04:05"), s.RefusedApplied)
} }
return nil return nil
} }
// lastSendLines is a machine's last send in full, as `node show` and `plan` say it (novox/hq issue 485): when,
// by whom, under which lease epoch, from which assignment generation, naming which modules, and what the
// machine answered of it — the refusal in its own words.
func lastSendLines(s inventory.Send) []string {
const stamp = "2006-01-02 15:04:05"
epoch := "under no lease epoch"
if s.Epoch > 0 {
epoch = fmt.Sprintf("under lease epoch %d", s.Epoch)
}
generation := "no assignment generation recorded"
if s.Generation > 0 {
generation = fmt.Sprintf("assignment generation %d", s.Generation)
}
lines := []string{
fmt.Sprintf(" last sent sequence %d at %s", s.Sequence, s.SentAt.Local().Format(stamp)),
fmt.Sprintf(" by %s", s.Sender),
fmt.Sprintf(" %s, composed from %s", epoch, generation),
fmt.Sprintf(" naming %s", modulesWords(s.Modules)),
}
answer := func(words string) { lines = append(lines, " its answer "+words) }
switch a := s.Answer; {
case s.RefusedAt != nil:
// Before the report: a refusal for the generation is kept on the send itself and is the machine's
// answer to this send in particular.
answer(fmt.Sprintf("refused it at %s: it had applied assignment generation %d, newer than this one",
s.RefusedAt.Local().Format(stamp), s.RefusedApplied))
case a.Outcome == "" || a.At == nil:
answer("the machine has not reported on it yet")
case a.Outcome == inventory.OutcomeApplied:
answer("applied it at " + a.At.Local().Format(stamp))
case a.Outcome == inventory.OutcomeRefused:
answer("refused it at " + a.At.Local().Format(stamp) + ", in its words:")
for _, line := range strings.Split(strings.TrimRight(a.Refused, "\n"), "\n") {
lines = append(lines, " "+strings.TrimSpace(line))
}
default:
answer(fmt.Sprintf("%s at %s: %d resource(s) failed — `status` says which", a.Outcome,
a.At.Local().Format(stamp), a.Failed))
}
return lines
}
// sendAnswerWord is what a machine answered of a send, in a word or few, as status says it in one line.
func sendAnswerWord(s inventory.Send) string {
switch a := s.Answer; {
case s.RefusedAt != nil:
return fmt.Sprintf("refused: it had applied generation %d", s.RefusedApplied)
case a.Outcome == "" || a.At == nil:
return "not reported on yet"
case a.Outcome == inventory.OutcomeFailed:
return fmt.Sprintf("failed (%d resource(s))", a.Failed)
default:
return a.Outcome
}
}
// printLastSends says, one line per machine, when it was last sent a declaration, by whom, from which assignment
// generation and what it answered (novox/hq issue 485). `node show <node>` says the send in full.
func printLastSends(nodes []inventory.Node, sends map[string]inventory.Send, unread string) {
if unread != "" {
fmt.Printf("what each machine was last sent could NOT be read: %s\n\n", unread)
return
}
if len(nodes) == 0 {
return
}
fmt.Println("last sent, per machine:")
for _, n := range nodes {
s, found := sends[n.Name]
if !found {
fmt.Printf(" %-12s no send recorded\n", n.Name)
continue
}
generation := "no generation"
if s.Generation > 0 {
generation = fmt.Sprintf("generation %d", s.Generation)
}
fmt.Printf(" %-12s %s by %s, %s — %s\n", n.Name, s.SentAt.Local().Format("2006-01-02 15:04"), s.Sender,
generation, sendAnswerWord(s))
}
fmt.Printf("\n `node show <node>` says a machine's last send in full\n\n")
}
@@ -0,0 +1,198 @@
package main
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/identity"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The grants step on a store (novox/hq issue 490): the machine holding the bus is sent its new user list,
// and nothing else — not the new build of a module it runs, not a module newly assigned there.
//
// The restic shape: anchor holds the bus and runs restic at c1, as does laptop; restic's c2 gives it a
// health tool. The grants step's send to anchor composes restic at c1, carries the user list that grants
// laptop's node-engine c2's tool, and records that anchor still runs c1. A module newly assigned to anchor
// makes the step fail, said, rather than install it unjudged.
func TestTheGrantsStepSendsTheUserListAndNothingElse(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
start := time.Now().Add(-time.Hour)
build := func(module, commit string, at time.Time, manifest map[string]any) link.BuildResult {
manifest["module"], manifest["version"] = module, "1"
raw, _ := json.Marshal(manifest)
return link.BuildResult{ID: link.NewBuildID(at), Repository: "novox/mesh-catalog", Path: "modules/" + module,
On: "anchor", Module: module, Commit: commit, Manifest: raw,
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
}
for _, b := range []link.BuildResult{
build("nats", "n1", start, natsFixture()),
build("restic", "c1", start.Add(time.Second), resticFixture(false)),
build("wallpaper", "w1", start.Add(2*time.Second), wallpaperFixture()),
} {
if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil {
t.Fatal(err)
}
}
for _, a := range [][2]string{{"anchor", "nats"}, {"anchor", "restic"}, {"laptop", "restic"}} {
if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
t.Fatal(err)
}
}
for _, n := range []string{"anchor", "laptop"} {
if _, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: "node." + n, Kind: inventory.BusNode, Node: n}); err != nil {
t.Fatal(err)
}
}
wasEpoch := epochForActs
epochForActs = func(context.Context) (uint64, error) { return 7, nil }
t.Cleanup(func() { epochForActs = wasEpoch })
// A send to anchor composed as sendToEach composes it: numbered, planned, declared, stamped.
compose := func(under context.Context) (catalogue.Resolution, sendable) {
t.Helper()
numbered, err := allot(under, inv, "anchor")
if err != nil {
t.Fatal(err)
}
plan, settings, err := planFor(under, open, "anchor")
if err != nil {
t.Fatal(err)
}
gens, err := generators(under, open)
if err != nil {
t.Fatal(err)
}
declared, err := declarationWith(under, open, "anchor", plan, settings, gens, Allocating)
if err != nil {
t.Fatal(err)
}
numbered.stamp(&declared)
return plan, declared
}
record := func(declared sendable) {
t.Helper()
body, err := declared.Body()
if err != nil {
t.Fatal(err)
}
if _, err := recordSent(ctx, inv, "anchor", body, declared.Builds, declared.Epoch,
sentRecordOf(ctx, declared)); err != nil {
t.Fatal(err)
}
}
// Kept for anchor alone. laptop is never recorded as sent, on purpose: composing anchor resolves laptop
// too (who is on the private network), and a step that kept every machine's builds refused anchor's
// composition for want of laptop's last send (repo-check of #230 at 0f853e93).
kept := keepingEveryBuild(keepingRecorded(ctx), "anchor")
// What anchor was last sent: everything at the builds of before the merge, as an ordinary send sends it.
_, before := compose(keepingRecorded(ctx))
record(before)
plan, declared := compose(kept)
if only, err := grantsOnlyIn(ctx, open, "anchor", plan, declared); err != nil || only != "" {
t.Fatalf("with nothing changed, the step reads %q, %v", only, err)
}
// The merge: restic's c2 gives it a health tool.
if _, _, err := takeIn(ctx, inv, build("restic", "c2", start.Add(time.Minute),
resticFixture(true))); err != nil {
t.Fatal(err)
}
// The step's send to anchor: restic as anchor runs it, c1, and the user list granting c2's tool.
generation, err := inv.AssignmentGeneration(ctx)
if err != nil {
t.Fatal(err)
}
plan, declared = compose(kept)
for _, m := range plan.Modules {
if m.Module == "restic" && len(m.Tools) > 0 {
t.Fatalf("the grants step composed restic's new build on anchor: tools %v", m.Tools)
}
}
if declared.Builds["restic"] != "c1" {
t.Fatalf("the step's send records it carries restic %q; want c1, which anchor runs", declared.Builds["restic"])
}
if !strings.Contains(declared.BusUsers, "mesh.mod.restic.tool.backup_health.laptop") {
t.Errorf("the step's declaration does not grant laptop's node-engine restic's new tool:\n%s", declared.BusUsers)
}
// The generation it carries is the current one (novox/hq issue 234), and the step does not raise it.
if declared.composedFrom != generation {
t.Errorf("the step's send was composed from generation %d; the mesh is at %d", declared.composedFrom, generation)
}
// The guard, on the very declaration the send sends: only the user list differs from the last send.
if only, err := grantsOnlyIn(ctx, open, "anchor", plan, declared); err != nil || only != "" {
t.Fatalf("the step reads as changing more than the user list: %q, %v", only, err)
}
// And the gate's refusal still reads the step's send: it moves nothing there.
if names, err := ungatedIn(kept, open, []string{"anchor"}, ""); err != nil || strings.Join(names, ",") != "anchor" {
t.Fatalf("the step's send is refused as a move: %v, %v", names, err)
}
// Recorded as the send records it: anchor still runs restic c1, its gate still to come there.
record(declared)
if sent, _, err := inv.SentBuilds(ctx, "anchor"); err != nil || sent["restic"] != "c1" {
t.Fatalf("after the step anchor reads as sent restic %q (%v); want c1", sent["restic"], err)
}
if after, err := inv.AssignmentGeneration(ctx); err != nil || after != generation {
t.Fatalf("the step moved the assignment generation from %d to %d (%v)", generation, after, err)
}
// A module newly assigned to anchor: the step would install it, unjudged, so it is refused unsent.
if _, err := inv.Assign(ctx, "anchor", "wallpaper"); err != nil {
t.Fatal(err)
}
plan, declared = compose(kept)
only, err := grantsOnlyIn(ctx, open, "anchor", plan, declared)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(only, "wallpaper newly assigned") || !strings.Contains(only, "+wallpaper") {
t.Fatalf("a new assignment on the bus's machine reads %q; want the step refused, naming it", only)
}
// And the send itself refuses it, unsent: judged where it is composed, before the bus is dialled.
identity.ForTest(t)
if _, err := sendToEach(kept, open, []string{"anchor"}); !errors.Is(err, errNotGrantsOnly) ||
!strings.Contains(err.Error(), "wallpaper") {
t.Fatalf("the grants step's send of a new assignment was not refused by the send: %v", err)
}
}
// fixtureImage is the image every fixture container of the grants step's tests runs.
var fixtureImage = "registry.invalid:5000/restic/backup@sha256:" + strings.Repeat("b", 64)
// resticFixture is restic's manifest in the shape of mesh-catalog #210: at c2 (withTool) its backup judged by
// its new tool `backup_health`, beside a check of another kind it does not run itself — a tool check alone
// is refused (ADR 0227 rule 8, ADR 0240 rule 2) — and at c1 neither.
func resticFixture(withTool bool) map[string]any {
backup := map[string]any{"id": "backup", "type": "container", "name": "restic-backup", "image": fixtureImage}
measure := map[string]any{"id": "measure", "type": "container", "name": "restic-measure", "image": fixtureImage}
m := map[string]any{"resources": []any{backup, measure}}
if withTool {
backup["health"] = map[string]any{"kind": catalogue.HealthTool, "tool": "backup_health"}
measure["health"] = map[string]any{"kind": "runtime"}
m["tools"] = []string{"backup_health"}
}
return m
}
// natsFixture is the bus's module: it holds mesh-broker and is sent the user list.
func natsFixture() map[string]any {
return map[string]any{"bus-users": "/var/lib/nats-module/conf/accounts.conf",
"claims": []any{map[string]any{"name": catalogue.BrokerSeat, "scope": catalogue.ScopeMesh}}}
}
// wallpaperFixture is a module the bus's machine is newly assigned.
func wallpaperFixture() map[string]any {
return map[string]any{"resources": []any{
map[string]any{"id": "paper", "type": "container", "name": "wallpaper", "image": fixtureImage}}}
}
+226
View File
@@ -0,0 +1,226 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The grants reach the bus before the gate (novox/hq issue 490).
//
// On 2026-10-11 a merge gave restic, assigned to every machine, a new health tool. Its walk sent restic to
// its first machine and judged it there; that machine's node-engine asked the new tool and the bus refused
// it, because the grant was in the bus's user list, which only the machine holding the bus carries — and
// that machine runs restic too, so it was left out of the send while restic's new build waited there for
// the very gate that could not pass. A person pushed it by hand, carrying restic's new build there unjudged.
// aSend is one send the walk made: to which machines, and whether every build there was kept.
type aSend struct {
names []string
keepsAll bool
judged []string
}
// sendsRecorded replaces the walk's send and the reading of the bus's user list for one test: the machine
// holding the bus is novox, and its list is behind as behind says.
func sendsRecorded(t *testing.T, holder string, behind bool, refuse error) *[]aSend {
t.Helper()
var sends []aSend
wasSend, wasBehind := sendRollout, brokerBehindOf
t.Cleanup(func() { sendRollout, brokerBehindOf = wasSend, wasBehind })
brokerBehindOf = func(_ context.Context, _ *stores, names []string) (string, bool, error) {
if slices.Contains(names, holder) {
return holder, false, nil
}
return holder, behind, nil
}
sendRollout = func(ctx context.Context, _ *stores, names []string) ([]string, error) {
s := aSend{names: append([]string(nil), names...), keepsAll: len(names) == 1 && everyBuildKept(ctx, names[0])}
for n := range scopeOf(ctx).judged {
s.judged = append(s.judged, n)
}
sends = append(sends, s)
if refuse != nil && s.keepsAll {
return nil, refuse
}
return names, nil
}
return &sends
}
// The restic shape: a new tool on a module on every machine, its first machine ace, the bus on novox.
func TestAWalkSendsTheGrantsToTheBusBeforeTheFirstMachineIsJudged(t *testing.T) {
sends := sendsRecorded(t, "novox", true, nil)
sent, grants, err := sendJudged(t.Context(), nil, "ace")
if err != nil {
t.Fatal(err)
}
if len(*sends) != 2 {
t.Fatalf("the walk made %d send(s), want the grants step and then the first machine: %+v", len(*sends), *sends)
}
first, then := (*sends)[0], (*sends)[1]
if strings.Join(first.names, ",") != "novox" || !first.keepsAll || len(first.judged) != 0 {
t.Errorf("the first send is %+v, want novox alone, every build there kept, judging nothing", first)
}
if strings.Join(then.names, ",") != "ace" || then.keepsAll || strings.Join(then.judged, ",") != "ace" {
t.Errorf("the second send is %+v, want ace, judged", then)
}
if strings.Join(sent, ",") != "ace" {
t.Errorf("the gate's machines are %v, want ace alone: the bus's machine is not judged", sent)
}
if grants == nil || grants.Node != "novox" || grants.At == nil || grants.Failed != "" {
t.Fatalf("the gate keeps %+v as its grants step", grants)
}
line := gateLine(&inventory.PlanGate{Machines: sent, Grants: grants})
if !strings.Contains(line, "grants step: novox sent the bus's user list first, its builds kept") {
t.Errorf("plans says the gate as %q", line)
}
}
// Nothing more when there is nothing to carry: the list unchanged, or the first machine holds the bus.
func TestAWalkTakesNoGrantsStepWhenTheListIsAlreadyThere(t *testing.T) {
for _, c := range []struct {
name string
node string
behind bool
}{{"the list unchanged", "ace", false}, {"the first machine holds the bus", "novox", true}} {
t.Run(c.name, func(t *testing.T) {
sends := sendsRecorded(t, "novox", c.behind, nil)
_, grants, err := sendJudged(t.Context(), nil, c.node)
if err != nil {
t.Fatal(err)
}
if len(*sends) != 1 || (*sends)[0].keepsAll || grants != nil {
t.Errorf("the walk made %+v with grants %+v, want the judged send alone", *sends, grants)
}
})
}
}
// A grants step that cannot be sent is said on the gate, and the walk goes on as it did before.
func TestAGrantsStepThatFailsIsSaidAndTheSendGoesOn(t *testing.T) {
sends := sendsRecorded(t, "novox", true, errors.New("a bus upgrade waits for a person"))
sent, grants, err := sendJudged(t.Context(), nil, "ace")
if err != nil {
t.Fatal(err)
}
if strings.Join(sent, ",") != "ace" || len(*sends) != 2 {
t.Errorf("the walk made %+v", *sends)
}
if grants == nil || grants.Failed == "" || grants.At != nil {
t.Fatalf("the gate keeps %+v", grants)
}
if line := gateLine(&inventory.PlanGate{Machines: sent, Grants: grants}); !strings.Contains(line,
"grants step to novox FAILED, sent without it: a bus upgrade waits for a person") {
t.Errorf("plans says the gate as %q", line)
}
}
// A step whose send would change more than the user list is refused unsent by the send (sendToEach): the
// gate says it FAILED with what differed, and the walk goes on.
func TestAGrantsStepThatWouldCarryMoreIsNotSent(t *testing.T) {
refusal := fmt.Errorf("%w: its send would change more than the bus's user list: -postgres.login-n8n", errNotGrantsOnly)
sends := sendsRecorded(t, "novox", true, refusal)
sent, grants, err := sendJudged(t.Context(), nil, "ace")
if err != nil {
t.Fatal(err)
}
if strings.Join(sent, ",") != "ace" || len(*sends) != 2 {
t.Errorf("the walk made %+v, want the refused step and then ace's judged send", *sends)
}
if grants == nil || grants.At != nil || !strings.Contains(grants.Failed, "-postgres.login-n8n") {
t.Fatalf("the gate keeps %+v", grants)
}
}
// The step's exemption is the list's content alone: its path or mode moving is a change like any other.
func TestOnlyTheUserListsContentIsExempt(t *testing.T) {
list := "nats.bus-users"
file := func(fields map[string]any) sentResource {
r := map[string]any{"id": list, "type": "file", "path": "/conf/accounts.conf", "mode": "0600", "content": "a"}
for k, v := range fields {
r[k] = v
}
body, _ := json.Marshal(map[string]any{"resources": []any{r}})
s, err := summarize(body)
if err != nil {
t.Fatal(err)
}
return s[0]
}
was := file(nil)
if other := otherThanTheList(diffSent([]sentResource{was}, []sentResource{file(map[string]any{"content": "b"})}), list); len(other) != 0 {
t.Errorf("the list's new content reads as more: %v", other)
}
for _, f := range []map[string]any{{"mode": "0644"}, {"path": "/elsewhere"}, {"content": "b", "owner": "nats"}} {
other := otherThanTheList(diffSent([]sentResource{was}, []sentResource{file(f)}), list)
if len(other) != 1 || !strings.HasPrefix(other[0], "~"+list) {
t.Errorf("the list's resource changed by %v reads %v; want it refused", f, other)
}
}
}
// The step keeps the builds of the machine it sends alone: the others its composition resolves are composed
// as any send composes them.
func TestTheGrantsStepKeepsOneMachinesBuilds(t *testing.T) {
ctx := keepingEveryBuild(t.Context(), "novox")
if !everyBuildKept(ctx, "novox") || everyBuildKept(ctx, "ace") || everyBuildKept(t.Context(), "novox") {
t.Error("the grants step keeps the builds of a machine it does not send")
}
}
// The store test's manifests are manifests the controller takes in: checked here, where no store is needed,
// so a fixture the module rules refuse fails before the store test is run (repo-check of #230 at 325575b2).
func TestTheGrantsStepFixturesAreManifests(t *testing.T) {
for name, m := range map[string]map[string]any{"nats": natsFixture(), "restic-c1": resticFixture(false),
"restic-c2": resticFixture(true), "wallpaper": wallpaperFixture()} {
m["module"], m["version"] = strings.SplitN(name, "-", 2)[0], "1"
raw, err := json.Marshal(m)
if err != nil {
t.Fatal(err)
}
if _, err := catalogue.ParseManifest(raw); err != nil {
t.Errorf("%s: %v", name, err)
}
}
}
// The delivery plan says the step before the merge.
func TestAChangePlanSaysTheGrantsStep(t *testing.T) {
const catalogue_ = "http://forge.internal:20000/novox/mesh-catalog.git"
entry := func(name string, on ...string) inventory.Entry {
e := fromRepo(name, catalogue_, "modules/"+name)
e.Source.BuiltFrom = "old"
e.On = on
return e
}
nats := entry("nats", "novox")
nats.Manifest.BusUsers = "/etc/nats/users.conf"
nats.Manifest.Claims = []catalogue.Claim{{Name: catalogue.BrokerSeat, Scope: catalogue.ScopeMesh}}
restic := entry("restic", "ace", "novox", "shanks")
only := entry("bus-tools", "novox")
entries := []inventory.Entry{nats, restic, only}
rollsOut := func(string) (inventory.Upgrade, bool) { return inventory.Upgrade{RollOut: true}, true }
plan := func(paths ...string) link.ChangePlan {
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: paths}
return changePlanOf("novox/mesh-catalog", "main", "head", reachOfMerge(m, entries, nil, nil), entries, rollsOut)
}
text := planText(plan("modules/restic/module.json"))
if !strings.Contains(text, "grants step: if this changes what the bus's user list says (a new tool, subject or "+
"user), novox is sent that list alone, every build there kept, before restic is judged on its first machine") {
t.Errorf("the change plan does not say the grants step:\n%s", text)
}
// A module only on the bus's machine goes there with the list in its own send: no step of its own.
if text := planText(plan("modules/bus-tools/module.json")); strings.Contains(text, "grants step") {
t.Errorf("a module on the bus's machine alone reads:\n%s", text)
}
}
+3
View File
@@ -84,6 +84,9 @@ var handActVerbs = []handActVerb{
"deliberate test, never a repair", DecidedFor: []string{causeDrill}}, "deliberate test, never a repair", DecidedFor: []string{causeDrill}},
// A drill: something broken on purpose to see the mesh raise and clear it. A person's test, never a // A drill: something broken on purpose to see the mesh raise and clear it. A person's test, never a
// repair, however often it is run. // repair, however often it is run.
// A credential a provider's software issued, taken from the provider and kept for its consumers (novox/hq ADR
// 0305): the mesh working as decided, however often the software changes it.
{Verb: handActIssued, Decision: "a credential the provider's software issued, kept for its consumers (ADR 0305)"},
{Verb: "hand-act drill", Decision: "a drill is a person's deliberate test of the mesh, never a repair"}, {Verb: "hand-act drill", Decision: "a drill is a person's deliberate test of the mesh, never a repair"},
// A person's decisions by design. // A person's decisions by design.
{Verb: "retire approve", Decision: "nothing is retired past its bound without a person (ADR 0230)"}, {Verb: "retire approve", Decision: "nothing is retired past its bound without a person (ADR 0230)"},
+4 -1
View File
@@ -25,6 +25,7 @@ import (
// another send reaches anchor, unreported; the gate still passes. And a send that moves a judged module // another send reaches anchor, unreported; the gate still passes. And a send that moves a judged module
// to another build supersedes the judging: no verdict, nothing put back. // to another build supersedes the judging: no verdict, nothing put back.
func TestReplay352(t *testing.T) { func TestReplay352(t *testing.T) {
judgeLeavesAsTheCore(t)
t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn) t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn)
t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging) t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging)
} }
@@ -66,10 +67,11 @@ func testANewerSendDoesNotUnreportTheGatesOwn(t *testing.T) {
// A plan's own first send waits while a release judges the same module on that machine with another build. // A plan's own first send waits while a release judges the same module on that machine with another build.
func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) { func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
judgeLeavesAsTheCore(t)
b := aBacklog(t) b := aBacklog(t)
ctx := t.Context() ctx := t.Context()
advancePlans(ctx, b.open) // the release judges app c2 on anchor advancePlans(ctx, b.open) // the release judges app c2 on anchor
_, _, err := gatedSend(ctx, b.open, "anchor", []inventory.CarriedMove{{Module: "app", Node: "anchor", From: "c2", To: "c3", Build: "build-app-c3"}}) _, _, _, err := gatedSend(ctx, b.open, "anchor", []inventory.CarriedMove{{Module: "app", Node: "anchor", From: "c2", To: "c3", Build: "build-app-c3"}})
if !errors.Is(err, errWalkedElsewhere) || !strings.Contains(err.Error(), "release-") { if !errors.Is(err, errWalkedElsewhere) || !strings.Contains(err.Error(), "release-") {
t.Fatalf("a newer build of a judged module was sent under the release's gate: %v", err) t.Fatalf("a newer build of a judged module was sent under the release's gate: %v", err)
} }
@@ -80,6 +82,7 @@ func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
// A merge plan's judging is superseded the same way: another send moved its module on the first machine. // A merge plan's judging is superseded the same way: another send moved its module on the first machine.
func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) { func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) {
judgeLeavesAsTheCore(t)
g := aGateMesh(t) g := aGateMesh(t)
ctx := t.Context() ctx := t.Context()
inv := g.open.inventory inv := g.open.inventory
+427
View File
@@ -0,0 +1,427 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"reflect"
"slices"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
// novox/hq issue 459, ADR 0299: since the merge window one walk carries every merge of its batch, so a stop of one
// group member through `delivery-stop` ended the walk of every other delivery in it. The stop now names the merges
// to leave out: the walk is ended, and every other merge it answered is walked again in a fresh batch without them
// — the stopped merge, and each later merge of its repository's branch, which contains it and ends with it.
// endsItsLine ends, as the test ends, a line the controller's output left open (a walk's ask prints " tier N: " and
// the asked build the rest, which the tests' fake ask never prints): `go test -v` would put the test's "--- PASS"
// after it, and the store-tests step of merge-check.sh, which reads that marker at a line's start, would say the
// test never ran.
func endsItsLine(t *testing.T) {
t.Cleanup(func() { fmt.Println() })
}
// leaved is a kept merge of a repository, made at a moment.
func leaved(repository, commit string, made time.Time) inventory.BatchedMerge {
return inventory.BatchedMerge{Repository: repository, Branch: "main", Commit: commit, Merged: made, Heard: made}
}
// The stopped merge is left out, and so is each later merge of its repository's branch, naming the merge it
// contains; an earlier merge of that repository and every other repository's merge are walked again.
func TestLeftOutAreTheStoppedMergeAndTheMergesThatContainIt(t *testing.T) {
c1 := leaved("novox/mesh-catalog", "c1aaaaaaaaaa", t0)
c2 := leaved("novox/mesh-catalog", "c2bbbbbbbbbb", t0.Add(10*time.Second))
c3 := leaved("novox/mesh-catalog", "c3cccccccccc", t0.Add(20*time.Second))
o1 := leaved("novox/one", "o1dddddddddd", t0.Add(5*time.Second))
again, out, err := leaveOut([]inventory.BatchedMerge{c1, o1, c2, c3}, []string{"novox/Mesh-Catalog@c2bbbbbbbbbb"})
if err != nil {
t.Fatal(err)
}
var walked []string
for _, m := range again {
walked = append(walked, m.Commit)
}
if !reflect.DeepEqual(walked, []string{c1.Commit, o1.Commit}) {
t.Fatalf("walked again: %v; want the earlier catalogue merge and the other repository's", walked)
}
want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: c2.Commit},
{Repository: "novox/mesh-catalog", Commit: c3.Commit, Contains: c2.Commit}}
if !reflect.DeepEqual(out, want) {
t.Fatalf("left out: %+v; want %+v", out, want)
}
// A merge the walk does not answer is refused, naming it: nothing ends.
if _, _, err := leaveOut([]inventory.BatchedMerge{c1, o1}, []string{"novox/mesh-catalog@ffffffffffff"}); err == nil ||
!strings.Contains(err.Error(), "ffffffffffff") {
t.Fatalf("a merge the walk does not answer was left out: %v", err)
}
if _, _, err := leaveOut([]inventory.BatchedMerge{c1}, []string{"no-commit"}); err == nil {
t.Fatal("a merge without its commit was taken")
}
}
// moving is a kept merge that moved modules when it was heard.
func moving(m inventory.BatchedMerge, modules ...string) inventory.BatchedMerge {
m.Event, _ = json.Marshal(keptMerge{Moves: modules})
return m
}
// A merge of another repository that moves a module the stopped merge moves is left out too, naming the modules
// it shares: the module is built at its branch, which holds the stopped merge (review of #232). And so on, until
// none more is: a merge sharing a module with that one is left out as well.
func TestAMergeSharingAStoppedMergesModuleIsLeftOut(t *testing.T) {
stopped := moving(leaved("novox/mesh-catalog", "c1aaaaaaaaaa", t0), "app")
reader := moving(leaved("novox/one", "o1bbbbbbbbbb", t0.Add(time.Second)), "app", "one")
next := moving(leaved("novox/two", "t1cccccccccc", t0.Add(2*time.Second)), "one")
free := moving(leaved("novox/three", "h1dddddddddd", t0.Add(3*time.Second)), "three")
again, out, err := leaveOut([]inventory.BatchedMerge{stopped, reader, next, free}, []string{"novox/mesh-catalog@c1aaaaaaa"})
if err != nil {
t.Fatal(err)
}
if len(again) != 1 || again[0].Commit != free.Commit {
t.Fatalf("walked again: %+v; want the merge that shares nothing alone", again)
}
want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: stopped.Commit},
{Repository: "novox/one", Commit: reader.Commit, Contains: stopped.Commit, Shares: []string{"app"}},
{Repository: "novox/two", Commit: next.Commit, Contains: stopped.Commit, Shares: []string{"one"}}}
if !reflect.DeepEqual(out, want) {
t.Fatalf("left out %+v; want %+v", out, want)
}
}
// A walk withholds a module a stopped merge moved where it would build it only as a dependent; one a merge of its
// own batch moves is built, and empty tiers go.
func TestAWalkWithholdsAStoppedMergesDependents(t *testing.T) {
plan := inventory.Plan{Tiers: [][]string{{"one"}, {"app"}, {"notes", "app2"}},
Modules: map[string]*inventory.PlanModule{"one": {}, "app": {}, "notes": {}, "app2": {}}}
got := withhold(&plan, []string{"app", "notes", "absent"}, []string{"notes"})
if !reflect.DeepEqual(got, []string{"app"}) || !reflect.DeepEqual(plan.Tiers, [][]string{{"one"}, {"notes", "app2"}}) {
t.Fatalf("withheld %v, tiers %v", got, plan.Tiers)
}
if _, in := plan.Modules["app"]; in {
t.Fatal("a withheld module is still in the plan")
}
}
// The seat verb passes the merges to leave out to the command.
func TestDeliveryStopTakesTheMergesToLeaveOut(t *testing.T) {
got, err := argvFor("delivery-stop", map[string]any{"plan": "plan-1", "why": "w", "by": "jochen",
"without": "novox/mesh-lab@b1,novox/mesh-lab@b2"})
want := []string{"delivery", "stop", "plan-1", "--why", "w", "--by", "mesh-delivery for jochen",
"--without", "novox/mesh-lab@b1,novox/mesh-lab@b2"}
if err != nil || !reflect.DeepEqual(got, want) {
t.Fatalf("delivery-stop with without → %v %v, wanted %v", got, err, want)
}
}
// A walk of the catalogue's merge and another repository's is stopped without the catalogue's: it ends stopped,
// names the batch that walks the other merge again and the merge it left out, and that batch is cut into a walk
// that builds the other repository's module alone.
func TestAStopWithoutAMergeWalksTheOthersAgain(t *testing.T) {
open := windowed(t)
endsItsLine(t)
asksRecorded(t)
ctx := t.Context()
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
one := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second))
hear(t, open, cat, t0.Add(time.Second))
hear(t, open, one, t0.Add(6*time.Second))
cutAt(t, open, t0.Add(2*time.Minute))
ws, _ := walks(t, open)
if len(ws) != 1 || !ws[0].Open() {
t.Fatalf("no open walk to stop: %+v", ws)
}
stopped := ws[0]
p, said, err := stopWalkWithout(ctx, open.inventory, stopped.ID, "mesh-delivery for jochen", "its group stopped",
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute))
if err != nil {
t.Fatal(err)
}
if p.State != inventory.PlanFailed || p.Delivery.Stopped == "" || p.Delivery.WalkedAgainBy == "" ||
!reflect.DeepEqual(p.Delivery.LeftOut, []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: cat.Commit}}) {
t.Fatalf("the stopped walk reads %s %+v", p.State, p.Delivery)
}
if said.WalkedAgainBy != p.Delivery.WalkedAgainBy || !reflect.DeepEqual(said.WalkedAgain, []string{"novox/one@" + one.Commit}) {
t.Fatalf("the stop answered %+v", said)
}
again, err := open.inventory.MergesOf(ctx, p.Delivery.WalkedAgainBy)
if err != nil || len(again) != 1 || again[0].Commit != one.Commit {
t.Fatalf("the fresh batch answers %+v (%v); want the other repository's merge alone", again, err)
}
if left, _ := open.inventory.MergesOf(ctx, stopped.ID); len(left) != 1 || left[0].Commit != cat.Commit {
t.Fatalf("the stopped walk still answers %+v; want the left-out merge alone", left)
}
cutAt(t, open, t0.Add(4*time.Minute))
fresh, err := open.inventory.PlanByID(ctx, p.Delivery.WalkedAgainBy)
if err != nil || fresh.Batch() || !fresh.Open() {
t.Fatalf("the fresh batch was not cut into a walk: %s %v", fresh.State, err)
}
if _, in := fresh.Modules["one"]; !in {
t.Fatalf("the fresh walk does not build one: %v", fresh.Modules)
}
if _, in := fresh.Modules["app"]; in {
t.Fatalf("the fresh walk builds app, whose merge was left out: %v", fresh.Modules)
}
if len(fresh.Delivery.Merges) != 1 || fresh.Delivery.Merges[0].Commit != one.Commit {
t.Fatalf("the fresh walk answers %+v", fresh.Delivery.Merges)
}
// A stopped walk starts no search of its own (ADR 0276 decision 3 is for a failed one).
cutAt(t, open, t0.Add(5*time.Minute))
if alone, _ := open.inventory.AloneMerges(ctx); len(alone) != 0 {
t.Fatalf("a merge of the stopped walk waits to be walked alone: %+v", alone)
}
}
// A later merge of the stopped merge's repository contains it: it is left out too, named with the merge it
// contains, and only the other repository's merge is walked again. With nothing left to walk, the walk is only
// ended. A merge the walk does not answer is refused, and nothing ends.
func TestAStopWithoutAMergeEndsTheMergesThatContainIt(t *testing.T) {
open := windowed(t)
endsItsLine(t)
asksRecorded(t)
ctx := t.Context()
first := catalogueMerge("cat1aaaaaaaa", "app", t0)
later := catalogueMerge("cat2cccccccc", "notes", t0.Add(10*time.Second))
one := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second))
hear(t, open, first, t0.Add(time.Second))
hear(t, open, one, t0.Add(6*time.Second))
hear(t, open, later, t0.Add(11*time.Second))
cutAt(t, open, t0.Add(2*time.Minute))
ws, _ := walks(t, open)
walk := ws[0]
if _, _, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "x",
[]string{"novox/mesh-catalog@ffffffffffff"}, t0.Add(3*time.Minute)); err == nil {
t.Fatal("a stop leaving out a merge the walk does not answer was taken")
}
if p, _ := open.inventory.PlanByID(ctx, walk.ID); !p.Open() {
t.Fatalf("a refused stop ended the walk: %s", p.State)
}
p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped",
[]string{"novox/mesh-catalog@" + first.Commit}, t0.Add(3*time.Minute))
if err != nil {
t.Fatal(err)
}
want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: first.Commit},
{Repository: "novox/mesh-catalog", Commit: later.Commit, Contains: first.Commit}}
if !reflect.DeepEqual(p.Delivery.LeftOut, want) || !reflect.DeepEqual(said.LeftOut, want) {
t.Fatalf("left out %+v, said %+v; want %+v", p.Delivery.LeftOut, said.LeftOut, want)
}
if !strings.Contains(p.Note, later.Commit[:8]) || !strings.Contains(p.Note, p.Delivery.WalkedAgainBy) {
t.Fatalf("the stopped walk's note does not name the merge ended with it and the fresh batch: %q", p.Note)
}
again, _ := open.inventory.MergesOf(ctx, p.Delivery.WalkedAgainBy)
if len(again) != 1 || again[0].Commit != one.Commit {
t.Fatalf("walked again: %+v; want the other repository's merge alone", again)
}
// Nothing left once the other repository's merge is left out too: the fresh walk is stopped, and nothing more.
cutAt(t, open, t0.Add(4*time.Minute))
fresh, _ := open.inventory.PlanByID(ctx, p.Delivery.WalkedAgainBy)
q, said, err := stopWalkWithout(ctx, open.inventory, fresh.ID, "mesh-delivery for jochen", "that one too",
[]string{"novox/one@" + one.Commit}, t0.Add(5*time.Minute))
if err != nil || q.State != inventory.PlanFailed || q.Delivery.WalkedAgainBy != "" || said.WalkedAgainBy != "" {
t.Fatalf("a stop leaving nothing to walk: %s %+v %+v %v", q.State, q.Delivery, said, err)
}
if _, bs := walks(t, open); len(bs) != 0 {
t.Fatalf("a batch of nothing was made: %+v", bs)
}
}
// A merge heard while the walk ran waits in the open batch, at a later commit of the same repository. The merge
// walked again joins that batch, never a batch of its own that would walk the older commit after the newer one
// (review of #232): its window closes, and its walk is at the newer commit, answering both.
func TestAStopWalksTheOthersAgainInTheOpenBatch(t *testing.T) {
open := windowed(t)
endsItsLine(t)
asksRecorded(t)
ctx := t.Context()
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
one1 := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second))
hear(t, open, cat, t0.Add(time.Second))
hear(t, open, one1, t0.Add(6*time.Second))
cutAt(t, open, t0.Add(2*time.Minute))
ws, _ := walks(t, open)
walk := ws[0]
one2 := repoMerge("one", "one2cccccccc", t0.Add(3*time.Minute))
hear(t, open, one2, t0.Add(3*time.Minute))
_, bs := walks(t, open)
if len(bs) != 1 {
t.Fatalf("the merge heard while the walk ran is in %d batches", len(bs))
}
waiting := bs[0]
p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped",
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second))
if err != nil {
t.Fatal(err)
}
if p.Delivery.WalkedAgainBy != waiting.ID || said.WalkedAgainBy != waiting.ID {
t.Fatalf("walked again by %q (said %q); want the open batch %s", p.Delivery.WalkedAgainBy, said.WalkedAgainBy, waiting.ID)
}
if _, bs := walks(t, open); len(bs) != 1 {
t.Fatalf("%d batches; want the open one alone", len(bs))
}
b, _ := open.inventory.PlanByID(ctx, waiting.ID)
if b.Delivery.Batch == nil || !b.Delivery.Batch.Closed || !slices.Contains(b.Delivery.Batch.Withheld, "app") {
t.Fatalf("the open batch reads %+v; want its window closed and app withheld", b.Delivery.Batch)
}
// Its window closed by the stop, not by the clock: cut at once.
cutAt(t, open, t0.Add(3*time.Minute+11*time.Second))
fresh, _ := open.inventory.PlanByID(ctx, waiting.ID)
if fresh.Batch() || fresh.CommitOf("novox/one") != one2.Commit {
t.Fatalf("the batch is %s at %q; want it cut, walking the newer commit", fresh.State, fresh.CommitOf("novox/one"))
}
var answered []string
for _, m := range fresh.Delivery.Merges {
answered = append(answered, m.Commit+">"+m.Carried)
}
if !slices.Contains(answered, one1.Commit+">"+one2.Commit) || !slices.Contains(answered, one2.Commit+">") {
t.Fatalf("the walk answers %v; want the earlier merge carried by the later", answered)
}
}
// A stop names what the stopped walk already sent and nothing walks again, on which machines; and a later merge of
// the stopped merge's repository that waits in a batch, which will deliver the stopped change. A stopped walk
// that walked its merges again is refused a retry, saying why.
func TestAStopNamesWhatStaysOnTheMachinesAndWhatStillCarriesIt(t *testing.T) {
open := windowed(t)
endsItsLine(t)
asksRecorded(t)
ctx := t.Context()
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
one1 := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second))
hear(t, open, cat, t0.Add(time.Second))
hear(t, open, one1, t0.Add(6*time.Second))
cutAt(t, open, t0.Add(2*time.Minute))
ws, _ := walks(t, open)
walk := ws[0]
sent := t0.Add(150 * time.Second)
for name, m := range walk.Modules {
if name == "app" || name == "one" {
m.First, m.FirstAt = []string{"anchor"}, &sent
}
}
if err := open.inventory.SavePlan(ctx, &walk); err != nil {
t.Fatal(err)
}
cat2 := catalogueMerge("cat2dddddddd", "notes", t0.Add(3*time.Minute))
hear(t, open, cat2, t0.Add(3*time.Minute))
_, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped",
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second))
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(said.Kept, []inventory.PlanKept{{Module: "app", Machines: []string{"anchor"}}}) {
t.Fatalf("kept %+v; want app on anchor alone (one is walked again)", said.Kept)
}
p, _ := open.inventory.PlanByID(ctx, walk.ID)
if !reflect.DeepEqual(p.Delivery.Kept, said.Kept) || !strings.Contains(p.Note, "app on anchor") {
t.Fatalf("the stopped walk's record does not name what stays: %+v %q", p.Delivery.Kept, p.Note)
}
if len(said.StillCarriedBy) != 1 || !strings.Contains(said.StillCarriedBy[0], cat2.Commit) {
t.Fatalf("still carried by %v; want the later catalogue merge waiting in the batch", said.StillCarriedBy)
}
if _, err := retryPlan(ctx, open, walk.ID); err == nil || !strings.Contains(err.Error(), "was stopped without") {
t.Fatalf("a walk stopped without some merges was retried, or refused for another reason: %v", err)
}
}
// A batch not yet cut is stopped without a merge: the merge goes to a stopped record of its own, the batch goes on
// with the others and withholds the stopped merge's modules; with every merge left out, the batch is stopped.
func TestAStopWithoutAMergeLeavesItOutOfABatch(t *testing.T) {
open := windowed(t)
endsItsLine(t)
asksRecorded(t)
ctx := t.Context()
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
one := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second))
hear(t, open, cat, t0.Add(time.Second))
hear(t, open, one, t0.Add(6*time.Second))
_, bs := walks(t, open)
b := bs[0]
r, said, err := stopWalkWithout(ctx, open.inventory, b.ID, "mesh-delivery for jochen", "its group stopped",
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(10*time.Second))
if err != nil {
t.Fatal(err)
}
if said.WalkedAgainBy != b.ID || said.Stopped == b.ID {
t.Fatalf("said %+v; want a stopped record of its own, and the batch walking the others", said)
}
if left, _ := open.inventory.MergesOf(ctx, said.Stopped); len(left) != 1 || left[0].Commit != cat.Commit {
t.Fatalf("the stopped record holds %+v", left)
}
if stopped, _ := open.inventory.PlanByID(ctx, said.Stopped); stopped.State != inventory.PlanFailed ||
stopped.Delivery.Stopped == "" || stopped.Delivery.WalkedAgainBy != b.ID {
t.Fatalf("the stopped record reads %s %+v", stopped.State, stopped.Delivery)
}
_ = r
cutAt(t, open, t0.Add(3*time.Minute))
w, _ := open.inventory.PlanByID(ctx, b.ID)
if _, in := w.Modules["app"]; in || w.Batch() {
t.Fatalf("the batch's walk is %s and builds %v; want it cut without app", w.State, w.Modules)
}
// A batch whose every merge is left out is stopped itself.
hear(t, open, catalogueMerge("cat3eeeeeeee", "notes", t0.Add(4*time.Minute)), t0.Add(4*time.Minute))
_, bs = walks(t, open)
last := bs[0]
if _, said, err := stopWalkWithout(ctx, open.inventory, last.ID, "mesh-delivery for jochen", "x",
[]string{"novox/mesh-catalog@cat3eeeeeeee"}, t0.Add(4*time.Minute+time.Second)); err != nil || said.Stopped != last.ID {
t.Fatalf("a batch left with nothing: %+v %v", said, err)
}
if p, _ := open.inventory.PlanByID(ctx, last.ID); p.State != inventory.PlanFailed || p.Batch() {
t.Fatalf("the emptied batch is %s", p.State)
}
}
// The stopped walk, the batch and the merges moved are one act (ADR 0299 decision 7): a plan written by somebody
// else since it was read refuses all of it, and no merge moves, no plan is written.
func TestSavingPlansAndMovingMergesIsOneAct(t *testing.T) {
open := windowed(t)
endsItsLine(t)
ctx := t.Context()
hear(t, open, catalogueMerge("cat1aaaaaaaa", "app", t0), t0.Add(time.Second))
_, bs := walks(t, open)
stale := bs[0]
moved := stale
moved.Note = "written by somebody else"
if err := open.inventory.SavePlan(ctx, &moved); err != nil {
t.Fatal(err)
}
fresh := inventory.Plan{ID: "plan-fresh", Repository: "novox/mesh-catalog", Branch: "main", Commit: "x", Created: t0,
State: inventory.PlanAssembling, Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{}}
err := open.inventory.SavePlansMoving(ctx, []*inventory.Plan{&fresh, &stale},
[]inventory.MergeMove{{Repository: "novox/mesh-catalog", Commit: "cat1aaaaaaaa", Plan: "plan-fresh"}})
if !errors.Is(err, inventory.ErrPlanMoved) {
t.Fatalf("a stale plan in the act: %v", err)
}
if _, err := open.inventory.PlanByID(ctx, "plan-fresh"); err == nil {
t.Fatal("the new plan was written though the act was refused")
}
if m, _, _ := open.inventory.MergeOf(ctx, "novox/mesh-catalog", "cat1aaaaaaaa"); m.Plan != stale.ID {
t.Fatalf("the merge moved to %q though the act was refused", m.Plan)
}
}
// What a stopped walk leaves on the machines names every machine its sends reached (review of #232): a module sent
// with no first machine reached every machine running it; one sent first reached those, and the rest it was sent
// to; a module a merge walked again moves is sent again, and is not named.
func TestKeptNamesEveryMachineASendReached(t *testing.T) {
at := t0
p := inventory.Plan{Modules: map[string]*inventory.PlanModule{
"together": {SentAt: &at},
"first": {First: []string{"anchor"}, FirstAt: &at, Rest: map[string]inventory.SentDeclaration{"laptop": {}}},
"again": {SentAt: &at},
"unsent": {},
}}
got := keptOf(p, []string{"again"}, map[string][]string{"together": {"laptop", "anchor"}, "again": {"anchor"}})
want := []inventory.PlanKept{{Module: "first", Machines: []string{"anchor", "laptop"}},
{Module: "together", Machines: []string{"anchor", "laptop"}}}
if !reflect.DeepEqual(got, want) {
t.Fatalf("kept %+v; want %+v", got, want)
}
}
+272
View File
@@ -0,0 +1,272 @@
package main
import (
"context"
"errors"
"fmt"
"sort"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/secrets"
)
// `secret issued <provision>`: the credential a provider's software issued, taken from the provider and kept as
// the provision's one credential for every consumer (novox/hq ADR 0305, issue 526).
//
// **The value comes from the provider, never from whoever asks.** A module served by its machine's runtime calls
// under the runtime's one bus account, which carries that machine's agents' calls too, so a value handed to a
// verb could be anybody's. This line carries none: it asks the provider's own tool, on the provider's machine, in
// the controller's own name — the subject that names its caller, which the bus lets the controller alone publish
// (issue 365) — with a sealing key made for this one call. The tool answers only the controller, the value sealed
// to that key; the controller opens it, seals it to every holder and forgets it, as the desk's value is (ADR 0277
// §23). Asking again fetches the same value again, and a value every holder already holds is not sealed anew.
//
// **What this trusts: the provider's machine.** The answer is taken from whoever answers the provider's tool
// subject on the provider's machine first. On that machine its runtime serves the subject for every module it
// carries, so anything that runtime runs, or anything else granted that subject there, could answer before the
// provider does and hand the controller a value of its own. ADR 0305 accepts that by trusting the provider's
// machine, as the provider itself is trusted; the bus keeps every other machine from answering it.
// handActIssued is the verb a change of an issued value is recorded under: the mesh working as decided.
const handActIssued = "secret issued"
// issuedAsk is how long the provider's tool is given to answer.
const issuedAsk = 20 * time.Second
// issuedProvider is one assignment of a module whose offer says its software issues the provision's credential.
type issuedProvider struct {
Node, Module, Tool string
}
// issuedTaker is the line's work, with what it reaches handed in: the store, the bus and the sends.
type issuedTaker struct {
// providers are the assignments to ask, and whether any module offers the provision so at all.
providers func(provision string) ([]issuedProvider, bool, error)
ask func(p issuedProvider, args map[string]any) (link.Answer, error)
keep func(provision, provider, value string) (inventory.IssuedKept, error)
record func(act link.HandAct) error
send func(machines []string) error
}
// take asks every provider of the provision for its software's value and keeps each; it answers what it did, in
// words that never carry the value or its fingerprint.
func (t issuedTaker) take(provision string) (string, error) {
providers, offered, err := t.providers(provision)
if err != nil {
return "", err
}
if !offered {
return "", fmt.Errorf("no module offers %q with a credential its software issues (\"credential\": "+
"{\"issued-by\": \"software\", \"tool\": …}, ADR 0305), so there is nothing to take from a provider; "+
"nothing was asked", provision)
}
if len(providers) == 0 {
return fmt.Sprintf("no consumer holds %s from a provider whose software issues it yet, so there is nothing "+
"to seal; a consumer that binds holds a value the mesh made until the provider next asks", provision), nil
}
var said []string
var failed []string
for _, p := range providers {
words, err := t.takeFrom(provision, p)
if err != nil {
failed = append(failed, fmt.Sprintf("%s on %s: %v", p.Module, p.Node, err))
continue
}
said = append(said, words)
}
out := strings.Join(said, "\n")
if len(failed) > 0 {
return out, fmt.Errorf("%s was not taken from every provider:\n %s", provision, strings.Join(failed, "\n "))
}
return out, nil
}
// takeFrom is one provider: asked, opened, kept, recorded and sent when it changed.
func (t issuedTaker) takeFrom(provision string, p issuedProvider) (string, error) {
public, private, err := secrets.Keypair()
if err != nil {
return "", fmt.Errorf("no key could be made to take the value: %w", err)
}
answer, err := t.ask(p, map[string]any{"provision": provision, "seal_to": public})
if err != nil {
return "", fmt.Errorf("its tool %s could not be asked: %w", p.Tool, err)
}
if answer.Error != "" {
return "", fmt.Errorf("its tool %s refused: %s", p.Tool, answer.Error)
}
var given struct {
Sealed string `json:"sealed"`
}
if err := unmarshalAnswer(answer, &given); err != nil || given.Sealed == "" {
return "", fmt.Errorf("its tool %s answered no sealed value", p.Tool)
}
opened, err := secrets.Open(private, given.Sealed)
if err != nil {
// Never the value, never what failed to open: only that it was not sealed to this call.
return "", fmt.Errorf("its tool %s answered a value not sealed to this call; nothing was taken", p.Tool)
}
value := strings.TrimSpace(string(opened))
for i := range opened {
opened[i] = 0
}
if value == "" {
return "", fmt.Errorf("its tool %s answered an empty value; nothing was taken", p.Tool)
}
kept, err := t.keep(provision, p.Node, value)
value = ""
if err != nil {
return "", err
}
holders := make([]string, 0, len(kept.Holders))
for _, h := range kept.Holders {
holders = append(holders, fmt.Sprintf("%s on %s%s", h.ConsumerModule, h.Consumer, asLocal(h.Local)))
}
words := ""
if len(kept.Unsealable) > 0 {
var names []string
for _, h := range kept.Unsealable {
names = append(names, fmt.Sprintf("%s on %s", h.ConsumerModule, h.Consumer))
}
words = fmt.Sprintf("\n not sealed to %s: no sealing key on that machine", strings.Join(names, ", "))
}
if !kept.Changed {
return fmt.Sprintf("%s from %s on %s: every holder already holds the value its software issued (%s); "+
"nothing was sent%s", provision, p.Module, p.Node, strings.Join(holders, ", "), words), nil
}
sealedTo := make([]string, 0, len(kept.Sealed))
for _, h := range kept.Sealed {
sealedTo = append(sealedTo, fmt.Sprintf("%s on %s%s", h.ConsumerModule, h.Consumer, asLocal(h.Local)))
}
recorded := ""
if err := t.record(link.HandAct{Verb: handActIssued, Args: []string{provision},
Why: fmt.Sprintf("%s on %s gave the credential its software issued for %s; sealed to %s", p.Module, p.Node,
provision, strings.Join(sealedTo, ", ")),
Cause: "issued-by-software"}); err != nil {
recorded = fmt.Sprintf("\n this was NOT recorded in the hand-act log, and is done anyway: %v", err)
}
if err := t.send(kept.Machines); err != nil {
return "", fmt.Errorf("%s from %s on %s is sealed to %s and could not be sent: %w; `push --behind` sends it",
provision, p.Module, p.Node, strings.Join(sealedTo, ", "), err)
}
return fmt.Sprintf("%s from %s on %s: the value its software issued is sealed to %s; sent %s, and each of "+
"them starts again on it%s%s", provision, p.Module, p.Node, strings.Join(sealedTo, ", "),
strings.Join(kept.Machines, ", "), words, recorded), nil
}
// issuedProvidersOf are the assignments to ask for a provision: every machine providing it to a pair the mesh
// holds, and on it the assigned module whose offer says its software issues the credential (ADR 0305 §1).
func issuedProvidersOf(ctx context.Context, inv *inventory.Inventory, provision string) ([]issuedProvider, bool, error) {
shelf, err := inv.Catalogue(ctx)
if err != nil {
return nil, false, err
}
if !offersIssued(shelf, provision) {
return nil, false, nil
}
holders, err := inv.HoldersOf(ctx, provision, "")
if err != nil {
return nil, true, err
}
nodes := map[string]bool{}
for _, h := range holders {
nodes[h.Provider] = true
}
names := make([]string, 0, len(nodes))
for n := range nodes {
names = append(names, n)
}
sort.Strings(names)
var out []issuedProvider
for _, node := range names {
assigned, err := inv.Assigned(ctx, node)
if err != nil {
return nil, true, err
}
out = append(out, issuedOn(shelf, node, assigned, provision)...)
}
return out, true, nil
}
// offersIssued is whether any module offers the provision with a credential its software issues.
func offersIssued(shelf map[string]catalogue.Manifest, provision string) bool {
for _, m := range shelf {
if _, issued := m.IssuedCredentialOf(provision); issued {
return true
}
}
return false
}
// issuedOn are the modules assigned to one machine whose offer says their software issues the provision's
// credential, each with the tool that answers it.
func issuedOn(shelf map[string]catalogue.Manifest, node string, assigned []string, provision string) []issuedProvider {
var out []issuedProvider
for _, module := range assigned {
if tool, issued := shelf[module].IssuedCredentialOf(provision); issued {
out = append(out, issuedProvider{Node: node, Module: module, Tool: tool})
}
}
return out
}
// issuedRefusesRotation refuses `rotate` of a provision its provider's software issues (ADR 0305 §9): a value the
// mesh made would replace one the software holds, and every consumer would hold a dead credential until the
// provider next asks.
func issuedRefusesRotation(shelf map[string]catalogue.Manifest, provision string) error {
if !offersIssued(shelf, provision) {
return nil
}
return fmt.Errorf("%s's credential is issued by its provider's software, which the mesh cannot make or "+
"rotate (ADR 0305): change it in that software, and `secret issued %s` takes the value it holds now. "+
"Nothing was rotated", provision, provision)
}
// secretIssued is `secret issued <provision>`, on this controller's stores and bus.
func secretIssued(ctx context.Context, args []string) error {
if len(args) != 1 || args[0] == "" || strings.HasPrefix(args[0], "-") {
return errors.New("secret issued <provision>")
}
provision := args[0]
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
t := issuedTaker{
providers: func(provision string) ([]issuedProvider, bool, error) {
return issuedProvidersOf(ctx, open.inventory, provision)
},
ask: func(p issuedProvider, args map[string]any) (link.Answer, error) {
var answer link.Answer
err := onTheBus(func(conn *nats.Conn) error {
var err error
answer, err = link.AskModuleToolOnAs(ctx, conn, broker.ControllerName, p.Module, p.Tool, p.Node,
args, issuedAsk)
return err
})
return answer, err
},
keep: func(provision, provider, value string) (inventory.IssuedKept, error) {
return open.inventory.KeepIssuedSecret(ctx, provision, provider, value)
},
record: func(act link.HandAct) error {
return onTheBus(func(conn *nats.Conn) error {
_, err := link.RecordHandAct(ctx, conn, act)
return err
})
},
send: func(machines []string) error { return sendTo(ctx, open, machines) },
}
words, err := t.take(provision)
if words != "" {
fmt.Println(words)
}
return err
}
+218
View File
@@ -0,0 +1,218 @@
package main
import (
"encoding/json"
"errors"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/secrets"
)
// The `issued` verb runs `secret issued <provision>` and nothing else, and that line is the one `secret` line
// beside rotate and the desk's that a verb may run (novox/hq ADR 0305 §3, §10, issue 526).
func TestTheIssuedVerbRunsTheOneLineThatCarriesNoValue(t *testing.T) {
argv, err := argvFor("issued", map[string]any{"provision": "plex-api"})
if err != nil {
t.Fatal(err)
}
if strings.Join(argv, " ") != "secret issued plex-api" {
t.Fatalf("the verb composed %q", argv)
}
if _, err := argvFor("issued", map[string]any{}); err == nil {
t.Fatal("the verb ran with no provision")
}
if _, err := argvFor("issued", map[string]any{"provision": "plex-api", "secret": "a value"}); err == nil {
t.Fatal("the verb took a value beside the provision")
}
for _, line := range [][]string{
{"secret", "issued", "plex-api", "--from", "/tmp/x"},
{"secret", "issued", "--from"},
{"secret", "issued"},
{"secret", "accept", "ace", "kometa", "plex-api", "--provider", "ace"},
{"secret", "recover", "ace", "kometa", "plex-api"},
} {
if err := terminalOnly(line); err == nil {
t.Errorf("%q ran through a verb", line)
}
}
if err := terminalOnly([]string{"secret", "issued", "plex-api"}); err != nil {
t.Fatalf("the issued line was refused through its verb: %v", err)
}
found := false
for _, v := range catalogue.ControllerVerbs {
if v.Name == "issued" {
found = true
}
}
if !found {
t.Fatal("the controller's seat does not declare the issued verb")
}
}
// A provider's tool answering the value sealed to the call's key: what a provider does (ADR 0305 §2).
func issuedSealed(t *testing.T, value string) func(issuedProvider, map[string]any) (link.Answer, error) {
return func(_ issuedProvider, args map[string]any) (link.Answer, error) {
key, _ := args["seal_to"].(string)
if key == "" {
t.Fatal("the provider was asked without a key to seal to")
}
if _, carried := args["value"]; carried {
t.Fatal("the ask carried a value")
}
sealed, err := secrets.Seal(key, []byte(value))
if err != nil {
t.Fatal(err)
}
raw, _ := json.Marshal(map[string]string{"sealed": sealed})
return link.Answer{Result: raw}, nil
}
}
type issuedRecorder struct {
kept []string
acts []link.HandAct
sent [][]string
again bool // the value is held by every holder already
}
func (r *issuedRecorder) taker(t *testing.T, ask func(issuedProvider, map[string]any) (link.Answer, error)) issuedTaker {
return issuedTaker{
providers: func(provision string) ([]issuedProvider, bool, error) {
return []issuedProvider{{Node: "ace", Module: "plex", Tool: "plex_issued_credential"}}, true, nil
},
ask: ask,
keep: func(provision, provider, value string) (inventory.IssuedKept, error) {
r.kept = append(r.kept, value)
holders := []inventory.Holder{{Provision: provision, Consumer: "ace", ConsumerModule: "kometa", Provider: provider},
{Provision: provision, Consumer: "ace", ConsumerModule: "tautulli", Provider: provider}}
if r.again {
return inventory.IssuedKept{Holders: holders}, nil
}
return inventory.IssuedKept{Changed: true, Holders: holders, Sealed: holders, Machines: []string{"ace"}}, nil
},
record: func(act link.HandAct) error { r.acts = append(r.acts, act); return nil },
send: func(machines []string) error { r.sent = append(r.sent, machines); return nil },
}
}
// The value the provider's tool sealed to the call is kept, recorded and sent, and the answer never says it.
func TestAnIssuedValueIsTakenSealedFromTheProviderAndSent(t *testing.T) {
var r issuedRecorder
words, err := r.taker(t, issuedSealed(t, "the-servers-own-token")).take("plex-api")
if err != nil {
t.Fatal(err)
}
if len(r.kept) != 1 || r.kept[0] != "the-servers-own-token" {
t.Fatalf("the provider's value was not kept: %d kept", len(r.kept))
}
if len(r.sent) != 1 || strings.Join(r.sent[0], ",") != "ace" {
t.Fatalf("the holders' machines were not sent: %v", r.sent)
}
if len(r.acts) != 1 || r.acts[0].Verb != handActIssued {
t.Fatalf("the change was not recorded in the hand-act log: %+v", r.acts)
}
for _, said := range []string{words, r.acts[0].Why, strings.Join(r.acts[0].Args, " ")} {
if strings.Contains(said, "the-servers-own-token") || strings.Contains(said, inventory.IssuedStamp("the-servers-own-token")) {
t.Fatalf("the value or its fingerprint was said: %q", said)
}
}
if !strings.Contains(words, "kometa on ace") || !strings.Contains(words, "tautulli on ace") {
t.Fatalf("the answer does not name the holders: %q", words)
}
}
// The same value again writes nothing, records nothing and sends nobody (ADR 0305 §5).
func TestAnIssuedValueEveryHolderHoldsSendsNobody(t *testing.T) {
r := issuedRecorder{again: true}
words, err := r.taker(t, issuedSealed(t, "the-servers-own-token")).take("plex-api")
if err != nil {
t.Fatal(err)
}
if len(r.sent) != 0 || len(r.acts) != 0 {
t.Fatalf("an unchanged value was sent or recorded: %v %v", r.sent, r.acts)
}
if !strings.Contains(words, "nothing was sent") {
t.Fatalf("the answer does not say nothing was sent: %q", words)
}
}
// A value not sealed to the call's key is not taken, nor is a refusal, nor an empty answer (ADR 0305 §2).
func TestOnlyAValueSealedToTheCallIsTaken(t *testing.T) {
other, _, _ := secrets.Keypair()
elsewhere, _ := secrets.Seal(other, []byte("somebody's value"))
raw, _ := json.Marshal(map[string]string{"sealed": elsewhere})
plain, _ := json.Marshal(map[string]string{"value": "the-servers-own-token"})
for name, ask := range map[string]func(issuedProvider, map[string]any) (link.Answer, error){
"sealed to another key": func(issuedProvider, map[string]any) (link.Answer, error) { return link.Answer{Result: raw}, nil },
"answered in the clear": func(issuedProvider, map[string]any) (link.Answer, error) { return link.Answer{Result: plain}, nil },
"refused": func(issuedProvider, map[string]any) (link.Answer, error) {
return link.Answer{Error: "the controller alone"}, nil
},
"not reached": func(issuedProvider, map[string]any) (link.Answer, error) {
return link.Answer{}, errors.New("503 no responders")
},
"sealed empty": issuedSealed(t, " "),
} {
var r issuedRecorder
if _, err := r.taker(t, ask).take("plex-api"); err == nil {
t.Errorf("%s: taken", name)
}
if len(r.kept) != 0 || len(r.sent) != 0 {
t.Errorf("%s: something was kept or sent", name)
}
}
}
// A provision no offer says its software issues is refused before anything is asked.
func TestAProvisionNoOfferSaysIsIssuedIsRefused(t *testing.T) {
asked := false
taker := issuedTaker{
providers: func(string) ([]issuedProvider, bool, error) { return nil, false, nil },
ask: func(issuedProvider, map[string]any) (link.Answer, error) {
asked = true
return link.Answer{}, nil
},
}
if _, err := taker.take("postgres-database"); err == nil || !strings.Contains(err.Error(), "ADR 0305") {
t.Fatalf("expected a refusal naming the record, got %v", err)
}
if asked {
t.Fatal("a provider was asked for a provision no offer says its software issues")
}
}
// Which assignments are asked: the modules on the providing machine whose offer says so, with their tool.
func TestTheProvidersAskedAreTheOffersThatSaySo(t *testing.T) {
shelf := map[string]catalogue.Manifest{
"plex": {Module: "plex", Provides: []catalogue.Offer{{Name: "plex-api", Credential: &catalogue.OfferCredential{
IssuedBy: catalogue.IssuedBySoftware, Tool: "plex_issued_credential"}}}},
"jellyfin": {Module: "jellyfin", Provides: []catalogue.Offer{{Name: "plex-api"}}},
}
got := issuedOn(shelf, "ace", []string{"jellyfin", "plex", "kometa"}, "plex-api")
if len(got) != 1 || got[0] != (issuedProvider{Node: "ace", Module: "plex", Tool: "plex_issued_credential"}) {
t.Fatalf("the providers asked: %+v", got)
}
if err := issuedRefusesRotation(shelf, "plex-api"); err == nil || !strings.Contains(err.Error(), "secret issued plex-api") {
t.Fatalf("rotate of an issued provision was not refused with the way to take it: %v", err)
}
if err := issuedRefusesRotation(shelf, "postgres-database"); err != nil {
t.Fatalf("rotate of an ordinary provision was refused: %v", err)
}
}
// A change of an issued value is the mesh working as decided, never a repair (ADR 0305 §8).
func TestAnIssuedValueIsADecisionInTheHandActLog(t *testing.T) {
for _, v := range handActVerbs {
if v.Verb == handActIssued {
if v.Decision == "" {
t.Fatal("secret issued counts as a repair")
}
return
}
}
t.Fatal("secret issued is not in the hand-act table")
}
+160
View File
@@ -0,0 +1,160 @@
package main
import (
"encoding/json"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
// novox/hq issue 485: the mesh records every send — sender, lease epoch, generation, modules — and only `plan`
// said it. `status` says per machine when it was last sent a declaration and by whom, in one line; `node show`
// says the last send in full and what the machine answered; and `status --json` carries it as fields, since
// the store's reader may not read the controller's tables and these verbs are the only window onto them.
func aLastSend(answer inventory.SendAnswer) inventory.Send {
sent := time.Date(2026, 10, 11, 18, 40, 5, 0, time.Local)
return inventory.Send{NodeName: "laptop", Sequence: 12, Epoch: 57,
Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12",
Modules: []string{"docker", "sudo"}, SentAt: sent, Recorded: true, Answer: answer}
}
func TestStatusSaysEachMachinesLastSendInOneLine(t *testing.T) {
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.Local)
asked := answers{
nodes: []inventory.Node{{Name: "anchor", LastSeen: time.Now()}, {Name: "laptop", LastSeen: time.Now()}},
lastSent: map[string]inventory.Send{"laptop": aLastSend(inventory.SendAnswer{
Outcome: inventory.OutcomeApplied, At: &at})},
}
shown := printed(t, func() error { return printStatus(asked) })
var laptop, anchor []string
for _, line := range strings.Split(shown, "\n") {
fields := strings.Fields(line)
if len(fields) > 0 && fields[0] == "laptop" {
laptop = append(laptop, line)
}
if len(fields) > 0 && fields[0] == "anchor" {
anchor = append(anchor, line)
}
}
if len(laptop) != 1 || !strings.Contains(laptop[0], "2026-10-11 18:40") ||
!strings.Contains(laptop[0], "by a one-shot push by jochen at a shell on anchor") ||
!strings.Contains(laptop[0], "generation 40") || !strings.Contains(laptop[0], "applied") {
t.Fatalf("status does not say laptop's last send in one line:\n%s", shown)
}
if len(anchor) != 1 || !strings.Contains(anchor[0], "no send recorded") {
t.Fatalf("status does not say anchor has no send recorded, in one line:\n%s", shown)
}
}
func TestStatusSaysTheLastSendsCouldNotBeRead(t *testing.T) {
asked := answers{nodes: []inventory.Node{{Name: "laptop", LastSeen: time.Now()}},
lastSentUnread: "the store went away"}
if shown := printed(t, func() error { return printStatus(asked) }); !strings.Contains(shown, "the store went away") {
t.Fatalf("a record of sends that could not be read is not said:\n%s", shown)
}
if asked.well() {
t.Error("a mesh whose sends could not be read is called well")
}
}
func TestNodeShowSaysTheLastSendInFull(t *testing.T) {
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.Local)
refused := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeRefused,
Refused: "unknown field \"generation\"\nsecond line", At: &at})), "\n")
for _, want := range []string{"sequence 12", "2026-10-11 18:40:05", "a one-shot push by jochen at a shell on anchor",
"lease epoch 57", "generation 40", "docker, sudo", "refused", "unknown field \"generation\""} {
if !strings.Contains(refused, want) {
t.Errorf("node show's last send does not say %q:\n%s", want, refused)
}
}
if applied := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeApplied,
At: &at})), "\n"); !strings.Contains(applied, "applied it at 2026-10-11 18:41:00") {
t.Errorf("an applied send is not said applied:\n%s", applied)
}
if waiting := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{})), "\n"); !strings.Contains(waiting,
"not reported on it yet") {
t.Errorf("a send not reported on is not said so:\n%s", waiting)
}
stale := aLastSend(inventory.SendAnswer{})
when := at
stale.RefusedAt, stale.RefusedApplied = &when, 44
if s := strings.Join(lastSendLines(stale), "\n"); !strings.Contains(s, "refused it at 2026-10-11 18:41:00") ||
!strings.Contains(s, "generation 44") {
t.Errorf("a send refused for its generation is not said so:\n%s", s)
}
}
func TestStatusJSONCarriesEachMachinesLastSend(t *testing.T) {
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.UTC)
asked := answers{nodes: []inventory.Node{{Name: "anchor"}, {Name: "laptop"}},
lastSent: map[string]inventory.Send{"laptop": aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeRefused,
Refused: "unknown field", At: &at})}}
body, err := statusAsJSON(asked)
if err != nil {
t.Fatal(err)
}
var got struct {
LastSent []map[string]any `json:"lastSent"`
}
if err := json.Unmarshal(body, &got); err != nil {
t.Fatal(err)
}
if len(got.LastSent) != 1 {
t.Fatalf("status --json carries %d last sends, want laptop's only:\n%s", len(got.LastSent), body)
}
s := got.LastSent[0]
if s["node"] != "laptop" || s["sequence"] != float64(12) || s["epoch"] != float64(57) ||
s["generation"] != float64(40) || s["sender"] != "a one-shot push by jochen at a shell on anchor" ||
s["digest"] != "d12" || s["sentAt"] == nil || s["answer"] != "refused" || s["refused"] != "unknown field" ||
s["answeredAt"] == nil {
t.Fatalf("laptop's last send reads %v", s)
}
if modules, _ := s["modules"].([]any); len(modules) != 2 || modules[0] != "docker" {
t.Fatalf("the modules it named read %v", s["modules"])
}
}
// And through the store: a send recorded and refused by the machine is what `node show` and `status --json`
// read back.
func TestNodeShowAndStatusReadTheLastSendFromTheStore(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
record, err := open.inventory.NodeByName(ctx, "laptop")
if err != nil {
t.Fatal(err)
}
if err := open.inventory.RecordSentWith(ctx, record.ID, "d12", nil, 57, 40, inventory.Send{Sequence: 12,
Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12",
Modules: []string{"docker", "sudo"}}); err != nil {
t.Fatal(err)
}
if _, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeRefused,
Declared: "d12", Refused: "unknown field \"generation\""}); err != nil {
t.Fatal(err)
}
shown := printed(t, func() error { return showNode(ctx, open, "laptop") })
for _, want := range []string{"a one-shot push by jochen at a shell on anchor", "lease epoch 57", "generation 40",
"docker, sudo", "unknown field \"generation\""} {
if !strings.Contains(shown, want) {
t.Errorf("node show laptop does not say %q:\n%s", want, shown)
}
}
body := printed(t, func() error { return statusFor(ctx, open, true) })
var got struct {
LastSent []struct {
Node string `json:"node"`
Sender string `json:"sender"`
Answer string `json:"answer"`
} `json:"lastSent"`
}
if err := json.Unmarshal([]byte(body), &got); err != nil {
t.Fatalf("%v:\n%s", err, body)
}
if len(got.LastSent) != 1 || got.LastSent[0].Node != "laptop" || got.LastSent[0].Answer != "refused" ||
got.LastSent[0].Sender != "a one-shot push by jochen at a shell on anchor" {
t.Fatalf("status --json's last sends read %+v", got.LastSent)
}
}
+222
View File
@@ -0,0 +1,222 @@
package main
import (
"errors"
"fmt"
"sort"
"strings"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
)
// A module assigned to a machine and left out of its declaration is said (novox/hq issue 380).
//
// **An omission is a finding, never a refusal of the push** (ADR 0163, rule 6): the machine is sent everything
// else, and the module's held things are kept. Until issue 380 the only place it showed was `plan`: nfs-server was
// assigned to the home server for days, every send left it out for a setting nobody gave, and the operator believed
// it ran. So the self-check (D1) judges every machine as the next send would (Resolution.LeftOutBecause, the send's
// own judgement) and raises a condition on that machine for each module it leaves out:
//
// - a setting nobody gave (catalogue.UnsetSettingError) is the operator's to give: kind needs-operator, naming
// the module, the setting and the command that sets it;
// - any other cause is said as a module not working is: kind left-out, a warning with the reason as evidence.
//
// Never urgent and never escalated by age (as ADR 0283 decision 5): nothing the machine ran was undone. It clears
// on the first run that no longer finds it — the module composed again, or no longer assigned. `status` and
// `node show` list the same modules under "assigned, not applied" with the same reason.
//
// A module left out because its stored manifest has a key this controller does not know is not raised here: the
// catalogue's own unknown-field condition says it once for the whole mesh (ADR 0262); it is still listed.
const (
// probeLeftOutID is the self-check's probe that raises and clears these conditions: D1, which already
// resolves every machine (probeDeclarations), so the judgement costs no resolution of its own.
probeLeftOutID = "D1"
// kindLeftOut is a module left out for any cause but a setting nobody gave; it is also every such condition's
// token, whichever its kind, so a cause that changes is the same condition said anew.
kindLeftOut = "left-out"
)
// leftOutModule is one module of a machine's set that its declaration leaves out, and why.
type leftOutModule struct {
Module string
// Setting is the setting nobody gave, when that is the cause.
Setting string
// Why is the declaration's own reason, whole.
Why string
// Unread is a stored manifest this controller cannot read whole (said by the catalogue's condition).
Unread bool
}
// leftOutOf is every module of a machine's resolution that a push would leave out, sorted. Pure: the judgement
// the push makes (catalogue.Resolution.LeftOutBecause), nothing allocated.
func leftOutOf(plan catalogue.Resolution, settings catalogue.SettingsBy, adopted bool) []leftOutModule {
because := plan.LeftOutBecause(settings, adopted)
out := make([]leftOutModule, 0, len(because))
for module, why := range because {
l := leftOutModule{Module: module, Why: oneLine(why.Error())}
var unset *catalogue.UnsetSettingError
var unread *catalogue.UnreadManifestError
switch {
case errors.As(why, &unset):
l.Setting = unset.Setting
case errors.As(why, &unread):
l.Unread = true
}
out = append(out, l)
}
sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module })
return out
}
// settingCommand is the command that gives a module's setting on one machine.
func settingCommand(module, setting, node string) string {
return fmt.Sprintf("`settings set %s '{%q: …}' --node %s`", module, setting, node)
}
// reason is why a module is left out, as `status`, `node show` and the condition's summary say it: for a setting
// nobody gave, the setting and the command that gives it; otherwise the declaration's own words.
func (l leftOutModule) reason(node string) string {
if l.Setting != "" {
return fmt.Sprintf("nothing sets its setting %q, so every send leaves it out of its declaration — %s sets it", l.Setting,
settingCommand(l.Module, l.Setting, node))
}
return "every send leaves it out of its declaration: " + l.Why
}
// leftOutObservations are the conditions a machine's left-out modules raise, one each.
func leftOutObservations(node string, left []leftOutModule) []conditions.Observation {
var out []conditions.Observation
for _, l := range left {
if l.Unread {
continue
}
out = append(out, leftOutObservation(node, l))
}
return out
}
// leftOutObservation is one module left out of one machine's declaration: needs-operator for a setting nobody
// gave, left-out otherwise; a warning either way, the operator's to resolve.
func leftOutObservation(node string, l leftOutModule) conditions.Observation {
o := conditions.Observation{Scope: conditions.ScopeModule, ID: l.Module + "." + node, Token: kindLeftOut,
Kind: kindLeftOut, Machine: node, Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("%s is assigned to %s and not applied: %s", l.Module, node, l.reason(node)),
Said: l.Why}
w := leftOutWords(l.Module, node, l.Setting)
if l.Setting != "" {
o.Kind = kindNeedsOperator
} else {
// The words of why may name a path or an address, which the operator's channel withholds: the
// summary sends them to the evidence, and `plan` says them in full.
o.Summary = fmt.Sprintf("%s is assigned to %s and not applied: every send leaves it out of its declaration, "+
"because what is set for it does not fit its manifest — the evidence and `plan %s` say why", l.Module, node, node)
}
o.Headline, o.Explanation, o.Needs, o.Resolved = w.Headline, w.Explanation, w.Needs, w.Resolved
return o
}
// leftOutWords is what the operator reads of a module left out (ADR 0253): plain, the act named.
func leftOutWords(module, node, setting string) words {
w := words{
Headline: fmt.Sprintf("%s is not applied on %s", module, node),
Explanation: fmt.Sprintf("%s is assigned to %s, and every send to %s leaves it out of the declaration because what "+
"is set for it does not fit its manifest. Nothing of it changes there; the rest of %s is sent as usual.",
module, node, node, node),
Needs: fmt.Sprintf("read why in the details, then change what is set for %s or unassign it.", module),
Resolved: fmt.Sprintf("%s on %s is no longer left out", module, node),
}
if setting != "" {
w.Explanation = fmt.Sprintf("%s is assigned to %s, and every send to %s leaves it out of the declaration because "+
"nothing sets its setting %s. Nothing of it runs there until it is set; the rest of %s is sent as usual.",
module, node, node, setting, node)
w.Needs = fmt.Sprintf("set %s for %s on %s, or approve it when it is proposed to you.", setting, module, node)
// A setting's name that is not plain (a dotted key) is in the summary instead.
if _, ok := conditions.PlainWords(w, node); !ok {
w.Explanation = fmt.Sprintf("%s is assigned to %s, and every send to %s leaves it out of the declaration because a "+
"setting it needs is not set. Nothing of it runs there until it is set; the details name it.",
module, node, node)
w.Needs = fmt.Sprintf("set what %s needs on %s; the details name the setting.", module, node)
}
}
return w
}
// notAppliedLines is a machine's "assigned, not applied" as `node show` prints it: one line a module, with the
// reason its condition says.
func notAppliedLines(node string, left []leftOutModule) []string {
if len(left) == 0 {
return nil
}
lines := []string{"", " assigned, not applied:"}
for _, l := range left {
lines = append(lines, fmt.Sprintf(" %-22s %s", l.Module, l.reason(node)))
}
return lines
}
// machineNotApplied is one module assigned to a machine and left out of its declaration, in `status --json`.
type machineNotApplied struct {
Node string `json:"node"`
Module string `json:"module"`
Setting string `json:"setting,omitempty"`
Reason string `json:"reason"`
// UnreadManifest is a module left out because this controller cannot read its manifest whole: it raises no
// condition of its own on the machine, since the catalogue's unknown-field condition says it once (ADR 0262).
UnreadManifest bool `json:"unread-manifest,omitempty"`
}
// notApplied is every machine's left-out modules, in a stated order, as `status --json` carries them.
func notApplied(left map[string][]leftOutModule) []machineNotApplied {
names := make([]string, 0, len(left))
for name := range left {
names = append(names, name)
}
sort.Strings(names)
var out []machineNotApplied
for _, name := range names {
for _, l := range left[name] {
out = append(out, machineNotApplied{Node: name, Module: l.Module, Setting: l.Setting, Reason: l.reason(name),
UnreadManifest: l.Unread})
}
}
return out
}
// printNotApplied is status's "assigned, not applied", every machine's.
func printNotApplied(left map[string][]leftOutModule) {
rows := notApplied(left)
if len(rows) == 0 {
return
}
unread := 0
for _, r := range rows {
if r.UnreadManifest {
unread++
}
}
// Which raise a condition is said, not implied (review of #223): one whose manifest this controller cannot
// read is listed here and raises none of its own on the machine — the catalogue's condition says it.
raises := "each raises a condition on its machine"
switch {
case unread == len(rows):
raises = "none raises a condition on its machine: this controller cannot read their manifests, which the " +
"catalogue's own condition says"
case unread > 0:
raises += fmt.Sprintf(", except the %d whose manifest this controller cannot read, which the catalogue's own "+
"condition says", unread)
}
fmt.Printf("%d module(s) assigned, not applied — every send leaves them out of their declaration; %s:\n",
len(rows), raises)
for _, r := range rows {
fmt.Printf(" %-12s %-22s %s\n", r.Node, r.Module, r.Reason)
}
fmt.Println()
}
// isLeftOutCondition is whether a condition is a module left out of its declaration, which the machine's health
// statements neither raise nor clear: by its token, which every one carries whatever its kind.
func isLeftOutCondition(c conditions.Condition) bool {
return c.Subject.Scope == conditions.ScopeModule && strings.HasSuffix(c.Key, "."+kindLeftOut)
}
+259
View File
@@ -0,0 +1,259 @@
package main
import (
"context"
"encoding/json"
"os"
"slices"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// novox/hq issue 380: nfs-server was assigned to the home server for days and every send left it out — "nfs-server
// has a file that says ${setting:shares}, and nothing sets shares for it" — and nothing but `plan` said so. The
// manifest is the catalogue's own at the commit that added it (mesh-catalog 48fba44), which still says
// ${setting:shares}; the reason below is the one the live push printed.
// leftOutNFS is the resolution of a machine assigned that nfs-server, with the settings given.
func leftOutNFS(t *testing.T) catalogue.Resolution {
t.Helper()
raw, err := os.ReadFile("testdata/left-out/nfs-server.json")
if err != nil {
t.Fatal(err)
}
m, err := catalogue.ParseManifest(raw)
if err != nil {
t.Fatal(err)
}
return catalogue.Resolution{Modules: []catalogue.Manifest{m}}
}
// sharesGiven is the operator's setting for it on the machine.
func sharesGiven(value string) catalogue.SettingsBy {
return catalogue.SettingsBy{"nfs-server": {{From: "anchor", Values: map[string]any{"shares": value}}}}
}
func TestAModuleLeftOutForASettingNobodyGaveNeedsTheOperatorNamingTheSettingAndTheCommand(t *testing.T) {
left := leftOutOf(leftOutNFS(t), nil, false)
if len(left) != 1 || left[0].Module != "nfs-server" || left[0].Setting != "shares" {
t.Fatalf("left out: %+v", left)
}
if !strings.Contains(left[0].Why, `nothing sets "shares" for it`) {
t.Fatalf("the reason is not the push's own: %s", left[0].Why)
}
obs := leftOutObservations("anchor", left)
if len(obs) != 1 {
t.Fatalf("raised %+v", obs)
}
o := obs[0]
if o.Key() != "module.nfs-server.anchor.left-out" || o.Kind != kindNeedsOperator || o.Machine != "anchor" ||
o.Severity != conditions.Warning || o.Resolver != conditions.ResolverOperator {
t.Fatalf("raised %s as %s, %s, by %s, on %q", o.Key(), o.Kind, o.Severity, o.Resolver, o.Machine)
}
for _, want := range []string{"nfs-server", "anchor", `"shares"`, "`settings set nfs-server '{\"shares\": …}' --node anchor`"} {
if !strings.Contains(o.Summary, want) {
t.Errorf("the summary does not name %s: %s", want, o.Summary)
}
}
if o.Said != left[0].Why {
t.Errorf("the evidence is not the reason the send gives: %s", o.Said)
}
plainExample(t, o, "nfs-server is not applied on anchor",
"Needs you: set shares for nfs-server on anchor, or approve it when it is proposed to you. nfs-server is assigned to "+
"anchor, and every send to anchor leaves it out of the declaration because nothing sets its setting shares. "+
"Nothing of it runs there until it is set; the rest of anchor is sent as usual.")
}
func TestAModuleLeftOutForAnotherCauseIsAWarningWithTheReasonAsEvidence(t *testing.T) {
// A setting stored that its manifest can no longer take: one with a line break (issue 339).
left := leftOutOf(leftOutNFS(t), sharesGiven("library=/srv/library\nmedia=/srv/media"), false)
if len(left) != 1 || left[0].Setting != "" {
t.Fatalf("left out: %+v", left)
}
obs := leftOutObservations("anchor", left)
if len(obs) != 1 {
t.Fatalf("raised %+v", obs)
}
o := obs[0]
if o.Key() != "module.nfs-server.anchor.left-out" || o.Kind != kindLeftOut || o.Severity != conditions.Warning {
t.Fatalf("raised %s as %s, %s", o.Key(), o.Kind, o.Severity)
}
if !strings.Contains(o.Said, "holds a line break") || strings.Contains(o.Summary, "/srv/") {
t.Fatalf("the reason is not the evidence, or the summary carries it to the channel:\n%s\n%s", o.Summary, o.Said)
}
plainExample(t, o, "nfs-server is not applied on anchor",
"Needs you: read why in the details, then change what is set for nfs-server or unassign it. nfs-server is assigned to "+
"anchor, and every send to anchor leaves it out of the declaration because what is set for it does not fit "+
"its manifest. Nothing of it changes there; the rest of anchor is sent as usual.")
}
// The self-check raises it, keeps it a warning however long it stands, and clears it when the module composes
// again or is no longer assigned; a machine's health statement neither clears nor raises it.
func TestALeftOutModulesConditionClearsWhenItComposesAgainOrIsUnassigned(t *testing.T) {
plan, settings := leftOutNFS(t), catalogue.SettingsBy(nil)
withProbes(t, probe{ID: probeLeftOutID, Asserts: "the test's", Kind: kindLeftOut, Phase: 1,
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
return leftOutObservations("anchor", leftOutOf(plan, settings, false)), nil
}})
store := conditions.NewInMemory()
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
defer k.Close(context.Background())
d := &doctor{keeper: k, teller: &conditions.Told{}, host: "anchor"}
key := "module.nfs-server.anchor.left-out"
openOnes := func() map[string]conditions.Condition {
t.Helper()
open, err := k.Open(t.Context())
if err != nil {
t.Fatal(err)
}
out := map[string]conditions.Condition{}
for _, c := range open {
out[c.Key] = c
}
return out
}
d.runOnce(t.Context(), "a test")
c, raised := openOnes()[key]
if !raised || c.Kind != kindNeedsOperator || c.Severity != conditions.Warning {
t.Fatalf("a module left out raised %+v", openOnes())
}
// A statement from the machine that says nothing of it — the module runs nothing there — leaves it open.
if err := judgeModuleHealth(t.Context(), nil, k, "anchor", map[string][]inventory.ResourceHealth{}, nil,
time.Now().Add(72*time.Hour)); err != nil {
t.Fatal(err)
}
if _, still := openOnes()[key]; !still {
t.Fatal("a health statement that says nothing of the module cleared its left-out condition")
}
d.runOnce(t.Context(), "a test")
if c := openOnes()[key]; c.Severity != conditions.Warning {
t.Fatalf("after a health statement and days, it is %+v", openOnes())
}
// The setting given: it composes, and the condition clears.
settings = sharesGiven("library=/srv/library")
d.runOnce(t.Context(), "a test")
if _, still := openOnes()[key]; still {
t.Fatalf("composed again, still open: %+v", openOnes())
}
// Left out again, then unassigned: no longer in the machine's set, and it clears.
settings = nil
d.runOnce(t.Context(), "a test")
if _, raised := openOnes()[key]; !raised {
t.Fatal("left out again and not raised")
}
plan = catalogue.Resolution{}
d.runOnce(t.Context(), "a test")
if _, still := openOnes()[key]; still {
t.Fatalf("unassigned, still open: %+v", openOnes())
}
}
func TestTheLeftOutProbeIsInTheRegistry(t *testing.T) {
for _, p := range probeRegistry {
if p.ID == probeLeftOutID {
if p.run == nil || !slices.Contains(p.Raises, kindLeftOut) || !slices.Contains(p.Raises, kindNeedsOperator) {
t.Fatalf("%+v", p)
}
return
}
}
t.Fatalf("no probe %s: a module left out is said nowhere", probeLeftOutID)
}
// status and node show list it under "assigned, not applied" with the reason the condition says, and status is
// not well while there is one.
func TestStatusAndNodeListAModuleAssignedAndNotApplied(t *testing.T) {
left := map[string][]leftOutModule{"anchor": leftOutOf(leftOutNFS(t), nil, false)}
reason := leftOutObservations("anchor", left["anchor"])[0].Summary
asked := answers{leftOut: left}
if asked.well() {
t.Fatal("a mesh with a module assigned and not applied is called well")
}
shown := printed(t, func() error { return printStatus(asked) })
if !strings.Contains(shown, "1 module(s) assigned, not applied") || !strings.Contains(shown, "nfs-server") ||
!strings.Contains(shown, "`settings set nfs-server '{\"shares\": …}' --node anchor` sets it") {
t.Fatalf("status says:\n%s", shown)
}
if !strings.Contains(reason, left["anchor"][0].reason("anchor")) {
t.Fatalf("status and the condition say different reasons:\n%s\n%s", shown, reason)
}
body, err := statusAsJSON(asked)
if err != nil {
t.Fatal(err)
}
var doc struct {
NotApplied []machineNotApplied `json:"not-applied"`
}
if err := json.Unmarshal(body, &doc); err != nil || len(doc.NotApplied) != 1 ||
doc.NotApplied[0].Setting != "shares" || doc.NotApplied[0].Node != "anchor" {
t.Fatalf("status --json: %v %s", err, body)
}
lines := strings.Join(notAppliedLines("anchor", left["anchor"]), "\n")
if !strings.Contains(lines, "assigned, not applied:") || !strings.Contains(lines, "nfs-server") ||
!strings.Contains(lines, `nothing sets its setting "shares"`) {
t.Fatalf("node show says:\n%s", lines)
}
}
// The skip is this probe's alone (review of #223): a part waiting for the operator (ADR 0283) keeps its own
// needs-operator condition, `module.<m>.<node>.needs-operator`, which still clears on the first statement that no
// longer names it — beside a left-out condition on the same machine, which stays.
func TestAWaitsNeedsOperatorStillClearsWhenItsStatementStopsNamingItBesideALeftOutOne(t *testing.T) {
store := conditions.NewInMemory()
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
defer k.Close(context.Background())
left := leftOutObservations("anchor", leftOutOf(leftOutNFS(t), nil, false))
if err := k.Reconcile(t.Context(), probeLeftOutID, left); err != nil {
t.Fatal(err)
}
waiting := map[string][]inventory.ResourceHealth{"notes": {{Module: "notes", Resource: "server", State: link.StateWaiting,
Waits: []inventory.Wait{{Setting: "domain", What: "the domain it serves"}}}}}
if err := judgeModuleHealth(t.Context(), nil, k, "anchor", waiting, map[string]int{"notes": 2}, time.Now()); err != nil {
t.Fatal(err)
}
waitKey, leftKey := needsOperatorKey("notes", "anchor"), "module.nfs-server.anchor.left-out"
open := func() map[string]conditions.Condition {
t.Helper()
list, err := k.Open(t.Context())
if err != nil {
t.Fatal(err)
}
out := map[string]conditions.Condition{}
for _, c := range list {
out[c.Key] = c
}
return out
}
if c, raised := open()[waitKey]; !raised || c.Kind != kindNeedsOperator {
t.Fatalf("the wait raised %+v", open())
}
if c := open()[leftKey]; c.Kind != kindNeedsOperator {
t.Fatalf("the left-out condition is not open as needs-operator: %+v", open())
}
// D1 runs again and still finds nfs-server left out: its reconcile clears only what D1 raised, never the wait,
// which the machine's statement raised.
if err := k.Reconcile(t.Context(), probeLeftOutID, left); err != nil {
t.Fatal(err)
}
if _, still := open()[waitKey]; !still {
t.Fatalf("D1's reconcile cleared the wait's needs-operator: %+v", open())
}
if err := judgeModuleHealth(t.Context(), nil, k, "anchor", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil {
t.Fatal(err)
}
if _, still := open()[waitKey]; still {
t.Fatalf("the statement no longer names the wait, and its needs-operator is still open: %+v", open())
}
if _, still := open()[leftKey]; !still {
t.Fatalf("the left-out condition was cleared by a health statement: %+v", open())
}
}
+3 -1
View File
@@ -147,9 +147,11 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
} }
standing := map[string]conditions.Condition{} standing := map[string]conditions.Condition{}
for _, c := range open { for _, c := range open {
// A module left out of the composition is the self-check's to raise and clear, never a statement's
// (novox/hq issue 380): its needs-operator is not cleared for not being in what the machine runs.
if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound || if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound ||
c.Kind == kindNeedsOperator) && c.Kind == kindNeedsOperator) &&
c.Subject.Machine == node { c.Subject.Machine == node && !isLeftOutCondition(c) {
standing[c.Key] = c standing[c.Key] = c
} }
} }
+27 -2
View File
@@ -44,7 +44,7 @@ func nodeCommand(ctx context.Context, args []string) error {
if len(args) != 2 { if len(args) != 2 {
return errors.New("node show <name>") return errors.New("node show <name>")
} }
return showNode(ctx, inv, args[1]) return showNode(ctx, open, args[1])
case "add": case "add":
return addNode(ctx, inv, args[1:]) return addNode(ctx, inv, args[1:])
@@ -787,7 +787,8 @@ func roughly(d time.Duration) string {
// //
// It is also where "what should it be configured as" is read. The same line that gates an // It is also where "what should it be configured as" is read. The same line that gates an
// assignment carries `card1-DP-1`, and a person composing settings for that machine needs it. // assignment carries `card1-DP-1`, and a person composing settings for that machine needs it.
func showNode(ctx context.Context, inv *inventory.Inventory, name string) error { func showNode(ctx context.Context, stored *stores, name string) error {
inv := stored.inventory
node, err := inv.NodeByName(ctx, name) node, err := inv.NodeByName(ctx, name)
if err != nil { if err != nil {
return err return err
@@ -803,6 +804,19 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
if err := showMode(ctx, inv, node); err != nil { if err := showMode(ctx, inv, node); err != nil {
return err return err
} }
// What it was last sent, by whom, under which epoch, from which generation, naming which modules, and what
// it answered (novox/hq issue 485): the record of sends is read through this verb, never the store's reader.
// Unreadable is said, not taken for none.
switch last, found, err := inv.LastSend(ctx, name); {
case err != nil:
fmt.Printf(" what it was last sent could NOT be read: %v\n", err)
case !found:
fmt.Printf(" last sent no send recorded since the mesh began keeping them\n")
default:
for _, line := range lastSendLines(last) {
fmt.Println(line)
}
}
// Whom agents run as here, and whether that account can become root without a person (ADR 0266). // Whom agents run as here, and whether that account can become root without a person (ADR 0266).
for _, line := range agentAccountLines(ctx, inv, node) { for _, line := range agentAccountLines(ctx, inv, node) {
fmt.Println(line) fmt.Println(line)
@@ -889,6 +903,17 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
if len(assigned) > 0 { if len(assigned) > 0 {
fmt.Printf("\n assigned: %s\n", strings.Join(assigned, ", ")) fmt.Printf("\n assigned: %s\n", strings.Join(assigned, ", "))
} }
// And which of them a push leaves out, and why (novox/hq issue 380): judged as the push judges it, the same
// reason its condition says. Not computable is said, never read as "all applied".
plan, settings, err := planFor(ctx, stored, name)
switch {
case err != nil:
fmt.Printf("\n whether a send leaves any assigned module out is NOT known: %s\n", oneLine(err.Error()))
default:
for _, line := range notAppliedLines(name, leftOutOf(plan, settings, node.Adopted)) {
fmt.Println(line)
}
}
return nil return nil
} }
+5 -4
View File
@@ -17,8 +17,8 @@ import (
// A delivery over its budget is loud (novox/hq ADR 0282 decision 7, issue 382): the self-check reads // A delivery over its budget is loud (novox/hq ADR 0282 decision 7, issue 382): the self-check reads
// mesh-delivery's `times` and raises the warning `delivery.<class>.over-budget` when the newest delivery of a // mesh-delivery's `times` and raises the warning `delivery.<class>.over-budget` when the newest delivery of a
// class whose delivery time is known took longer than its class's budget — five minutes for a leaf module, ten // class whose delivery time is known took longer than its class's budget — three minutes for a leaf module (ADR
// for a core module — naming the delivery and its longest phase. It clears when the next delivery of that class // 0304 decision 4), ten for a core module — naming the delivery and its longest phase. It clears when the next delivery of that class
// lands within its budget. Measurement only: the condition says, it never holds or acts on a delivery. // lands within its budget. Measurement only: the condition says, it never holds or acts on a delivery.
// probeBudgetsID is the probe that reads the delivery times. // probeBudgetsID is the probe that reads the delivery times.
@@ -27,8 +27,9 @@ const probeBudgetsID = "D16"
// kindOverBudget is what a delivery over its class's budget raises. // kindOverBudget is what a delivery over its class's budget raises.
const kindOverBudget = "over-budget" const kindOverBudget = "over-budget"
// deliveryBudgets are the budgets by class (ADR 0282 decision 1); the probe raises nothing for another class. // deliveryBudgets are the budgets by class (ADR 0282 decision 1, the leaf's as ADR 0304 decision 4 changed it);
var deliveryBudgets = map[string]time.Duration{inventory.ClassLeaf: 5 * time.Minute, inventory.ClassCore: 10 * time.Minute} // the probe raises nothing for another class.
var deliveryBudgets = map[string]time.Duration{inventory.ClassLeaf: 3 * time.Minute, inventory.ClassCore: 10 * time.Minute}
// timesAnswer is what mesh-delivery's `times` answers, as far as the probe reads it. // timesAnswer is what mesh-delivery's `times` answers, as far as the probe reads it.
type timesAnswer struct { type timesAnswer struct {
+5 -4
View File
@@ -10,11 +10,12 @@ import (
"github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/inventory"
) )
// A leaf delivery of 5 minutes 10 seconds raises delivery.leaf.over-budget naming its longest phase; one of // A leaf delivery of 3 minutes 10 seconds raises delivery.leaf.over-budget naming its longest phase; one of
// exactly five minutes, a core one of nine and a class without a budget raise nothing (ADR 0282 decision 7). // exactly three minutes, a core one of nine and a class without a budget raise nothing (ADR 0282 decision 7,
// the leaf's budget as ADR 0304 decision 4 changed it).
func TestADeliveryOverItsBudgetIsLoud(t *testing.T) { func TestADeliveryOverItsBudgetIsLoud(t *testing.T) {
a := &timesAnswer{Classes: []timesClass{ a := &timesAnswer{Classes: []timesClass{
{Class: inventory.ClassLeaf, Latest: &timesLatest{ID: "novox/mesh-catalog@abc", TookMS: (5*time.Minute + 10*time.Second).Milliseconds(), {Class: inventory.ClassLeaf, Latest: &timesLatest{ID: "novox/mesh-catalog@abc", TookMS: (3*time.Minute + 10*time.Second).Milliseconds(),
Longest: "judgement 2m40s"}}, Longest: "judgement 2m40s"}},
{Class: inventory.ClassCore, Latest: &timesLatest{ID: "novox/mesh-controller@def", TookMS: (9 * time.Minute).Milliseconds(), {Class: inventory.ClassCore, Latest: &timesLatest{ID: "novox/mesh-controller@def", TookMS: (9 * time.Minute).Milliseconds(),
Longest: "build 1m"}}, Longest: "build 1m"}},
@@ -30,7 +31,7 @@ func TestADeliveryOverItsBudgetIsLoud(t *testing.T) {
t.Fatalf("the condition names its delivery and longest phase: %s — %s", o.Key(), o.Summary) t.Fatalf("the condition names its delivery and longest phase: %s — %s", o.Key(), o.Summary)
} }
// The next leaf delivery within its budget clears it: the probe raises nothing for the class. // The next leaf delivery within its budget clears it: the probe raises nothing for the class.
a.Classes[0].Latest = &timesLatest{ID: "novox/mesh-catalog@ghi", TookMS: (5 * time.Minute).Milliseconds()} a.Classes[0].Latest = &timesLatest{ID: "novox/mesh-catalog@ghi", TookMS: (3 * time.Minute).Milliseconds()}
if obs := overBudgetObservations(a); len(obs) != 0 { if obs := overBudgetObservations(a); len(obs) != 0 {
t.Fatalf("a delivery of exactly its budget is within it: %+v", obs) t.Fatalf("a delivery of exactly its budget is within it: %+v", obs)
} }
+8
View File
@@ -231,6 +231,14 @@ var plainWordings = map[string]func(conditions.Observation) words{
w := needsOperatorWords(orModule(module), node, nil) w := needsOperatorWords(orModule(module), node, nil)
return w return w
}), }),
kindLeftOut: worded(func(o conditions.Observation) words {
// The observation carries its own words (novox/hq issue 380); these are its kind's alone.
module := ""
if o.Scope == conditions.ScopeModule && o.Machine != "" {
module = strings.TrimSuffix(o.ID, "."+o.Machine)
}
return leftOutWords(orModule(module), machineOr(o, "a machine"), "")
}),
kindProviderFailing: worded(func(o conditions.Observation) words { kindProviderFailing: worded(func(o conditions.Observation) words {
thing, consumer := conditions.ThingWords(o), idPart(o, 2) thing, consumer := conditions.ThingWords(o), idPart(o, 2)
if consumer == "" { if consumer == "" {
+7
View File
@@ -278,6 +278,13 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
plans = append(plans, recent...) plans = append(plans, recent...)
// **A failed walk whose earlier merges are walked alone is not retried** (novox/hq ADR 0276): the search for // **A failed walk whose earlier merges are walked alone is not retried** (novox/hq ADR 0276): the search for
// the merge that brought the failure answers them now, and a retried walk would name them twice. // the merge that brought the failure answers them now, and a retried walk would name them twice.
// **A walk stopped without some merges is not retried** (novox/hq ADR 0299): the merges it left out were stopped,
// and the others are another walk's now.
if p.Delivery != nil && (p.Delivery.WalkedAgainBy != "" || len(p.Delivery.LeftOut) > 0) {
return "", fmt.Errorf("%s was stopped without %s: those merges were stopped, and %s walks the others; a "+
"newer merge, or `rebuild <module>`, builds again", p.ID, mergesLeftOutWords(p.Delivery.LeftOut),
orWords(p.Delivery.WalkedAgainBy, "nothing"))
}
if p.Delivery != nil && len(p.Delivery.Merges) > 0 { if p.Delivery != nil && len(p.Delivery.Merges) > 0 {
kept, err := inv.MergesOf(ctx, p.ID) kept, err := inv.MergesOf(ctx, p.ID)
if err != nil { if err != nil {
+5
View File
@@ -78,6 +78,11 @@ func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation
return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err) return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err)
} }
var problems, foreseen []string var problems, foreseen []string
// Each module the next send leaves out of this machine's declaration (novox/hq issue 380), judged from this
// resolution as the send judges it: a finding, never a refusal.
if err == nil {
out = append(out, leftOutObservations(n.Name, leftOutOf(plan, settings, n.Adopted))...)
}
if err == nil && gensErr == nil { if err == nil && gensErr == nil {
var declared sendable var declared sendable
if declared, err = declarationWith(ctx, open, n.Name, plan, settings, gens, Foreseeing); err == nil { if declared, err = declarationWith(ctx, open, n.Name, plan, settings, gens, Foreseeing); err == nil {
+11
View File
@@ -1236,6 +1236,17 @@ func sendToEach(ctx context.Context, open *stores, names []string) ([]string, er
continue continue
} }
numbered.stamp(&declared) numbered.stamp(&declared)
// **The grants step sends the user list and nothing else** (novox/hq issue 490): judged on this very
// declaration, the one sent, so nothing composed between a check and the send can slip through.
if everyBuildKept(ctx, name) {
other, err := grantsOnlyIn(ctx, open, name, plan, declared)
if err != nil {
return nil, err
}
if other != "" {
return nil, fmt.Errorf("%w: %s", errNotGrantsOnly, other)
}
}
reportLeftOut(name, declared) reportLeftOut(name, declared)
sending = append(sending, readyNode{name, declared}) sending = append(sending, readyNode{name, declared})
} }
+49
View File
@@ -69,6 +69,9 @@ type meshStatus struct {
// **A document without this said an outage was a well mesh.** Read from what each machine // **A document without this said an outage was a well mesh.** Read from what each machine
// reported, so it is the machine's account and not the mesh's take-time listing. // reported, so it is the machine's account and not the mesh's take-time listing.
Untaken []machineUntaken `json:"untaken,omitempty"` Untaken []machineUntaken `json:"untaken,omitempty"`
// NotApplied is every module assigned to a machine and left out of its composition, with why (novox/hq
// issue 380). Absent when every module composes.
NotApplied []machineNotApplied `json:"not-applied,omitempty"`
// Filtered is every converged machine that is not filtered by the mesh alone (novox/hq ADR // Filtered is every converged machine that is not filtered by the mesh alone (novox/hq ADR
// 0168), one entry per rule set the mesh did not write — the found firewall in force again, // 0168), one entry per rule set the mesh did not write — the found firewall in force again,
// or a chain nobody speaks for. Absent when every converged machine is filtered by the mesh // or a chain nobody speaks for. Absent when every converged machine is filtered by the mesh
@@ -106,6 +109,33 @@ type meshStatus struct {
// why they could not be read. // why they could not be read.
Pending []pendingStatus `json:"pending,omitempty"` Pending []pendingStatus `json:"pending,omitempty"`
PendingUnread string `json:"pendingUnread,omitempty"` PendingUnread string `json:"pendingUnread,omitempty"`
// LastSent is every machine's last recorded send — who sent it, under which lease epoch, from which
// assignment generation, naming which modules — and what the machine answered of it (novox/hq issue 485).
// A machine the mesh has recorded no send to is absent. LastSentUnread says why it could not be read.
LastSent []machineLastSent `json:"lastSent"`
LastSentUnread string `json:"lastSentUnread,omitempty"`
}
// machineLastSent is one machine's last send as status says it.
type machineLastSent struct {
Node string `json:"node"`
Sequence int64 `json:"sequence"`
SentAt time.Time `json:"sentAt"`
Sender string `json:"sender"`
Epoch int64 `json:"epoch"`
Generation int64 `json:"generation"`
Digest string `json:"digest"`
Modules []string `json:"modules"`
// Answer is what the machine said of it: applied, failed or refused, as its report has it; empty when it
// has not reported on it. Refused is its words when it refused it whole, Failed how many resources failed.
Answer string `json:"answer"`
Refused string `json:"refused,omitempty"`
Failed int `json:"failed,omitempty"`
AnsweredAt *time.Time `json:"answeredAt,omitempty"`
// RefusedForGeneration is when the machine refused it for coming from an older assignment generation than
// it applied, and AppliedGeneration that generation; absent when it did not.
RefusedForGeneration *time.Time `json:"refusedForGeneration,omitempty"`
AppliedGeneration int64 `json:"appliedGeneration,omitempty"`
} }
// pendingStatus is one pending assignment as status says it. // pendingStatus is one pending assignment as status says it.
@@ -251,6 +281,7 @@ func statusAsJSON(asked answers) ([]byte, error) {
} }
} }
out.Unheld = asked.unheld out.Unheld = asked.unheld
out.NotApplied = notApplied(asked.leftOut)
out.HandActsThisWeek, out.HandActsUnread = asked.handActs, asked.handActsUnread out.HandActsThisWeek, out.HandActsUnread = asked.handActs, asked.handActsUnread
out.HealsThisWeek, out.HealsUnread = asked.heals, asked.healsUnread out.HealsThisWeek, out.HealsUnread = asked.heals, asked.healsUnread
// In brief, as `conditions` lists them: status leads with every open condition, and their whole // In brief, as `conditions` lists them: status leads with every open condition, and their whole
@@ -263,6 +294,7 @@ func statusAsJSON(asked answers) ([]byte, error) {
out.Pending = append(out.Pending, pendingStatus{Node: p.Node, Module: p.Module, State: p.State, out.Pending = append(out.Pending, pendingStatus{Node: p.Node, Module: p.Module, State: p.State,
Build: p.Build, Repository: p.Repository, Path: p.Path, Since: p.Since, Settled: p.Settled, Note: p.Note}) Build: p.Build, Repository: p.Repository, Path: p.Path, Since: p.Since, Settled: p.Settled, Note: p.Note})
} }
out.LastSent, out.LastSentUnread = lastSentStatus(asked.lastSent), asked.lastSentUnread
for name := range asked.refused { for name := range asked.refused {
out.Unresolved = append(out.Unresolved, machineUnresolved{ out.Unresolved = append(out.Unresolved, machineUnresolved{
Node: name, Problem: asked.refused[name]}) Node: name, Problem: asked.refused[name]})
@@ -319,3 +351,20 @@ func say(value any) error {
fmt.Println(string(body)) fmt.Println(string(body))
return nil return nil
} }
// lastSentStatus is every machine's last send as status --json says it, by machine name (novox/hq issue 485).
func lastSentStatus(sends map[string]inventory.Send) []machineLastSent {
out := []machineLastSent{}
for _, s := range sends {
modules := s.Modules
if modules == nil {
modules = []string{}
}
out = append(out, machineLastSent{Node: s.NodeName, Sequence: s.Sequence, SentAt: s.SentAt, Sender: s.Sender,
Epoch: s.Epoch, Generation: s.Generation, Digest: s.Digest, Modules: modules, Answer: s.Answer.Outcome,
Refused: s.Answer.Refused, Failed: s.Answer.Failed, AnsweredAt: s.Answer.At,
RefusedForGeneration: s.RefusedAt, AppliedGeneration: s.RefusedApplied})
}
sort.Slice(out, func(i, j int) bool { return out[i].Node < out[j].Node })
return out
}
+44 -3
View File
@@ -61,20 +61,57 @@ func keptExcept(ctx context.Context) (map[string]bool, bool) {
return skip, on return skip, on
} }
type keepEveryBuildKey struct{}
// keepingEveryBuild is a context whose sends compose every module of one machine — recorded or rolling out —
// at the build that machine was last sent (novox/hq issue 490): the grants step, which sends the machine
// holding the bus its new user list and nothing of any module's new code. That code waits there for a gate
// like any other move; the list must not, or the first machine's gate is judged against a bus that refuses
// what the change newly grants.
//
// **That machine alone.** Composing one machine resolves the others too — who is on the private network,
// who answers a requirement — on the same context, and those are no part of the step's send: they are
// composed as any send composes them. Kept for every machine, a machine whose last send is not known
// refused the bus's machine's composition.
func keepingEveryBuild(ctx context.Context, node string) context.Context {
return context.WithValue(ctx, keepEveryBuildKey{}, node)
}
// everyBuildKept is whether this context keeps every module of this machine at the build it runs.
func everyBuildKept(ctx context.Context, node string) bool {
on, _ := ctx.Value(keepEveryBuildKey{}).(string)
return on != "" && on == node
}
// recordedKept is, for a send under keepingRecorded, every recorded module the machine was last sent a // recordedKept is, for a send under keepingRecorded, every recorded module the machine was last sent a
// build of that the mesh's build is not identical to: module → the commit it keeps. Nil when the context // build of that the mesh's build is not identical to: module → the commit it keeps. Nil when the context
// keeps nothing, or when what the machine was last sent is not known (it is then held whole elsewhere — // keeps nothing, or when what the machine was last sent is not known (it is then held whole elsewhere —
// ADR 0221). // ADR 0221).
//
// Under keepingEveryBuild, every module the machine was sent is kept, whatever its policy (novox/hq issue
// 490), and a machine whose last send is not known refuses the send: there is nothing to keep it at, and
// composing the mesh's builds would be the very move the grants step must not make.
func recordedKept(ctx context.Context, open *stores, node string) (map[string]string, error) { func recordedKept(ctx context.Context, open *stores, node string) (map[string]string, error) {
every := everyBuildKept(ctx, node)
skip, on := keptExcept(ctx) skip, on := keptExcept(ctx)
if !on { if !on && !every {
return nil, nil return nil, nil
} }
if every {
skip = nil
}
inv := open.inventory inv := open.inventory
sent, known, err := inv.SentBuilds(ctx, node) sent, known, err := inv.SentBuilds(ctx, node)
if err != nil || !known { if err != nil {
return nil, err return nil, err
} }
if !known {
if every {
return nil, fmt.Errorf("what %s was last sent is not known, so the bus's user list cannot be sent "+
"there alone with every build kept (novox/hq issue 490)", node)
}
return nil, nil
}
current, err := inv.CurrentBuilds(ctx) current, err := inv.CurrentBuilds(ctx)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -83,7 +120,7 @@ func recordedKept(ctx context.Context, open *stores, node string) (map[string]st
out := map[string]string{} out := map[string]string{}
for m, was := range sent { for m, was := range sent {
now, held := current[m] now, held := current[m]
if !held || now.RollOut || skip[m] || was == "" || sameCommit(was, now.Commit) { if !held || (now.RollOut && !every) || skip[m] || was == "" || sameCommit(was, now.Commit) {
continue continue
} }
if f == nil { if f == nil {
@@ -118,6 +155,10 @@ func keepRecorded(ctx context.Context, open *stores, node string, shelf map[stri
if err != nil { if err != nil {
return nil, err return nil, err
} }
if !found && everyBuildKept(ctx, node) {
return nil, fmt.Errorf("%s runs %s's build %s, which the build records no longer hold: the bus's user "+
"list cannot be sent there alone with it kept (novox/hq issue 490)", node, m, short(kept[m]))
}
if !found { if !found {
return nil, fmt.Errorf("%s records rather than rolls out, and %s runs its build %s, which the build "+ return nil, fmt.Errorf("%s records rather than rolls out, and %s runs its build %s, which the build "+
"records no longer hold: this send cannot keep it and does not move it — `push %s` sends the new "+ "records no longer hold: this send cannot keep it and does not move it — `push %s` sends the new "+
+190 -12
View File
@@ -2,6 +2,7 @@ package main
import ( import (
"context" "context"
"encoding/json"
"errors" "errors"
"flag" "flag"
"fmt" "fmt"
@@ -253,8 +254,19 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
if err != nil { if err != nil {
return nil, err return nil, err
} }
// The grants step composes every module at the build its machine was last sent (novox/hq issue 490):
// a move it keeps is not made. Read, not assumed, so a move it could not keep is still refused here.
var keeps map[string]string
if everyBuildKept(ctx, n) {
if keeps, err = recordedKept(ctx, open, n); err != nil {
return nil, err
}
}
var waiting []string var waiting []string
for _, mv := range moves { for _, mv := range moves {
if was, kept := keeps[mv.Module]; kept && sameCommit(was, mv.From) {
continue
}
if !scope.judged[n] && !scope.modules[mv.Module] { if !scope.judged[n] && !scope.modules[mv.Module] {
waiting = append(waiting, fmt.Sprintf("%s %s → %s", mv.Module, short(mv.From), short(mv.To))) waiting = append(waiting, fmt.Sprintf("%s %s → %s", mv.Module, short(mv.From), short(mv.To)))
} }
@@ -283,15 +295,18 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
// owns are the moves the send exists for — every module of a plan's tier whose first machine this is, in // owns are the moves the send exists for — every module of a plan's tier whose first machine this is, in
// one send (novox/hq issue 281): a send carries the machine's whole declaration (ADR 0221), so a send per // one send (novox/hq issue 281): a send carries the machine's whole declaration (ADR 0221), so a send per
// module was the same declaration sent again and again, each one setting aside the one before. // module was the same declaration sent again and again, each one setting aside the one before.
func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, []string, error) { //
// And the grants step first, when the send changes what the bus's user list must say (novox/hq issue 490):
// answered for the gate to keep, nil when there was none.
func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, []string, *inventory.GrantsStep, error) {
inv := open.inventory inv := open.inventory
f, err := readMoveFacts(ctx, inv) f, err := readMoveFacts(ctx, inv)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, nil, err
} }
moves, err := machineMoves(ctx, open, f, node, true) moves, err := machineMoves(ctx, open, f, node, true)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, nil, err
} }
own := func(module string) bool { own := func(module string) bool {
return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module }) return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module })
@@ -301,7 +316,7 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
continue continue
} }
if id := f.walkedBy(mv.Module, node, mv.To); id != "" { if id := f.walkedBy(mv.Module, node, mv.To); id != "" {
return nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere, return nil, nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
mv.Module, short(mv.To), node, id) mv.Module, short(mv.To), node, id)
} }
} }
@@ -309,7 +324,7 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
// plan, or a release, on this machine is not crossed with a newer build; this send waits for its gate. // plan, or a release, on this machine is not crossed with a newer build; this send waits for its gate.
for _, o := range owns { for _, o := range owns {
if id := f.walkedBy(o.Module, node, o.To); id != "" { if id := f.walkedBy(o.Module, node, o.To); id != "" {
return nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere, return nil, nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
o.Module, short(o.To), node, id) o.Module, short(o.To), node, id)
} }
} }
@@ -323,22 +338,22 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
} }
} }
if len(moves) == 0 && len(owns) == 0 { if len(moves) == 0 && len(owns) == 0 {
return nil, nil, nil return nil, nil, nil, nil
} }
for i := range moves { for i := range moves {
if moves[i].Build == "" { if moves[i].Build == "" {
if moves[i].Build, err = inv.BuildOf(ctx, moves[i].Module, moves[i].To); err != nil { if moves[i].Build, err = inv.BuildOf(ctx, moves[i].Module, moves[i].To); err != nil {
return nil, nil, err return nil, nil, nil, err
} }
} }
} }
sort.Slice(moves, func(i, j int) bool { return moves[i].Module < moves[j].Module }) sort.Slice(moves, func(i, j int) bool { return moves[i].Module < moves[j].Module })
sayRecreations(ctx, open, moves) sayRecreations(ctx, open, moves)
sent, err := sendRollout(withScope(ctx, sendScope{judged: map[string]bool{node: true}}), open, []string{node}) sent, grants, err := sendJudged(ctx, open, node)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, nil, err
} }
return moves, sent, nil return moves, sent, grants, nil
} }
// passCarried keeps a pass as the verdict of every build the gate judged beside its own module. // passCarried keeps a pass as the verdict of every build the gate judged beside its own module.
@@ -674,7 +689,7 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
r.Next++ r.Next++
continue continue
} }
moves, sent, err := gatedSend(ctx, open, node, nil) moves, sent, grants, err := gatedSend(ctx, open, node, nil)
if errors.Is(err, errWalkedElsewhere) { if errors.Is(err, errWalkedElsewhere) {
note := fmt.Sprintf("waiting before %s: %v", node, err) note := fmt.Sprintf("waiting before %s: %v", node, err)
changed := p.Note != note changed := p.Note != note
@@ -689,8 +704,12 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
r.Next++ r.Next++
continue continue
} }
r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves, Sent: sentNow(ctx, open.inventory, sent)} r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves, Sent: sentNow(ctx, open.inventory, sent),
Grants: grants}
p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves)) p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves))
if said := grantsSaid(grants); said != "" {
p.Note += "; " + said
}
if said := recreationsSaid(moves); said != "" { if said := recreationsSaid(moves); said != "" {
p.Note += "; " + said p.Note += "; " + said
} }
@@ -916,3 +935,162 @@ func recreationsSaid(moves []inventory.CarriedMove) string {
} }
return strings.Join(said, "; ") return strings.Join(said, "; ")
} }
// The grants step (novox/hq issue 490).
//
// The bus's user list — every module's grants, composed mesh-wide from the catalogue — travels only in the
// declaration of the machine holding the bus. A send to any other machine that changed it put that machine
// first (issue 249), unless a build waited there for a gate: then ungatedIn left it out, and said so. On
// 2026-10-11 a merge gave restic a new health tool; its walk sent restic to its first machine and judged it
// there, the machine's node-engine asked the tool and the bus refused it ("this host's grant does not name
// …"), because the machine holding the bus also runs restic, whose new build waited for that very gate. The
// gate could not pass; a person pushed the bus's machine by hand, which carried restic's new build there
// unjudged.
//
// So before a gated send, when the user list composed now is not the one the machine holding the bus was
// last sent, that machine is sent its declaration with **every build kept at the one it runs**: the new
// list, and nothing of any module's new code. Said on the gate (`plans`), and, when it cannot be sent, said
// with why and passed over: the send goes on as it did before, and its gate says what it finds.
// brokerBehindOf is brokerBehind: a variable so a test of the walk needs no store.
var brokerBehindOf = brokerBehind
// grantsStep sends the machine holding the bus its user list alone, ahead of a gated send to node, when the
// list changed; answers what it did, or nil when there was nothing to send.
func grantsStep(ctx context.Context, open *stores, node string) *inventory.GrantsStep {
holder, behind, err := brokerBehindOf(ctx, open, []string{node})
if err != nil {
fmt.Printf("grants step before %s: whether the bus's user list changed could not be read: %v\n", node, err)
return &inventory.GrantsStep{Node: "the machine holding the bus", Failed: err.Error()}
}
if holder == "" || holder == node || !behind {
// No bus with a user list, the gate's own machine holds it (its send carries the list first), or the
// list is the one already sent.
return nil
}
if _, err := sendRollout(keepingEveryBuild(withScope(ctx, sendScope{}), holder), open, []string{holder}); err != nil {
fmt.Printf("grants step: the bus's user list could not be sent to %s ahead of %s, which is sent without "+
"it — the bus may refuse what the send newly grants: %v\n", holder, node, err)
return &inventory.GrantsStep{Node: holder, Failed: err.Error()}
}
now := time.Now().UTC()
fmt.Printf("grants step: %s sent the bus's user list alone, every build there kept, before %s is judged\n",
holder, node)
return &inventory.GrantsStep{Node: holder, At: &now}
}
// errNotGrantsOnly is a grants step refused unsent: its declaration would change more than the user list.
var errNotGrantsOnly = errors.New("the grants step would change more than the bus's user list, and is not sent")
// grantsOnlyIn is what a grants step's declaration, composed for sending, would change on the machine holding
// the bus besides its user list, against what it was last sent (novox/hq issue 490); empty when nothing else.
// Judged by sendToEach on the very declaration it then sends — one composition, judged, then sent or refused
// unsent. Kept builds are not the whole of a declaration: a new assignment, a settings change, an assignment
// taken away, a provision's logins are composed as the mesh holds them now, and a step that carried any of
// it would be the unjudged change this step exists to avoid. Compared resource by resource with the summary
// the last send kept (ADR 0217): exact, rather than a list of the cases that can differ. A module there that
// was never sent is a new assignment, refused whatever its resources.
func grantsOnlyIn(ctx context.Context, open *stores, holder string, plan catalogue.Resolution, declared sendable) (string, error) {
inv := open.inventory
sent, known, err := inv.SentBuilds(ctx, holder)
if err != nil {
return "", err
}
if !known {
return fmt.Sprintf("what %s was last sent is not known", holder), nil
}
var other []string
listID := ""
for _, m := range plan.Modules {
if _, was := sent[m.Module]; !was {
other = append(other, m.Module+" newly assigned")
}
if m.BusUsers != "" && m.ClaimsSeat(catalogue.BrokerSeat) {
listID = m.Module + "." + catalogue.BusUsersID()
}
}
var facts *moveFacts
for m, was := range sent {
now, carried := declared.Builds[m]
if !carried || sameCommit(now, was) {
continue
}
if facts == nil {
f, err := readMoveFacts(ctx, inv)
if err != nil {
return "", err
}
facts = &f
}
if !facts.identical(m, was, now) {
other = append(other, fmt.Sprintf("%s %s → %s", m, short(was), short(now)))
}
}
for _, f := range declared.foreseen {
other = append(other, f+" would be minted")
}
body, err := declared.Body()
if err != nil {
return "", err
}
after, err := summarize(body)
if err != nil {
return "", err
}
prev, err := inv.SentSummary(ctx, holder)
if err != nil {
return "", err
}
if len(prev) == 0 {
return fmt.Sprintf("what %s was last sent is not kept for comparison", holder), nil
}
var before []sentResource
if err := json.Unmarshal(prev, &before); err != nil {
return "", fmt.Errorf("what %s was last sent is kept in a form this version does not read: %w", holder, err)
}
other = append(other, otherThanTheList(diffSent(before, after), listID)...)
sort.Strings(other)
if len(other) == 0 {
return "", nil
}
return "its send would change more than the bus's user list: " + strings.Join(other, "; "), nil
}
// otherThanTheList is what a diff against the last send changes besides the user list's content: every
// resource added, removed or changed, but the list's own resource when its content is all that changed —
// its path, mode and every other field must be as last sent (novox/hq issue 490).
func otherThanTheList(d sentDiff, listID string) []string {
var other []string
for _, id := range d.Added {
other = append(other, "+"+id)
}
for _, id := range d.Removed {
other = append(other, "-"+id)
}
for _, c := range d.Changed {
if c.ID == listID && len(c.Fields) == 1 && c.Fields[0] == "content" {
continue
}
other = append(other, fmt.Sprintf("~%s (%s)", c.ID, strings.Join(c.Fields, ", ")))
}
return other
}
// sendJudged is a gated send's sends: the grants step when the user list changed, then the machine judged.
func sendJudged(ctx context.Context, open *stores, node string) ([]string, *inventory.GrantsStep, error) {
grants := grantsStep(ctx, open, node)
sent, err := sendRollout(withScope(ctx, sendScope{judged: map[string]bool{node: true}}), open, []string{node})
return sent, grants, err
}
// grantsSaid is a grants step as `plans` and a walk's note say it.
func grantsSaid(g *inventory.GrantsStep) string {
switch {
case g == nil:
return ""
case g.Failed != "":
return fmt.Sprintf(grantsStepWord+" to %s FAILED, sent without it: %s", g.Node, g.Failed)
default:
return fmt.Sprintf(grantsStepWord+": %s sent the bus's user list first, its builds kept", g.Node)
}
}
+6 -1
View File
@@ -971,7 +971,7 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
} }
// A gated send (ADR 0236): everything waiting on the machine goes with the tier, and the gate judges // A gated send (ADR 0236): everything waiting on the machine goes with the tier, and the gate judges
// all of it there. // all of it there.
carried, sent, err := gatedSend(ctx, open, node, owns) carried, sent, grants, err := gatedSend(ctx, open, node, owns)
if err != nil { if err != nil {
return err return err
} }
@@ -992,6 +992,7 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
s.GatedBy = "" s.GatedBy = ""
if m == lead { if m == lead {
s.Gate.Carried = carried s.Gate.Carried = carried
s.Gate.Grants = grants
} else { } else {
s.GatedBy = lead s.GatedBy = lead
} }
@@ -1001,6 +1002,10 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
strings.Join(sent, ", ")) strings.Join(sent, ", "))
fmt.Printf("%s: tier %d built; sent %d module(s) to %s first in one send (%s), the rest once its gate passes\n", fmt.Printf("%s: tier %d built; sent %d module(s) to %s first in one send (%s), the rest once its gate passes\n",
p.ID, p.Tier, len(modules), strings.Join(sent, ", "), strings.Join(modules, ", ")) p.ID, p.Tier, len(modules), strings.Join(sent, ", "), strings.Join(modules, ", "))
// The grants step taken before it (novox/hq issue 490), said with it.
if said := grantsSaid(grants); said != "" {
p.Note += "; " + said
}
// What the send recreates, said with it (novox/hq ADR 0245). // What the send recreates, said with it (novox/hq ADR 0245).
if said := recreationsSaid(carried); said != "" { if said := recreationsSaid(carried); said != "" {
p.Note += "; " + said p.Note += "; " + said
+1
View File
@@ -160,6 +160,7 @@ func (b *backlogMesh) release(t *testing.T) inventory.Plan {
// The backlog goes out one machine at a time: the first judged before the second is sent, each build's // The backlog goes out one machine at a time: the first judged before the second is sent, each build's
// pass kept; a rebuild with the same bytes is no move at all; and a send outside a gate is refused. // pass kept; a rebuild with the same bytes is no move at all; and a send outside a gate is refused.
func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) { func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) {
judgeLeavesAsTheCore(t)
b := aBacklog(t) b := aBacklog(t)
ctx := t.Context() ctx := t.Context()
inv := b.open.inventory inv := b.open.inventory
+1
View File
@@ -77,6 +77,7 @@ func testAWalkStoppedAfterItsGatePassedKeepsThePass(t *testing.T) {
// the first send is older than the wait for a first machine's report. Read before the pass is taken over, the // the first send is older than the wait for a first machine's report. Read before the pass is taken over, the
// carried module's step said the first machine never applied it, and the passed build was put back. // carried module's step said the first machine never applied it, and the passed build was put back.
func TestACarriedModuleTakesItsLeadsPassBeforeItsStepIsRead(t *testing.T) { func TestACarriedModuleTakesItsLeadsPassBeforeItsStepIsRead(t *testing.T) {
judgeLeavesAsTheCore(t)
tm := aTierMesh(t, "m01", "m02") tm := aTierMesh(t, "m01", "m02")
ctx := t.Context() ctx := t.Context()
inv := tm.open.inventory inv := tm.open.inventory
+9
View File
@@ -55,6 +55,15 @@ func rotateCommand(ctx context.Context, args []string) error {
defer open.Close() defer open.Close()
inv := open.inventory inv := open.inventory
// A credential the provider's software issued is not the mesh's to remake (novox/hq ADR 0305 §9).
shelf, err := inv.Catalogue(ctx)
if err != nil {
return err
}
if err := issuedRefusesRotation(shelf, provision); err != nil {
return err
}
holders, err := inv.HoldersOf(ctx, provision, *only) holders, err := inv.HoldersOf(ctx, provision, *only)
if err != nil { if err != nil {
return err return err
+1
View File
@@ -162,6 +162,7 @@ func TestABusRebuiltFromAnUnchangedSourceDemandsNoBusStep(t *testing.T) {
// A plan's build made from the source every machine running the module runs is not sent and not // A plan's build made from the source every machine running the module runs is not sent and not
// judged: nothing moves. A build of a changed source is sent to its first machine, gated, as before. // judged: nothing moves. A build of a changed source is sent to its first machine, gated, as before.
func TestAPlanSendsNothingForAnUnchangedSource(t *testing.T) { func TestAPlanSendsNothingForAnUnchangedSource(t *testing.T) {
judgeLeavesAsTheCore(t)
for _, tc := range []struct { for _, tc := range []struct {
name string name string
source string source string
+20 -1
View File
@@ -410,6 +410,10 @@ func (a *verbArguments) commandLine() ([]string, error) {
if b := str("by"); b != "" { if b := str("by"); b != "" {
argv = append(argv, "--by", catalogue.DeliverySeat+" for "+b) argv = append(argv, "--by", catalogue.DeliverySeat+" for "+b)
} }
// The merges to leave out: the walk's other merges are walked again without them (novox/hq ADR 0299).
if w := str("without"); w != "" {
argv = append(argv, "--without", w)
}
return argv, nil return argv, nil
case "delivery-walks": case "delivery-walks":
argv := []string{"delivery", "walks"} argv := []string{"delivery", "walks"}
@@ -773,6 +777,13 @@ func (a *verbArguments) commandLine() ([]string, error) {
return nil, err return nil, err
} }
return []string{"secret", "ask", str("node"), str("module"), str("secret"), "--at", str("at")}, nil return []string{"secret", "ask", str("node"), str("module"), str("secret"), "--at", str("at")}, nil
case "issued":
// The credential a provider's software issued, taken from the provider (novox/hq ADR 0305): no value in
// the arguments, the provision alone.
if err := need("provision"); err != nil {
return nil, err
}
return []string{"secret", "issued", str("provision")}, nil
case "secret-ask": case "secret-ask":
// A module's own secret asked for, typed by the operator at the desk (novox/hq ADR 0277): never a value // A module's own secret asked for, typed by the operator at the desk (novox/hq ADR 0277): never a value
// in the arguments. The desk is the module's machine unless at names another. // in the arguments. The desk is the module's machine unless at names another.
@@ -1579,6 +1590,14 @@ func givenAtTheDesk(argv []string) bool {
return argv[5] == "--at" && argv[6] != "" && !strings.HasPrefix(argv[6], "-") return argv[5] == "--at" && argv[6] != "" && !strings.HasPrefix(argv[6], "-")
} }
// takenFromTheProvider is exactly the line the `issued` verb composes, and nothing beside it: `secret issued
// <provision>` (novox/hq ADR 0305 §10). It carries no value — the value comes from the provider's own tool, sealed to
// the call — and answers none.
func takenFromTheProvider(argv []string) bool {
return len(argv) == 3 && argv[0] == "secret" && argv[1] == "issued" && argv[2] != "" &&
!strings.HasPrefix(argv[2], "-")
}
// terminalOnly refuses, through any verb, a command that is the operator's at the controller's terminal // terminalOnly refuses, through any verb, a command that is the operator's at the controller's terminal
// alone (novox/hq ADR 0266). **Every `node` subcommand that is not a read**: `node account` and // alone (novox/hq ADR 0266). **Every `node` subcommand that is not a read**: `node account` and
// `node agent-account` above all. Whoever may call a verb includes agents, and an agent that named itself // `node agent-account` above all. Whoever may call a verb includes agents, and an agent that named itself
@@ -1595,7 +1614,7 @@ func terminalOnly(argv []string) error {
// Of a secret's commands only rotation, which seals the new value to the machine that uses it, and the // Of a secret's commands only rotation, which seals the new value to the machine that uses it, and the
// `give` verb's own line: an own secret typed by the operator into the desk's hidden prompt, sealed to this // `give` verb's own line: an own secret typed by the operator into the desk's hidden prompt, sealed to this
// call and then to the module's machine, so no value travels in the verb or its answer (hq ADR 0259 §10). // call and then to the module's machine, so no value travels in the verb or its answer (hq ADR 0259 §10).
if argv[0] == "secret" && (len(argv) < 2 || argv[1] != "rotate") && !givenAtTheDesk(argv) { if argv[0] == "secret" && (len(argv) < 2 || argv[1] != "rotate") && !givenAtTheDesk(argv) && !takenFromTheProvider(argv) {
return terminalRefusal("secret %s is run at the controller's terminal only, never through a verb: accepting, "+ return terminalRefusal("secret %s is run at the controller's terminal only, never through a verb: accepting, "+
"recovering or exporting a secret hands it to whoever asks, and that includes agents (novox/hq ADR "+ "recovering or exporting a secret hands it to whoever asks, and that includes agents (novox/hq ADR "+
"0266). Nothing was done", strings.Join(argv[1:], " ")) "0266). Nothing was done", strings.Join(argv[1:], " "))
+4
View File
@@ -41,6 +41,9 @@ func secretCommand(ctx context.Context, args []string) error {
case "accept": case "accept":
case "ask": case "ask":
return secretAsk(ctx, args[1:]) return secretAsk(ctx, args[1:])
case "issued":
// The credential a provider's software issued, taken from the provider (novox/hq ADR 0305).
return secretIssued(ctx, args[1:])
case "rotate": case "rotate":
return secretRotate(ctx, args[1:]) return secretRotate(ctx, args[1:])
case "recover": case "recover":
@@ -168,6 +171,7 @@ func secretAsk(ctx context.Context, args []string) error {
const secretUsage = "secret rotate <node> <module> <name> [--why <text> [--cause <word>]]\n" + const secretUsage = "secret rotate <node> <module> <name> [--why <text> [--cause <word>]]\n" +
"secret ask <node> <module> <name> [--at <machine>]\n" + "secret ask <node> <module> <name> [--at <machine>]\n" +
"secret issued <provision>\n" +
"secret accept <node> <module> <name> [--from <file> | --at-desk <machine>] [--provider <node> [--local <name>]]\n" + "secret accept <node> <module> <name> [--from <file> | --at-desk <machine>] [--provider <node> [--local <name>]]\n" +
"secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>] [--provider <node>]\n" + "secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>] [--provider <node>]\n" +
"secret export [--out <file>]" "secret export [--out <file>]"
+1 -1
View File
@@ -66,7 +66,7 @@ func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
} }
// Both machines' `node show` name it: where the provider runs, and where the consumer is. // Both machines' `node show` name it: where the provider runs, and where the consumer is.
for _, node := range []string{"anchor", "laptop"} { for _, node := range []string{"anchor", "laptop"} {
shown := printed(t, func() error { return showNode(ctx, open.inventory, node) }) shown := printed(t, func() error { return showNode(ctx, open, node) })
if !strings.Contains(shown, "open condition(s) about this machine") || !strings.Contains(shown, "mesh_laptop_dashboard") { if !strings.Contains(shown, "open condition(s) about this machine") || !strings.Contains(shown, "mesh_laptop_dashboard") {
t.Fatalf("node show %s does not name it:\n%s", node, shown) t.Fatalf("node show %s does not name it:\n%s", node, shown)
} }
+24 -2
View File
@@ -289,6 +289,10 @@ func printStatus(asked answers) error {
fmt.Printf("\n `take <node> <module>` compares what runs against what it declares, and runs it\n\n") fmt.Printf("\n `take <node> <module>` compares what runs against what it declares, and runs it\n\n")
} }
// Assigned and not applied (novox/hq issue 380): before what is merely reported, because it reads like work
// finished and is none.
printNotApplied(asked.leftOut)
if len(asked.unheld) > 0 { if len(asked.unheld) > 0 {
// **Reported, and not refused yet** (novox/hq ADR 0207 §4). Each machine still resolves and // **Reported, and not refused yet** (novox/hq ADR 0207 §4). Each machine still resolves and
// is sent what it would be; this says which of its modules depend on a seat nothing there // is sent what it would be; this says which of its modules depend on a seat nothing there
@@ -337,6 +341,10 @@ func printStatus(asked answers) error {
"lists them, and each is in its condition's tried\n\n", asked.heals.Acts, asked.heals.Escalated) "lists them, and each is in its condition's tried\n\n", asked.heals.Acts, asked.heals.Escalated)
} }
// What each machine was last sent and by whom, one line each (novox/hq issue 485). Not a fault, so it does not
// break "all well"; a record that could not be read does.
printLastSends(nodes, asked.lastSent, asked.lastSentUnread)
if adopted := adoptedNodes(nodes); len(adopted) > 0 { if adopted := adoptedNodes(nodes); len(adopted) > 0 {
// Said, because nothing forces the flip: a node left adopted is visible here rather than // Said, because nothing forces the flip: a node left adopted is visible here rather than
// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well". // read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
@@ -456,6 +464,13 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
return answers{}, err return answers{}, err
} }
plans[n.Name] = planned{plan, settings} plans[n.Name] = planned{plan, settings}
// And which of its modules a push leaves out (novox/hq issue 380), judged as the push judges it.
if left := leftOutOf(plan, settings, n.Adopted); len(left) > 0 {
if out.leftOut == nil {
out.leftOut = map[string][]leftOutModule{}
}
out.leftOut[n.Name] = left
}
out.unheld = append(out.unheld, plan.Unheld...) out.unheld = append(out.unheld, plan.Unheld...)
// And which of its modules a provider leaves out of its grants, for an identity too long // And which of its modules a provider leaves out of its grants, for an identity too long
// for what the provision keeps (novox/hq ADR 0225) — judged from the consumer's own // for what the provision keeps (novox/hq ADR 0225) — judged from the consumer's own
@@ -474,6 +489,12 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
out.pendingUnread = err.Error() out.pendingUnread = err.Error()
err = nil err = nil
} }
// And what each machine was last sent, by whom (novox/hq issue 485): the record of sends is read through
// the controller's verbs alone, the store's reader being shut out of its tables. Unreadable is said.
if out.lastSent, err = inv.LastSends(ctx); err != nil {
out.lastSentUnread = err.Error()
err = nil
}
out.plans, err = inv.RecentPlans(ctx, 5) out.plans, err = inv.RecentPlans(ctx, 5)
if err != nil { if err != nil {
return answers{}, err return answers{}, err
@@ -604,9 +625,10 @@ func untakenModules(ctx context.Context, inv *inventory.Inventory, nodes []inven
// read as success for the whole of the edge cut-over outage (novox/hq 04-ISSUES/125). // read as success for the whole of the edge cut-over outage (novox/hq 04-ISSUES/125).
func (a answers) well() bool { func (a answers) well() bool {
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 && return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 && len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 && len(a.leftOut) == 0 &&
len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.overflowing) == 0 && len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.overflowing) == 0 &&
len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending) len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending) &&
a.lastSentUnread == ""
} }
// pendingOpen is whether any pending assignment is still to be made. One that ended without being made is // pendingOpen is whether any pending assignment is still to be made. One that ended without being made is
+142
View File
@@ -0,0 +1,142 @@
{
"module": "nfs-server",
"version": "1",
"upgrade": {
"policy": "record",
"why": "the folders other machines mount: a build that breaks the exports leaves every client's mount hanging or refused, and the gate on this one machine does not see the clients (hq ADR 0236, ADR 0263)"
},
"capabilities": [
"package-manager",
"service-manager"
],
"provides": [
{
"name": "nfs-share",
"scope": "mesh",
"identity": false
}
],
"data": {
"consumers": {
"nfs-share": {
"class": "none",
"why": "the shared folders are the operator's data (hq ADR 0051): what a client writes lands in them, and they are protected where the operator declares them, never by this module, which keeps nothing of a consumer's"
}
}
},
"claims": [
{
"name": "node-nfs-server",
"scope": "node",
"serves": [
"exports",
"clients",
"test",
"reload",
"adopt"
]
}
],
"state": [
{
"name": "exports",
"ttl-seconds": 120,
"per-machine": true
}
],
"tools": [
"nfs_health"
],
"listens": [
{
"name": "nfs",
"port": 2049,
"protocol": "tcp",
"from": "mesh",
"fixed": true,
"why": "the shares, to the mesh's machines only (hq ADR 0263): NFS version 4 alone, which needs no other port, and never the home network, where a device that is not a node could claim any user id"
}
],
"resources": [
{
"id": "package",
"type": "package",
"package": "nfs-utils"
},
{
"id": "nfs-conf",
"type": "file",
"path": "/etc/nfs.conf.d/50-mesh.conf",
"mode": "0644",
"content": "# Written by the mesh (module nfs-server, novox/hq ADR 0263). Replaced on every push; a drop-in of\n# the operator's that sorts after this one overrides it, and is theirs.\n#\n# NFS version 4 only: a client needs port 2049 and nothing else, so the module opens nothing more\n# than that, to the private network. Version 3 needs rpcbind and mountd, on ports the mesh does not open.\n[nfsd]\nvers2=n\nvers3=n\nvers4=y\nvers4.0=n\nvers4.1=y\nvers4.2=y\n"
},
{
"id": "config-dir",
"type": "directory",
"path": "/etc/nfs-server",
"mode": "0755"
},
{
"id": "config",
"type": "file",
"path": "/etc/nfs-server/shares.conf",
"mode": "0644",
"content": "# Written by the mesh (module nfs-server, novox/hq ADR 0263) from this machine's assignment.\n# Replaced on every push; change the `shares` setting, never this file.\n#\n# The shares: name=folder, or name=folder:ro, one share per folder. The module's process exports each\n# to the private network's range below, every client mapped to the folder's owner.\nshares=${setting:shares}\nrange=${machine:mesh-range}\n"
},
{
"id": "run-dir",
"type": "directory",
"path": "/run/nfs-server",
"mode": "0755"
},
{
"id": "server",
"type": "service",
"unit": "nfs-server.service",
"state": "running",
"boot": "enabled",
"restart-on": [
"nfs-conf"
],
"health": {
"kind": "unit"
}
},
{
"id": "exports",
"type": "process",
"name": "nfs-server-exports",
"artifact": "tools",
"run": [
"./nfs-server",
"exports"
],
"restart-on": [
"config"
],
"health": {
"kind": "tool",
"tool": "nfs_health",
"interval": "60s",
"timeout": "10s",
"looks": 2,
"grace": "90s"
}
}
],
"build": {
"artifacts": [
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/nfs-server",
"binary": "nfs-server",
"loads": [
"nfs-server"
]
}
]
}
}
+39
View File
@@ -167,6 +167,7 @@ func (tm *tierMesh) raise(machine, kind, summary string) {
// The incident's tier, small: a dozen modules whose first machine is the same are sent there in one // The incident's tier, small: a dozen modules whose first machine is the same are sent there in one
// send, judged by one gate, and the rest of their machines are sent once — two sends, not two dozen. // send, judged by one gate, and the rest of their machines are sent once — two sends, not two dozen.
func TestATierOfManyModulesIsSentToEachMachineOnce(t *testing.T) { func TestATierOfManyModulesIsSentToEachMachineOnce(t *testing.T) {
judgeLeavesAsTheCore(t)
var modules []string var modules []string
for i := 1; i <= 12; i++ { for i := 1; i <= 12; i++ {
modules = append(modules, fmt.Sprintf("m%02d", i)) modules = append(modules, fmt.Sprintf("m%02d", i))
@@ -268,6 +269,7 @@ func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) {
// A machine-level condition about anything else holds the machine back as a whole: everything the send // A machine-level condition about anything else holds the machine back as a whole: everything the send
// moved there fails together, once, with that reason — not one module for it. // moved there fails together, once, with that reason — not one module for it.
func TestAMachineUnhealthyAsAWholeFailsTheWholeSendTogether(t *testing.T) { func TestAMachineUnhealthyAsAWholeFailsTheWholeSendTogether(t *testing.T) {
judgeLeavesAsTheCore(t)
tm := aTierMesh(t, "app1", "app2", "app3") tm := aTierMesh(t, "app1", "app2", "app3")
ctx := t.Context() ctx := t.Context()
inv := tm.open.inventory inv := tm.open.inventory
@@ -330,6 +332,7 @@ func TestTheConditionsAboutAMachineItself(t *testing.T) {
// rollback that failed; and `plans retry` asks it again rather than refusing a build that was never // rollback that failed; and `plans retry` asks it again rather than refusing a build that was never
// judged (the incident's "NOT put back: no build kept"). // judged (the incident's "NOT put back: no build kept").
func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) { func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) {
judgeLeavesAsTheCore(t)
g := aGateMesh(t) g := aGateMesh(t)
ctx := t.Context() ctx := t.Context()
inv := g.open.inventory inv := g.open.inventory
@@ -384,6 +387,7 @@ func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) {
// them, they are registered back and marked in the judging that finds them broken, the plan goes on judging // them, they are registered back and marked in the judging that finds them broken, the plan goes on judging
// the other module to its own pass, and only then fails. // the other module to its own pass, and only then fails.
func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) { func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) {
judgeLeavesAsTheCore(t)
for _, order := range [][]string{{broker.RuntimeModule, "app1"}, {"app1", broker.RuntimeModule}} { for _, order := range [][]string{{broker.RuntimeModule, "app1"}, {"app1", broker.RuntimeModule}} {
t.Run("gate kept on "+order[0], func(t *testing.T) { t.Run("gate kept on "+order[0], func(t *testing.T) {
tm := aTierMesh(t, order...) tm := aTierMesh(t, order...)
@@ -435,3 +439,38 @@ func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) {
}) })
} }
} }
// **A send carrying a core module waits for the core's readings** (issue 524 review): the leaf beside the node
// tools has its one reading at once, but the send is judged as the core: not before the settle time, and three
// readings.
func TestASendCarryingTheCoreIsJudgedAsTheCore(t *testing.T) {
tm := aTierMesh(t, broker.RuntimeModule, "app1")
ctx := t.Context()
// One reading counted per advance, as every test with a gate left open does: a gate judged with no time
// between readings would count a pass at every look and never let an advance rest.
gateEvery, gateSettle = time.Hour, time.Hour
for i := 0; i < 6; i++ {
advancePlans(ctx, tm.open)
}
p := tm.plan(t)
lead := p.Modules[broker.RuntimeModule].Gate
if lead == nil {
lead = p.Modules["app1"].Gate
}
if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) || !p.Open() || lead == nil || lead.Verdict != "" || lead.Wants != gatePasses {
t.Fatalf("passed before the core's settle time: sent %v, plan %s (%s), gate %+v", tm.sent, p.State, p.Note, lead)
}
gateEvery, gateSettle = 0, 0
for i := 0; i < 6; i++ {
advancePlans(ctx, tm.open)
}
p = tm.plan(t)
if p.State != inventory.PlanDone || !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"laptop"}}) {
t.Fatalf("the plan is %s (%s), sent %v", p.State, p.Note, tm.sent)
}
for _, m := range []string{broker.RuntimeModule, "app1"} {
if g := p.Modules[m].Gate; g != nil && (g.Verdict != inventory.GatePassed || g.Passes < gatePasses) {
t.Fatalf("%s's gate: %+v; want passed on %d readings", m, g, gatePasses)
}
}
}
+53
View File
@@ -0,0 +1,53 @@
package broker
import "testing"
// The controller asks a tool that answers a software-issued credential in its own name, and nothing else under
// `call` (novox/hq ADR 0305 §2, issue 526): the grant is composed from the records, so the installer's first user
// list, a controller with no records, carries none.
func TestTheControllerAsksAnIssuedCredentialsToolInItsOwnNameAlone(t *testing.T) {
r := Records{Nodes: []string{"n", "ace"}, Assigned: map[string][]Declared{
"n": {{Module: "x", Issued: []string{"t"}}},
"ace": {{Module: "plex", Issued: []string{"plex_issued_credential"}}, {Module: "kometa"}},
}}
users, err := Users(r)
if err != nil {
t.Fatal(err)
}
controller := users[0]
if controller.Kind != KindController {
t.Fatalf("the first user is %s", controller.Username())
}
perms, err := PermissionsFor(controller)
if err != nil {
t.Fatal(err)
}
for _, s := range []string{
"mesh.mod.x.call.t.n.controller",
"mesh.mod.plex.call.plex_issued_credential.ace.controller",
} {
if !MayPublish(perms, s) {
t.Errorf("the controller may not publish %s", s)
}
}
for _, s := range []string{
"mesh.mod.x.call.t.n.ace.node-tools",
"mesh.mod.x.call.t.n." + CallerToken("ace.node-tools"),
"mesh.mod.plex.call.plex_issued_credential.ace." + CallerToken("ace.node-tools"),
"mesh.mod.plex.call.plex_status.ace.controller",
"mesh.mod.kometa.call.t.ace.controller",
"mesh.seat.mesh-controller.call.status.controller",
} {
if MayPublish(perms, s) {
t.Errorf("the controller may publish %s", s)
}
}
bare, err := PermissionsFor(Principal{Kind: KindController})
if err != nil {
t.Fatal(err)
}
if MayPublish(bare, "mesh.mod.x.call.t.n.controller") {
t.Fatal("a controller composed with no records is granted a caller-named call")
}
}
+18
View File
@@ -123,6 +123,13 @@ type Principal struct {
// permission and nothing beside it. // permission and nothing beside it.
Invokes []string Invokes []string
// AsksIssued are the tools the controller asks in its own name for a credential a provider's software
// issued, each `<module>.<tool>` (novox/hq ADR 0305): on the subject that names its caller, so the tool
// knows the controller asks. Composed from the records, so the installer's first user list — a controller
// with no records — carries none, and moves with issue 464 like the rest of the controller's caller-named
// grants.
AsksIssued []string
// State is the local names of the state this principal's module keeps, and Reads the state of // State is the local names of the state this principal's module keeps, and Reads the state of
// others it reads as `<module>.<name>` (novox/hq ADR 0201): a bucket each, kept by the owner's // others it reads as `<module>.<name>` (novox/hq ADR 0201): a bucket each, kept by the owner's
// instances and read by whoever declares it. // instances and read by whoever declares it.
@@ -900,6 +907,17 @@ func PermissionsFor(p Principal) (Permissions, error) {
// and moves with issue 464. // and moves with issue 464.
if p.Kind != KindController { if p.Kind != KindController {
pub, sub = callerNamed(p.Username(), pub, sub) pub, sub = callerNamed(p.Username(), pub, sub)
} else {
// **But the tools that answer a software-issued credential** (novox/hq ADR 0305 §2): those answer the
// controller alone, and know it only from the subject's last token, so it asks them in its own name —
// those tools, on any machine, and nothing else under `call`.
for _, key := range p.AsksIssued {
module, tool, ok := strings.Cut(key, ".")
if !ok || module == "" || tool == "" || strings.ContainsAny(key, "*> ") {
continue
}
pub = append(pub, "mesh.mod."+module+"."+CallKind+"."+tool+".*."+CallerToken(p.Username()))
}
} }
sort.Strings(pub) sort.Strings(pub)
+23 -1
View File
@@ -29,6 +29,9 @@ type Declared struct {
Holds []Seat Holds []Seat
// Uses are the seats this module sends to. // Uses are the seats this module sends to.
Uses []Seat Uses []Seat
// Issued are the tools that answer a credential its software issued, one per offer that says so (novox/hq
// ADR 0305): the controller is granted to ask each in its own name, and no other tool so.
Issued []string
// Watches are the seats whose events it consumes. // Watches are the seats whose events it consumes.
Watches []Seat Watches []Seat
// Invokes are the tools it calls, `<module>.<tool>` or `*` (novox/hq ADR 0152). // Invokes are the tools it calls, `<module>.<tool>` or `*` (novox/hq ADR 0152).
@@ -84,7 +87,7 @@ type Records struct {
// is a mesh that cannot be told anything, and there is no state of the records in which that is // is a mesh that cannot be told anything, and there is no state of the records in which that is
// correct. // correct.
func Users(r Records) ([]Principal, error) { func Users(r Records) ([]Principal, error) {
out := []Principal{{Kind: KindController, Uses: asksSeatOf(r)}} out := []Principal{{Kind: KindController, Uses: asksSeatOf(r), AsksIssued: issuedOf(r)}}
for _, node := range sortedCopy(r.Nodes) { for _, node := range sortedCopy(r.Nodes) {
witness := false witness := false
@@ -206,6 +209,25 @@ const controllerModule = "mesh-controller"
// asksSeatOf is the seat an ask is made on, as its holder declares it (novox/hq ADR 0259): the controller // asksSeatOf is the seat an ask is made on, as its holder declares it (novox/hq ADR 0259): the controller
// asks the operator through it like any other user, and is granted what its declaration names for a caller. // asks the operator through it like any other user, and is granted what its declaration names for a caller.
// None while nothing holds it. // None while nothing holds it.
// issuedOf is every tool of a module assigned anywhere that answers a credential its software issued, as
// `<module>.<tool>` (novox/hq ADR 0305): what the controller asks in its own name, and nothing beside.
func issuedOf(r Records) []string {
seen := map[string]bool{}
var out []string
for _, node := range sortedCopy(r.Nodes) {
for _, d := range r.Assigned[node] {
for _, t := range d.Issued {
if key := d.Module + "." + t; !seen[key] {
seen[key] = true
out = append(out, key)
}
}
}
}
sort.Strings(out)
return out
}
func asksSeatOf(r Records) []Seat { func asksSeatOf(r Records) []Seat {
for _, node := range sortedCopy(r.Nodes) { for _, node := range sortedCopy(r.Nodes) {
for _, d := range r.Assigned[node] { for _, d := range r.Assigned[node] {
@@ -79,23 +79,38 @@ func TestAHolderMeetsItsOwnContribution(t *testing.T) {
} }
} }
func TestTwoModulesDeclaringOnePackageAreRefusedBeforeAnythingIsRecorded(t *testing.T) { func TestTwoModulesMayDeclareOnePackage(t *testing.T) {
pacman := withResources(mod("pacman", nil, nil, nil, Claim{Name: PackageManagerSeat}), pacman := withResources(mod("pacman", nil, nil, nil, Claim{Name: PackageManagerSeat}),
res("package", "pacman-contrib")) res("package", "pacman-contrib"))
bar := withResources(mod("bar", nil, nil, nil), res("package", "bar"), res("package", "pacman-contrib")) bar := withResources(mod("bar", nil, nil, nil), res("package", "bar"), res("package", "pacman-contrib"))
other := withResources(mod("other", nil, nil, nil), res("package", "other")) catalogue := map[string]Manifest{"pacman": pacman, "bar": bar}
catalogue := map[string]Manifest{"pacman": pacman, "bar": bar, "other": other} if err := CollisionRefusal(catalogue, "laptop", []string{"pacman"}, []string{"bar"}); err != nil {
t.Fatalf("a second declaration of a package was refused (novox/hq ADR 0303): %v", err)
}
}
err := CollisionRefusal(catalogue, "laptop", []string{"pacman"}, []string{"bar"}) func TestAPackageDeclaredPresentAndAbsentIsRefusedNamingBoth(t *testing.T) {
if err == nil || !strings.Contains(err.Error(), "pacman-contrib") || !strings.Contains(err.Error(), "bar") { gone := res("package", "ufw")
t.Fatalf("the second owner of a package was not refused by name: %v", err) gone["absent"] = true
filter := withResources(mod("nftables", nil, nil, nil), gone)
wants := withResources(mod("legacy", nil, nil, nil), res("package", "ufw"))
other := withResources(mod("other", nil, nil, nil), res("package", "other"))
catalogue := map[string]Manifest{"nftables": filter, "legacy": wants, "other": other}
err := CollisionRefusal(catalogue, "laptop", []string{"nftables"}, []string{"legacy"})
if err == nil || !strings.Contains(err.Error(), "ufw") || !strings.Contains(err.Error(), "nftables") ||
!strings.Contains(err.Error(), "legacy") || !strings.Contains(err.Error(), "ADR 0303") {
t.Fatalf("a package declared present and absent was not refused naming both: %v", err)
} }
if err := CollisionRefusal(catalogue, "laptop", []string{"pacman"}, []string{"other"}); err != nil { if strings.Contains(err.Error(), "one owner per node") {
t.Errorf("a module declaring nothing shared is refused: %v", err) t.Errorf("the refusal gives the one-owner rule, which no longer holds for packages: %v", err)
} }
// A collision already on the node is status's, not a reason to refuse an unrelated assignment. if err := CollisionRefusal(catalogue, "laptop", []string{"nftables"}, []string{"other"}); err != nil {
if err := CollisionRefusal(catalogue, "laptop", []string{"pacman", "bar"}, []string{"other"}); err != nil { t.Errorf("a module declaring nothing contradictory is refused: %v", err)
t.Errorf("an unrelated assignment is refused for a collision already there: %v", err) }
// A contradiction already on the node is status's, not a reason to refuse an unrelated assignment.
if err := CollisionRefusal(catalogue, "laptop", []string{"nftables", "legacy"}, []string{"other"}); err != nil {
t.Errorf("an unrelated assignment is refused for a contradiction already there: %v", err)
} }
} }
+22 -2
View File
@@ -342,18 +342,38 @@ func (e *NotMadeError) Error() string {
// compose. Empty when every module composes. The same judgement SetSettings makes before storing. // compose. Empty when every module composes. The same judgement SetSettings makes before storing.
func (r Resolution) LeftOut(settings SettingsBy, adopted bool) map[string]string { func (r Resolution) LeftOut(settings SettingsBy, adopted bool) map[string]string {
out := map[string]string{} out := map[string]string{}
for module, why := range r.LeftOutBecause(settings, adopted) {
out[module] = why.Error()
}
return out
}
// LeftOutBecause is LeftOut with each reason as the error it was, so a reader can tell a setting nobody
// gave (an *UnsetSettingError) from any other cause and say it as the operator's to give (novox/hq issue
// 380). An UnreadManifestError for a stored manifest this controller cannot read whole.
func (r Resolution) LeftOutBecause(settings SettingsBy, adopted bool) map[string]error {
out := map[string]error{}
for _, m := range r.Modules { for _, m := range r.Modules {
if why := UnknownFieldReason(m); why != "" { if why := UnknownFieldReason(m); why != "" {
out[m.Module] = why out[m.Module] = &UnreadManifestError{Module: m.Module, said: why}
continue continue
} }
if err := JudgeSettings(m, settings[m.Module], adopted); err != nil { if err := JudgeSettings(m, settings[m.Module], adopted); err != nil {
out[m.Module] = err.Error() out[m.Module] = err
} }
} }
return out return out
} }
// UnreadManifestError is a module left out because its stored manifest has a key this controller does not know
// (novox/hq ADR 0262): said once for the whole mesh, by the catalogue's own condition, not per machine.
type UnreadManifestError struct {
Module string
said string
}
func (e *UnreadManifestError) Error() string { return e.said }
// Compose is Declaration with the owner of every resource said. // Compose is Declaration with the owner of every resource said.
func (r Resolution) Compose(with Rendering) (Composed, error) { func (r Resolution) Compose(with Rendering) (Composed, error) {
owner := map[string]string{} owner := map[string]string{}
+1 -1
View File
@@ -21,7 +21,7 @@ func deliveryVerbs() []Verb {
// Delivery time against the delivery budgets (novox/hq ADR 0282): what probe D16 reads, optional until its // Delivery time against the delivery budgets (novox/hq ADR 0282): what probe D16 reads, optional until its
// holder serves it. // holder serves it.
{Name: "times", Description: "Delivery time: from a merge to every machine of its walk running the build. " + {Name: "times", Description: "Delivery time: from a merge to every machine of its walk running the build. " +
"Each class's delivery budget (a leaf module five minutes, a core module ten), the last days' median and " + "Each class's delivery budget (a leaf module three minutes, a core module ten), the last days' median and " +
"worst, how many within and over, the newest delivery of each class, every delivery over its budget with its " + "worst, how many within and over, the newest delivery of each class, every delivery over its budget with its " +
"longest phase, and the delivered ones whose time is not known. Given a delivery's id, its time phase by phase.", "longest phase, and the delivered ones whose time is not known. Given a delivery's id, its time phase by phase.",
Input: schema(map[string]string{"days": "how many days back (default 7)", "id": "one delivery's id: its phases"}, nil), Input: schema(map[string]string{"days": "how many days back (default 7)", "id": "one delivery's id: its phases"}, nil),
+71
View File
@@ -0,0 +1,71 @@
package catalogue
import (
"go/ast"
"go/parser"
"go/token"
"path/filepath"
"slices"
"strconv"
"testing"
"github.com/novox/mesh-controller/internal/beside"
)
// The controller refuses a definition's resolved path by the node-engine's own rules, no more (novox/hq issue
// 496): the same lists, read here from mesh-host's internal/apply/placement_guard.go — in a merge check the clone
// beside it at the commit the mesh runs, elsewhere the copy captured in testdata/beside. When the engine's lists
// move, this fails until the controller's move with them, so the gate never refuses what the engine applies nor
// passes what it refuses by path alone.
func TestTheResolvedPathRulesAreTheNodeEnginesOwn(t *testing.T) {
file := filepath.Join(beside.Dir(t, "mesh-host"), "internal", "apply", "placement_guard.go")
parsed, err := parser.ParseFile(token.NewFileSet(), file, nil, 0)
if err != nil {
t.Fatalf("the node-engine's guard does not parse: %v", err)
}
lists := map[string][]string{}
consts := map[string]string{}
ast.Inspect(parsed, func(n ast.Node) bool {
spec, ok := n.(*ast.ValueSpec)
if !ok {
return true
}
for i, name := range spec.Names {
if i >= len(spec.Values) {
continue
}
switch v := spec.Values[i].(type) {
case *ast.CompositeLit:
for _, elt := range v.Elts {
if lit, ok := elt.(*ast.BasicLit); ok && lit.Kind == token.STRING {
s, _ := strconv.Unquote(lit.Value)
lists[name.Name] = append(lists[name.Name], s)
}
}
case *ast.BasicLit:
if v.Kind == token.STRING {
consts[name.Name], _ = strconv.Unquote(v.Value)
}
}
}
return true
})
for name, ours := range map[string][]string{
"protectedRoots": protectedRoots, "forbiddenBelow": forbiddenBelow, "engineTrees": engineTrees,
} {
theirs := lists[name]
if len(theirs) == 0 {
t.Errorf("the node-engine's guard names no %s any more; read it and say where its rule went", name)
continue
}
a, b := slices.Clone(ours), slices.Clone(theirs)
slices.Sort(a)
slices.Sort(b)
if !slices.Equal(a, b) {
t.Errorf("%s differs from the node-engine's:\n controller %v\n node-engine %v", name, a, b)
}
}
if consts["engineModule"] != engineModule {
t.Errorf("the node-engine's own module is %q there and %q here", consts["engineModule"], engineModule)
}
}
@@ -0,0 +1,42 @@
package catalogue
import (
"strings"
"testing"
)
// A provision's credential may be issued by its provider's software (novox/hq ADR 0305, issue 526): the offer
// names the tool that answers it and no own secret, and every other shape is refused at parse.
func TestAnOfferMaySayItsSoftwareIssuesTheCredential(t *testing.T) {
m, err := ParseManifest([]byte(`{"module":"media","version":"1",
"provides":[{"name":"media-api","scope":"mesh","credential":{"issued-by":"software","tool":"media_issued_credential"}}],
"serves":{"media-api":{"port":32400}}}`))
if err != nil {
t.Fatal(err)
}
if tool, issued := m.IssuedCredentialOf("media-api"); !issued || tool != "media_issued_credential" {
t.Fatalf("the offer's issued credential was not read: %q %v", tool, issued)
}
if _, shared := m.SharedCredentialOf("media-api"); shared {
t.Fatal("an issued credential was read as an own secret shared")
}
if _, issued := m.IssuedCredentialOf("other"); issued {
t.Fatal("a provision the module does not offer reads as issued")
}
for want, raw := range map[string]string{
"the one issuer an offer may name": `{"module":"d","version":"1",
"provides":[{"name":"d-api","credential":{"issued-by":"vendor","tool":"d_token"}}]}`,
"names no tool that answers it": `{"module":"d","version":"1",
"provides":[{"name":"d-api","credential":{"issued-by":"software"}}]}`,
"names its own secret": `{"module":"d","version":"1","own-secrets":{"token":{"path":"/p","taken":"at-start"}},
"provides":[{"name":"d-api","credential":{"issued-by":"software","tool":"d_token","own":"token"}}]}`,
"the one issuer an offer may name ": `{"module":"d","version":"1",
"provides":[{"name":"d-api","credential":{"tool":"d_token"}}]}`,
} {
want = strings.TrimSpace(want)
if _, err := ParseManifest([]byte(raw)); err == nil || !strings.Contains(err.Error(), want) {
t.Errorf("expected a refusal saying %q, got %v", want, err)
}
}
}
+41 -3
View File
@@ -268,9 +268,29 @@ func KeepsConsumerData(catalogue map[string]Manifest, provision string) bool {
// MachineReach is whether a provision is usable only on its provider's own machine. // MachineReach is whether a provision is usable only on its provider's own machine.
func (o Offer) MachineReach() bool { return o.Reach == ReachMachine } func (o Offer) MachineReach() bool { return o.Reach == ReachMachine }
// OfferCredential names which of the provider's own secrets a provision's consumers receive. // OfferCredential names which of the provider's own secrets a provision's consumers receive — or, with
// IssuedBy, that the provider's software issues the provision's one credential itself (novox/hq ADR 0305,
// issue 526): a media server's sign-in token from its vendor, which the mesh can neither make nor rotate.
// Tool is then the provider's own tool that answers that value, sealed to a key made for the controller's
// one call, and to the controller alone.
type OfferCredential struct { type OfferCredential struct {
Own string `json:"own"` Own string `json:"own,omitempty"`
IssuedBy string `json:"issued-by,omitempty"`
Tool string `json:"tool,omitempty"`
}
// IssuedBySoftware says a provision's credential is issued by its provider's software (novox/hq ADR 0305).
const IssuedBySoftware = "software"
// IssuedCredentialOf is the tool through which this module answers the credential its software issued for a
// provision, and whether its offer says the software issues it (novox/hq ADR 0305).
func (m Manifest) IssuedCredentialOf(provision string) (string, bool) {
for _, o := range m.Provides {
if o.Name == provision && o.Credential != nil && o.Credential.IssuedBy == IssuedBySoftware && o.Credential.Tool != "" {
return o.Credential.Tool, true
}
}
return "", false
} }
// SharedCredentialOf is the own secret an offer of this module names as the provision's credential, // SharedCredentialOf is the own secret an offer of this module names as the provision's credential,
@@ -1555,7 +1575,24 @@ func ParseManifest(raw []byte) (Manifest, error) {
"makes is %d", m.Module, p, id.Max, shortest)) "makes is %d", m.Module, p, id.Max, shortest))
} }
} }
if offer.Credential != nil { if c := offer.Credential; c != nil && (c.IssuedBy != "" || c.Tool != "") {
// The software issues it (novox/hq ADR 0305): named by the tool that answers it, never by an own
// secret, which would be a second copy for the mesh to keep in step with the software's.
switch {
case c.IssuedBy != IssuedBySoftware:
problems = append(problems, fmt.Sprintf(
"%s provides %q with a credential issued by %q; the one issuer an offer may name is %q (ADR 0305)",
m.Module, p, c.IssuedBy, IssuedBySoftware))
case c.Own != "":
problems = append(problems, fmt.Sprintf(
"%s provides %q with a credential its software issues and names its own secret %q too; the "+
"software's value is the one, so the offer names no own secret (ADR 0305)", m.Module, p, c.Own))
case strings.TrimSpace(c.Tool) == "":
problems = append(problems, fmt.Sprintf(
"%s provides %q with a credential its software issues and names no tool that answers it: "+
"\"tool\": \"<its tool>\" (ADR 0305)", m.Module, p))
}
} else if offer.Credential != nil {
own, declared := m.OwnSecrets[offer.Credential.Own] own, declared := m.OwnSecrets[offer.Credential.Own]
switch { switch {
case offer.Credential.Own == "": case offer.Credential.Own == "":
@@ -2145,6 +2182,7 @@ func ParseManifest(raw []byte) (Manifest, error) {
problems = append(problems, m.undeclaredMounts()...) problems = append(problems, m.undeclaredMounts()...)
problems = append(problems, m.unknownDirRefs()...) problems = append(problems, m.unknownDirRefs()...)
problems = append(problems, m.unknownAccessRefs()...) problems = append(problems, m.unknownAccessRefs()...)
problems = append(problems, m.resolvedPathProblems()...)
problems = append(problems, m.jailProblems()...) problems = append(problems, m.jailProblems()...)
// What a module adds to the account's environment and to the login shell, and the holder's // What a module adds to the account's environment and to the login shell, and the holder's
// placeholders for them (novox/hq ADR 0203, ADR 0204) — here, so the catalogue check refuses // placeholders for them (novox/hq ADR 0203, ADR 0204) — here, so the catalogue check refuses
+99
View File
@@ -92,6 +92,105 @@ func systemPath(path string) string {
return "" return ""
} }
// Where no directory or file a module's definition resolves to may be (novox/hq issue 496).
//
// mesh-catalog #205 gave docker's `state` directory `"place": "."`, which resolves to <root>/docker: with the
// default root, /var/lib/docker, every container's filesystem. systemPath judged only the `places` and `accesses`
// settings, so the default layout and a definition's own paths reached the merge gate unjudged; it composed every
// machine and passed, and only the node-engine refused the directory, at apply, failing the walk.
//
// **Judged where a definition is judged, never where a machine is composed.** resolvedPathProblems runs in
// ParseManifest, which every route a definition takes into the mesh passes: `module check`, registration of a
// build (the builder's and the registry verbs'), and the merge gate's reading of the changed repository. A
// refusal there stops one definition before it reaches any machine. Composition reads registered manifests
// without ParseManifest, and judges nothing of this: refusing there would fail the whole machine's declaration and
// freeze every module on it for one module's path, where the node-engine fails only that resource.
//
// **The node-engine's rules, no more** (mesh-host's internal/apply/placement_guard.go, with files judged as
// directories are after novox/hq issue 495, rule 6). A path is refused when it is one of protectedRoots or holds
// one, when it is at or below a tree in forbiddenBelow, or at or below one of engineTrees and its module is not
// the node-engine's. What the engine judges with what only the machine knows stays the engine's: where the
// runtimes really keep their data, links, the accounts' homes, a directory's owner below /etc. The lists are the
// engine's own words, and a test (engine_guard_test.go) holds them equal to mesh-host's beside this repository.
// systemPath stays the stricter rule for a setting: a setting is an operator's word about one machine, and the
// trees it lists (/etc, /usr, /run, the mesh's own) are where the mesh's own modules write by design.
var (
protectedRoots = []string{"/", "/bin", "/boot", "/dev", "/etc", "/home", "/lib", "/lib32", "/lib64",
"/media", "/mnt", "/opt", "/proc", "/root", "/run", "/sbin", "/srv", "/sys", "/tmp", "/usr", "/usr/bin",
"/usr/lib", "/usr/lib64", "/usr/local", "/usr/local/bin", "/usr/local/lib", "/usr/local/sbin", "/usr/sbin",
"/usr/share", "/var", "/var/cache", "/var/lib", "/var/lib/mesh", "/var/log", "/var/run", "/var/tmp",
"/var/spool"}
forbiddenBelow = []string{"/proc", "/sys", "/dev", "/boot", "/root", "/var/spool", "/opt", "/var/lib/docker",
"/var/lib/containers", "/var/lib/containerd"}
engineTrees = []string{"/var/lib/mesh-host", "/usr/lib/nox-mesh-host"}
)
// engineModule is the node-engine's own module, the one that places in engineTrees.
const engineModule = "mesh-host"
// enginePath says why the node-engine refuses a directory or file at path for module, or "".
func enginePath(path, module string) string {
path = filepath.Clean(path)
if !filepath.IsAbs(path) {
return ""
}
atOrBelow := func(tree string) bool { return path == tree || strings.HasPrefix(path, tree+"/") }
for _, root := range protectedRoots {
if path == root || path == "/" || strings.HasPrefix(root, path+"/") {
return root + " is one of the machine's own directories, and owning it is owning everything in it"
}
}
for _, tree := range forbiddenBelow {
if atOrBelow(tree) {
return "nothing is placed in " + tree
}
}
if module != engineModule {
for _, tree := range engineTrees {
if atOrBelow(tree) {
return tree + " is the node-engine's own, placed in by its own module alone"
}
}
}
return ""
}
// resolvedPathProblems is every directory and file of the definition whose path, resolved as a node with the
// default root resolves it, the node-engine would refuse (novox/hq issue 496). A path still holding a placeholder
// only a machine fills (a setting, an access the definition gives no default) is the engine's to judge.
func (m Manifest) resolvedPathProblems() []string {
dirs := dirsFor(m, Rendering{})
accesses := map[string]string{}
for _, a := range m.Accesses {
if a.ID != "" && a.Path != "" {
accesses[a.ID] = a.Path
}
}
var problems []string
for _, r := range m.Resources {
kind := fmt.Sprint(r["type"])
if kind != "directory" && kind != "file" {
continue
}
id := fmt.Sprint(r["id"])
path, _ := r["path"].(string)
if kind == "directory" && path == "" {
path = dirs[id]
}
path, _ = dirFill(path, dirs, m.Module)
path, _ = accessFill(path, accesses, m.Module)
if path == "" || strings.Contains(path, "${") {
continue
}
if why := enginePath(path, m.Module); why != "" {
problems = append(problems, fmt.Sprintf("%s's %s %q resolves to %s, which the node-engine refuses: %s. "+
"A module's directories and files are judged when its definition is, so the merge gate refuses it "+
"before any machine does (novox/hq issue 496)", m.Module, kind, id, filepath.Clean(path), why))
}
}
return problems
}
// accessRef is how a module names one of its accesses: ${access:<id>}. // accessRef is how a module names one of its accesses: ${access:<id>}.
var accessRef = regexp.MustCompile(`\$\{access:([a-z0-9][a-z0-9-]*)\}`) var accessRef = regexp.MustCompile(`\$\{access:([a-z0-9][a-z0-9-]*)\}`)
+33 -1
View File
@@ -960,6 +960,7 @@ func checkResources(modules []Manifest) []string {
var problems []string var problems []string
owner := map[string]string{} owner := map[string]string{}
ownedPath := map[string]string{} // path → owning module, for the access check below ownedPath := map[string]string{} // path → owning module, for the access check below
packagePresent, packageAbsent := map[string][]string{}, map[string][]string{}
for _, m := range modules { for _, m := range modules {
// Compared placed, not as written (novox/hq ADR 0112): ${dir:state} is the same six // Compared placed, not as written (novox/hq ADR 0112): ${dir:state} is the same six
@@ -992,7 +993,22 @@ func checkResources(modules []Manifest) []string {
} }
continue continue
} }
for _, field := range []string{"path", "unit", "name", "package"} { if fmt.Sprint(r["type"]) == "package" {
// **A package may be declared by several modules** (novox/hq ADR 0303): declaring it says
// it is needed, and the host installs a missing one, never removes one a declaration
// stopped naming, so a second declaration changes nothing on the machine. What two
// modules can contradict is presence: one declaring it present and another absent
// (ADR 0180) would have the host install and remove it in turn, so that pair is refused.
if name, _ := r["package"].(string); name != "" {
if r["absent"] == true {
packageAbsent[name] = append(packageAbsent[name], m.Module)
} else {
packagePresent[name] = append(packagePresent[name], m.Module)
}
}
continue
}
for _, field := range []string{"path", "unit", "name"} {
value, ok := r[field].(string) value, ok := r[field].(string)
if !ok || value == "" { if !ok || value == "" {
if field == "path" && fmt.Sprint(r["type"]) == "directory" { if field == "path" && fmt.Sprint(r["type"]) == "directory" {
@@ -1038,6 +1054,12 @@ func checkResources(modules []Manifest) []string {
ownedPath[fact.Path] = m.Module ownedPath[fact.Path] = m.Module
} }
} }
for _, name := range packageNames(packageAbsent) {
if wanted := packagePresent[name]; len(wanted) > 0 {
problems = append(problems, fmt.Sprintf("%s declares the package %q absent and %s declare it present (novox/hq ADR 0303)",
strings.Join(packageAbsent[name], ", "), name, strings.Join(wanted, ", ")))
}
}
// An accessed path is the operator's, so no module may declare it as one of its own // An accessed path is the operator's, so no module may declare it as one of its own
// (novox/hq ADR 0051). Refused here rather than silently tolerated: an owner would create and // (novox/hq ADR 0051). Refused here rather than silently tolerated: an owner would create and
@@ -1308,3 +1330,13 @@ func answeredElsewhere(want string, node Node, world World, brokered, keeps map[
holder, held := HolderAmong(want, world.Offered[want], world.Held) holder, held := HolderAmong(want, world.Offered[want], world.Held)
return held && holder.Node != node.Name return held && holder.Node != node.Name
} }
// packageNames are the packages a map names, in order.
func packageNames(m map[string][]string) []string {
names := make([]string, 0, len(m))
for n := range m {
names = append(names, n)
}
sort.Strings(names)
return names
}
+127
View File
@@ -0,0 +1,127 @@
package catalogue
// A definition's directories and files are judged at their resolved paths when the definition is (novox/hq issue
// 496). mesh-catalog #205 gave docker's `state` directory `"place": "."`, which resolves to <root>/docker —
// /var/lib/docker, every container's filesystem. The merge gate composed every machine and passed it; only the
// node-engine refused it, at apply, and failed the walk. ParseManifest is what `module check`, registration and the
// merge gate's reading of a changed repository all run, so a refusal here is a refusal at each of them.
import (
"strings"
"testing"
)
func TestADirectoryPlacedInDockersDataIsRefusedWhereTheDefinitionIsJudged(t *testing.T) {
// The #205 shape, as it was merged.
_, err := ParseManifest([]byte(`{"module": "docker", "version": "1",
"resources": [{"id": "state", "type": "directory", "place": "."}]}`))
if err == nil {
t.Fatal("a directory resolving to /var/lib/docker was accepted; the node-engine refuses it at apply")
}
for _, said := range []string{"docker", `"state"`, "/var/lib/docker", "issue 496"} {
if !strings.Contains(err.Error(), said) {
t.Errorf("the refusal names the module, the resource, the path and why; %q is missing from %q", said, err)
}
}
}
func TestTheMeshsPlaceForDockerIsAccepted(t *testing.T) {
// The fix #205 needed: the mesh's own directory for the module, <root>/mesh/docker.
m, err := ParseManifest([]byte(`{"module": "docker", "version": "1", "resources": [
{"id": "state", "type": "directory", "place": "mesh"},
{"id": "marker", "type": "file", "path": "${dir:state}/applied", "content": "x"}]}`))
if err != nil {
t.Fatalf("place %q is in the mesh's tree, where the mesh writes for every module: %v", "mesh", err)
}
if got := dirsFor(m, Rendering{}); got["state"] != "/var/lib/mesh/docker" {
t.Fatalf("got %v", got)
}
}
// The node-engine's rules, for directories and — as the engine judges them since novox/hq issue 495 — files.
func TestADefinitionIsRefusedWhereTheNodeEngineRefusesItsPaths(t *testing.T) {
refused := map[string]string{
"a stated directory in docker's data": `{"module": "sidecar", "version": "1", "resources": [
{"id": "volumes", "type": "directory", "path": "/var/lib/docker/volumes/x"}]}`,
"a file in containerd's data": `{"module": "images", "version": "1", "resources": [
{"id": "f", "type": "file", "path": "/var/lib/containerd/x", "content": "x"}]}`,
"a file beneath a placed directory that climbs out of it": `{"module": "docker", "version": "1", "resources": [
{"id": "state", "type": "directory", "place": "mesh"},
{"id": "f", "type": "file", "path": "${dir:state}/../../containers/x", "content": "x"}]}`,
"a file in /boot": `{"module": "grub", "version": "1", "resources": [
{"id": "cfg", "type": "file", "path": "/boot/grub/custom.cfg", "content": "x"}]}`,
"a directory that is the mesh's whole tree": `{"module": "mesh", "version": "1", "resources": [
{"id": "all", "type": "directory", "place": "."}]}`,
"a directory that holds /etc": `{"module": "x", "version": "1", "resources": [
{"id": "d", "type": "directory", "path": "/"}]}`,
"a directory in the node-engine's own tree, by another module": `{"module": "intruder", "version": "1",
"resources": [{"id": "d", "type": "directory", "path": "/var/lib/mesh-host/x"}]}`,
}
for name, raw := range refused {
if _, err := ParseManifest([]byte(raw)); err == nil || !strings.Contains(err.Error(), "issue 496") {
t.Errorf("%s: accepted, or refused for another reason: %v", name, err)
}
}
// What the engine applies, the mesh's own modules' paths among them (read from every machine's live plan):
// the machine's configuration, run directories, programs below /usr/local, the node-engine's own trees by its
// own module, an account's keys, a module's own root, and — not on the engine's lists, so not refused here —
// below /lib and at /storage, /data, /services and /var/lock.
accepted := map[string]string{
"a unit in /etc": `{"module": "power", "version": "1", "resources": [
{"id": "d", "type": "directory", "path": "/etc/systemd/system/x.service.d"},
{"id": "u", "type": "file", "path": "/etc/systemd/system/x.service.d/a.conf", "content": "x"}]}`,
"a run directory": `{"module": "fail2ban", "version": "1", "resources": [
{"id": "run-dir", "type": "directory", "path": "/var/run/fail2ban"}]}`,
"a program in /usr/local/bin": `{"module": "claude-code", "version": "1", "resources": [
{"id": "start", "type": "file", "path": "/usr/local/bin/claude-agent", "content": "x"}]}`,
"the node-engine's launcher and state, by the node-engine": `{"module": "mesh-host", "version": "1", "resources": [
{"id": "launcher", "type": "file", "path": "/usr/lib/nox-mesh-host/launch", "content": "x"},
{"id": "state", "type": "directory", "path": "/var/lib/mesh-host"}]}`,
"an account's .ssh": `{"module": "ssh-client", "version": "1", "resources": [
{"id": "ssh-dir", "type": "directory", "path": "/home/someone/.ssh"},
{"id": "config", "type": "file", "path": "/home/someone/.ssh/config", "content": "x"}]}`,
"a module's own root": `{"module": "mailu", "version": "1", "resources": [
{"id": "state", "type": "directory", "place": "."}]}`,
"a file below /lib": `{"module": "udev", "version": "1", "resources": [
{"id": "rule", "type": "file", "path": "/lib/udev/rules.d/99-x.rules", "content": "x"}]}`,
"directories at /storage, /data, /services and /var/lock": `{"module": "roots", "version": "1", "resources": [
{"id": "a", "type": "directory", "path": "/storage"}, {"id": "b", "type": "directory", "path": "/data"},
{"id": "c", "type": "directory", "path": "/services"}, {"id": "d", "type": "directory", "path": "/var/lock"}]}`,
}
for name, raw := range accepted {
if _, err := ParseManifest([]byte(raw)); err != nil {
t.Errorf("%s: refused: %v", name, err)
}
}
}
func TestAPathThroughAnAccessIsJudgedWithTheAccessFilledIn(t *testing.T) {
// A file's path may name an access; the access's default path is the definition's, so it is judged with it.
_, err := ParseManifest([]byte(`{"module": "backup", "version": "1",
"accesses": [{"id": "images", "path": "/var/lib/docker/volumes"}],
"resources": [{"id": "marker", "type": "file", "path": "${access:images}/marker", "content": "x"}]}`))
if err == nil || !strings.Contains(err.Error(), "/var/lib/docker/volumes/marker") ||
!strings.Contains(err.Error(), "issue 496") {
t.Fatalf("a file reaching Docker's data through an access's default path was accepted: %v", err)
}
// An access the definition gives no path is placed by a setting, which Places and AccessPlaces judge, and on
// the machine by the node-engine: nothing here to resolve it against, so nothing is refused for it.
if _, err := ParseManifest([]byte(`{"module": "backup", "version": "1",
"accesses": [{"id": "images"}],
"resources": [{"id": "marker", "type": "file", "path": "${access:images}/marker", "content": "x"}]}`)); err != nil {
t.Fatalf("an access placed only by a setting cannot be judged at the definition: %v", err)
}
}
func TestComposingAMachineIsNeverStoppedByOneModulesPath(t *testing.T) {
// A module registered from outside the catalogue never passes the gate. Refusing its path while a machine's
// declaration is composed would fail the whole declaration and freeze every module on that machine; the
// node-engine fails only the one resource. So composition leaves it to the engine.
m := Manifest{Module: "docker", Version: "1", Resources: []map[string]any{
{"id": "state", "type": "directory", "place": "."},
}}
if _, err := (Resolution{Node: "anchor", Modules: []Manifest{m}}).Declaration(Rendering{}); err != nil {
t.Fatalf("the machine's declaration failed for one module's path: %v", err)
}
}
+4
View File
@@ -349,6 +349,10 @@ func CollisionRefusal(catalogue map[string]Manifest, node string, assigned, addi
if before[p] { if before[p] {
continue // on the node already; not this assignment's doing continue // on the node already; not this assignment's doing
} }
if strings.Contains(p, "ADR 0303") {
problems = append(problems, p) // a package declared present and absent says its own rule
continue
}
problems = append(problems, p+" (novox/hq ADR 0210: one owner per node; the other module "+ problems = append(problems, p+" (novox/hq ADR 0210: one owner per node; the other module "+
"depends on the owner's seat instead)") "depends on the owner's seat instead)")
} }
+15 -4
View File
@@ -38,6 +38,17 @@ func settingsUsed(content string) []string {
return keys return keys
} }
// UnsetSettingError is a module whose definition says ${setting:<key>} where nothing sets that key: typed, so
// that whoever reads why a module was left out of a machine can tell a setting nobody gave — the operator's
// to give, named with the command that gives it — from any other reason (novox/hq issue 380). Its words are
// the refusal's, unchanged.
type UnsetSettingError struct {
Module, Setting string
said string
}
func (e *UnsetSettingError) Error() string { return e.said }
// settingInto fills a file's ${setting:…} placeholders from the layers over a module. // settingInto fills a file's ${setting:…} placeholders from the layers over a module.
// //
// The last layer setting a key wins, which is the node's over the mesh's over the module's own // The last layer setting a key wins, which is the node's over the mesh's over the module's own
@@ -58,12 +69,12 @@ func settingInto(resource map[string]any, layers []Layer, module string) error {
for _, key := range settingsUsed(content) { for _, key := range settingsUsed(content) {
value, set := settingValue(layers, key) value, set := settingValue(layers, key)
if !set { if !set {
return fmt.Errorf( return &UnsetSettingError{Module: module, Setting: key, said: fmt.Sprintf(
"%s has a file that says ${setting:%s}, and nothing sets %q for it — an operator's "+ "%s has a file that says ${setting:%s}, and nothing sets %q for it — an operator's "+
"value is the assignment's, never the definition's (novox/hq ADR 0112), and only a "+ "value is the assignment's, never the definition's (novox/hq ADR 0112), and only a "+
"preference has a default in the definition (ADR 0262): "+ "preference has a default in the definition (ADR 0262): "+
"`settings set %s <file>` with {%q: …}%s", "`settings set %s <file>` with {%q: …}%s",
module, key, key, module, key, orNoSettings(layers)) module, key, key, module, key, orNoSettings(layers))}
} }
content = strings.ReplaceAll(content, "${setting:"+key+"}", plainly(value)) content = strings.ReplaceAll(content, "${setting:"+key+"}", plainly(value))
} }
@@ -114,11 +125,11 @@ func settingIntoUnit(resource map[string]any, layers []Layer, module string) err
for _, key := range settingsUsed(unit) { for _, key := range settingsUsed(unit) {
value, set := settingValue(layers, key) value, set := settingValue(layers, key)
if !set { if !set {
return fmt.Errorf( return &UnsetSettingError{Module: module, Setting: key, said: fmt.Sprintf(
"%s has a service whose unit says ${setting:%s}, and nothing sets %q for it — an operator's "+ "%s has a service whose unit says ${setting:%s}, and nothing sets %q for it — an operator's "+
"value is the assignment's, never the definition's (novox/hq ADR 0112): "+ "value is the assignment's, never the definition's (novox/hq ADR 0112): "+
"`settings set %s <file>` with {%q: …}%s", "`settings set %s <file>` with {%q: …}%s",
module, key, key, module, key, orNoSettings(layers)) module, key, key, module, key, orNoSettings(layers))}
} }
v := plainly(value) v := plainly(value)
if !unitPart.MatchString(v) { if !unitPart.MatchString(v) {
+17 -2
View File
@@ -160,8 +160,14 @@ var ControllerVerbs = []Verb{
Input: schema(map[string]string{"plan": "the walk's id", "why": "what lets it go: its turn, a person's release"}, Input: schema(map[string]string{"plan": "the walk's id", "why": "what lets it go: its turn, a person's release"},
[]string{"plan"})}, []string{"plan"})},
{Name: "delivery-stop", Description: "End a walk on its delivery's word (novox/hq ADR 0239): failed, said as stopped " + {Name: "delivery-stop", Description: "End a walk on its delivery's word (novox/hq ADR 0239): failed, said as stopped " +
"by whom and why; what it asked still builds and registers, nothing further is asked or sent.", "by whom and why; what it asked still builds and registers, nothing further is asked or sent. Given merges to " +
Input: schema(map[string]string{"plan": "the walk's id", "why": "why", "by": "who stopped the delivery"}, "leave out, every other merge the walk answered is walked again — in the open batch of its kind, or a new one — " +
"without them, without each later merge of their repository, which contains one, and without each merge moving " +
"a module one moves, each left out too and named (novox/hq ADR 0299); a batch not yet cut drops the merges. The " +
"answer names the batch, the merges walked again, the merges left out, the modules the stopped walk sent that " +
"stay on their machines, and the merges elsewhere that will still deliver a stopped change.",
Input: schema(map[string]string{"plan": "the walk's id", "why": "why", "by": "who stopped the delivery",
"without": "merges to leave out, <repository>@<commit>, comma-separated"},
[]string{"plan", "why"})}, []string{"plan", "why"})},
{Name: "delivery-walks", Description: "The walks the controller keeps (novox/hq ADR 0239): every open one and the " + {Name: "delivery-walks", Description: "The walks the controller keeps (novox/hq ADR 0239): every open one and the " +
"last ended ones, each whole — its tiers, each module's state, first machines and first-node gate with its readings, and its " + "last ended ones, each whole — its tiers, each module's state, first machines and first-node gate with its readings, and its " +
@@ -243,6 +249,15 @@ var ControllerVerbs = []Verb{
"secret": "the own secret's name in the module's definition", "secret": "the own secret's name in the module's definition",
"at": "the machine the operator sits at, where the prompt opens", "at": "the machine the operator sits at, where the prompt opens",
}, []string{"node", "module", "secret", "at"})}, }, []string{"node", "module", "secret", "at"})},
{Name: "issued", Description: "Take the credential a provider's software issued for a provision — a media " +
"server's sign-in token — and keep it as that provision's one credential for every consumer (novox/hq ADR " +
"0305): the controller asks the provider's own tool on its machine, in the controller's own name, for the " +
"value sealed to a key made for this call alone, seals it to every holder and forgets it, and sends their " +
"machines when it changed. Carries no value and answers none; a value every holder already holds is not " +
"sealed again, so asking again restarts nothing. Refused for a provision no offer says its software issues.",
Input: schema(map[string]string{
"provision": "the provision whose provider's software issues its credential, e.g. plex-api",
}, []string{"provision"})},
{Name: "secret-ask", Description: "Ask the operator for a module's own secret (novox/hq ADR 0277): a prompt " + {Name: "secret-ask", Description: "Ask the operator for a module's own secret (novox/hq ADR 0277): a prompt " +
"that shows nothing of what is typed opens at the desk — the module's machine, or the one at names — " + "that shows nothing of what is typed opens at the desk — the module's machine, or the one at names — " +
"naming the module, the secret and who asked; the operator types the value there, never on a channel " + "naming the module, the secret and who asked; the operator types the value there, never on a channel " +
+15
View File
@@ -3,6 +3,7 @@ package inventory
import ( import (
"context" "context"
"fmt" "fmt"
"slices"
"strings" "strings"
"github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/broker"
@@ -172,6 +173,9 @@ func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaratio
Checks: catalogue.HealthChecks(m), Checks: catalogue.HealthChecks(m),
// And whether it runs as an account of its own (novox/hq ADR 0259 §8). // And whether it runs as an account of its own (novox/hq ADR 0259 §8).
RunsAs: m.RunsAs, RunsAs: m.RunsAs,
// And the tools that answer a credential its software issued, which the controller asks in its own
// name (novox/hq ADR 0305).
Issued: issuedTools(m),
} }
// Whether it can be given an account at all: delivered as its own secret named broker, so one // Whether it can be given an account at all: delivered as its own secret named broker, so one
// that declares none has nowhere to read it (novox/hq issue 195). // that declares none has nowhere to read it (novox/hq issue 195).
@@ -199,6 +203,17 @@ func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaratio
return d return d
} }
// issuedTools are the tools through which a module answers the credentials its software issues (novox/hq ADR 0305).
func issuedTools(m catalogue.Manifest) []string {
var out []string
for _, o := range m.Provides {
if tool, issued := m.IssuedCredentialOf(o.Name); issued && !slices.Contains(out, tool) {
out = append(out, tool)
}
}
return out
}
// bucketsOf is the state a module keeps, as the bus holds it. // bucketsOf is the state a module keeps, as the bus holds it.
func bucketsOf(m catalogue.Manifest) []broker.Bucket { func bucketsOf(m catalogue.Manifest) []broker.Bucket {
var out []broker.Bucket var out []broker.Bucket
+155
View File
@@ -0,0 +1,155 @@
package inventory
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"sort"
"strings"
"github.com/novox/mesh-controller/internal/secrets"
)
// A credential a provider's software issued, kept as the provision's one credential (novox/hq ADR 0305, issue
// 526).
//
// **The software made it, so the mesh carries it.** A media server's sign-in token comes from its vendor; the mesh
// can neither make nor rotate it. The controller fetches it from the provider's own tool, sealed to a key made for
// that one call, and hands the value here, which seals it to every consumer that holds the provision from that
// provider — in the pair credential's shape, so a consumer's definition does not know where it came from (ADR 0158
// §5) — and to the operator, and forgets it.
//
// **Asked often, written seldom.** The provider asks hourly and on every start. The mesh keeps no plaintext, so it
// cannot compare values; it stamps every row with the value's fingerprint instead, and a value whose stamp every
// row already carries is not sealed again — sealing anew would change every consumer's declaration and restart
// each of them for nothing.
// OriginIssued is a pair credential the provider's software issued and the provider gave (novox/hq ADR 0305).
const OriginIssued = "issued"
// IssuedStamp is the generation a value issued by a provider's software is kept under: a fingerprint of the value,
// so the same value fetched again is known as the same without being kept.
func IssuedStamp(value string) string {
sum := sha256.Sum256([]byte(value))
return "issued:sha256:" + hex.EncodeToString(sum[:8])
}
// IssuedKept is what keeping an issued value did: whether it was sealed anew, the pairs that hold it, and the
// machines a send must reach — the provider's and every holder's — when it was.
type IssuedKept struct {
Changed bool
// Holders are every pair that holds the provision from the provider; Sealed those sealed anew by this act.
Holders []Holder
Sealed []Holder
Machines []string
// Unsealable are the holders whose machine has no sealing key: left as they are, said by name.
Unsealable []Holder
}
// KeepIssuedSecret seals a value a provider's software issued to every pair of provision from provider that does
// not hold it yet, under one stamp; a pair that does is left as it is, and when every pair does nothing is written
// (novox/hq ADR 0305 §4, §5).
func (i *Inventory) KeepIssuedSecret(ctx context.Context, provision, provider, value string) (IssuedKept, error) {
if strings.TrimSpace(value) == "" {
return IssuedKept{}, fmt.Errorf("there is nothing to seal")
}
providerNode, err := i.NodeByName(ctx, provider)
if err != nil {
return IssuedKept{}, err
}
providerKey, err := i.SealingKeyOf(ctx, provider)
if err != nil {
return IssuedKept{}, err
}
if providerKey == "" {
return IssuedKept{}, fmt.Errorf("%s has no sealing key, so %s cannot be sealed to it", provider, provision)
}
stamp := IssuedStamp(value)
type row struct {
holder Holder
consumerID any
generation, heldC, heldP string
consumerKey string
}
rows, err := i.store.Pool().Query(ctx,
`select s.consumer, c.name, s.consumer_module, s.local, coalesce(s.generation, ''),
coalesce(s.consumer_key, ''), coalesce(s.provider_key, ''), coalesce(c.sealing_key, '')
from secret s join node c on c.id = s.consumer
where s.provider = $1 and s.name = $2
order by c.name, s.consumer_module, s.local`, providerNode.ID, provision)
if err != nil {
return IssuedKept{}, err
}
var held []row
for rows.Next() {
var r row
if err := rows.Scan(&r.consumerID, &r.holder.Consumer, &r.holder.ConsumerModule, &r.holder.Local,
&r.generation, &r.heldC, &r.heldP, &r.consumerKey); err != nil {
rows.Close()
return IssuedKept{}, err
}
r.holder.Provision, r.holder.Provider = provision, provider
held = append(held, r)
}
rows.Close()
if err := rows.Err(); err != nil {
return IssuedKept{}, err
}
var kept IssuedKept
var stale []row
for _, r := range held {
if r.consumerKey == "" {
kept.Unsealable = append(kept.Unsealable, r.holder)
continue
}
kept.Holders = append(kept.Holders, r.holder)
// Only a pair that does not hold this value, sealed to the keys its ends have now, is sealed anew: a
// holder that already does is left byte for byte as it is, so it is not restarted for another's sake.
if r.generation != stamp || r.heldC != r.consumerKey || r.heldP != providerKey {
stale = append(stale, r)
}
}
if len(stale) == 0 {
return kept, nil
}
forOperator, operatorKey, err := i.operatorSeal(ctx, value)
if err != nil {
return IssuedKept{}, err
}
tx, err := i.store.Pool().Begin(ctx)
if err != nil {
return IssuedKept{}, err
}
defer func() { _ = tx.Rollback(ctx) }()
machines := map[string]bool{provider: true}
for _, r := range stale {
sealed, err := secrets.Accept(value, r.consumerKey, providerKey)
if err != nil {
return IssuedKept{}, err
}
if _, err := tx.Exec(ctx,
`update secret set for_consumer = $6, for_provider = $7, consumer_key = $8, provider_key = $9,
operator_sealed = $10, operator_key = $11, origin = $12, generation = $13, created_at = now()
where name = $1 and consumer = $2 and consumer_module = $3 and provider = $4 and local = $5`,
provision, r.consumerID, r.holder.ConsumerModule, providerNode.ID, r.holder.Local,
sealed.ForConsumer, sealed.ForProvider, sealed.ConsumerKey, sealed.ProviderKey,
forOperator, operatorKey, OriginIssued, stamp); err != nil {
return IssuedKept{}, err
}
machines[r.holder.Consumer] = true
kept.Sealed = append(kept.Sealed, r.holder)
}
if err := tx.Commit(ctx); err != nil {
return IssuedKept{}, err
}
kept.Changed = true
for m := range machines {
kept.Machines = append(kept.Machines, m)
}
sort.Strings(kept.Machines)
return kept, nil
}
+178
View File
@@ -0,0 +1,178 @@
package inventory
import (
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// A value a provider's software issued is sealed to every pair that holds the provision from that provider, under
// one stamp of the value, and the same value given again writes nothing and sends nobody (novox/hq ADR 0305 §4,
// §5, issue 526).
func TestAnIssuedValueIsSealedToEveryHolderOnceAndNotAgain(t *testing.T) {
inv, ctx := twoNodesWithKeys(t)
consumerKey, consumerOpens := aSealingKey(t)
cn, _ := inv.NodeByName(ctx, "consumer")
if err := inv.RecordSealingKey(ctx, cn.ID, consumerKey); err != nil {
t.Fatal(err)
}
for _, m := range []string{"kometa", "tautulli"} {
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1", Requires: []string{"plex-api"},
Secrets: map[string]string{"plex-api": "/run/plex-api.secret"}}, Source{}); err != nil {
t.Fatal(err)
}
// What the mesh made before the provider gave its value: a random value per pair (issue 164).
if _, err := inv.SecretFor(ctx, "plex-api", "consumer", m, "provider", ""); err != nil {
t.Fatal(err)
}
}
kept, err := inv.KeepIssuedSecret(ctx, "plex-api", "provider", "the-servers-own-token")
if err != nil {
t.Fatal(err)
}
if !kept.Changed || len(kept.Holders) != 2 {
t.Fatalf("the value was not sealed to both holders: %+v", kept)
}
if len(kept.Machines) != 2 || kept.Machines[0] != "consumer" || kept.Machines[1] != "provider" {
t.Fatalf("the machines to send are the provider's and every holder's: %v", kept.Machines)
}
for _, m := range []string{"kometa", "tautulli"} {
got, err := inv.SecretFor(ctx, "plex-api", "consumer", m, "provider", "")
if err != nil {
t.Fatal(err)
}
if got.Origin != OriginIssued {
t.Fatalf("%s's pair says it came from %q, not the provider's software", m, got.Origin)
}
opened, ok := consumerOpens(got.ForConsumer)
if !ok || string(opened) != "the-servers-own-token" {
t.Fatalf("%s's copy is not the provider's value", m)
}
var generation string
pn, _ := inv.NodeByName(ctx, "provider")
if err := inv.store.Pool().QueryRow(ctx, `select generation from secret where name = 'plex-api' and consumer = $1
and consumer_module = $2 and provider = $3`, cn.ID, m, pn.ID).Scan(&generation); err != nil {
t.Fatal(err)
}
if generation != IssuedStamp("the-servers-own-token") {
t.Fatalf("%s's pair is not stamped with the value's fingerprint: %q", m, generation)
}
}
before, _ := inv.SecretFor(ctx, "plex-api", "consumer", "kometa", "provider", "")
// A consumer that binds later gets a pair of its own, and no holder's value is remade for it: the offer names
// no own secret, so ADR 0158 §6's remaking on a bind does not reach it (ADR 0305 §7).
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "overseerr", Version: "1", Requires: []string{"plex-api"},
Secrets: map[string]string{"plex-api": "/run/plex-api.secret"}}, Source{}); err != nil {
t.Fatal(err)
}
late, err := inv.SecretFor(ctx, "plex-api", "consumer", "overseerr", "provider", "")
if err != nil {
t.Fatal(err)
}
if late.Origin == OriginIssued {
t.Fatal("a consumer that bound after the ask holds the issued value without an ask")
}
if held, _ := inv.SecretFor(ctx, "plex-api", "consumer", "kometa", "provider", ""); held.ForConsumer != before.ForConsumer {
t.Fatal("a later bind remade a holder's issued value")
}
// The next ask seals the provider's value to the late consumer alone; the holders that had it are not touched.
again, err := inv.KeepIssuedSecret(ctx, "plex-api", "provider", "the-servers-own-token")
if err != nil {
t.Fatal(err)
}
if !again.Changed || len(again.Sealed) != 1 || again.Sealed[0].ConsumerModule != "overseerr" {
t.Fatalf("the late consumer was not sealed to, or not alone: %+v", again)
}
if held, _ := inv.SecretFor(ctx, "plex-api", "consumer", "kometa", "provider", ""); held.ForConsumer != before.ForConsumer {
t.Fatal("sealing the late consumer changed what a holder already held")
}
if got, _ := inv.SecretFor(ctx, "plex-api", "consumer", "overseerr", "provider", ""); got.Origin != OriginIssued {
t.Fatal("the late consumer does not hold the issued value after the ask")
}
// And once every holder holds it, the same value writes nothing and sends nobody.
still, err := inv.KeepIssuedSecret(ctx, "plex-api", "provider", "the-servers-own-token")
if err != nil {
t.Fatal(err)
}
if still.Changed || len(still.Machines) != 0 {
t.Fatalf("the same value was sealed again, which restarts every consumer for nothing: %+v", still)
}
after, _ := inv.SecretFor(ctx, "plex-api", "consumer", "kometa", "provider", "")
if after.ForConsumer != before.ForConsumer {
t.Fatal("the same value given again changed what the consumer holds")
}
changed, err := inv.KeepIssuedSecret(ctx, "plex-api", "provider", "a-token-plex-changed")
if err != nil {
t.Fatal(err)
}
if !changed.Changed {
t.Fatal("a changed value was not sealed")
}
got, _ := inv.SecretFor(ctx, "plex-api", "consumer", "tautulli", "provider", "")
if opened, ok := consumerOpens(got.ForConsumer); !ok || string(opened) != "a-token-plex-changed" {
t.Fatal("the changed value did not reach the holder")
}
if _, err := inv.KeepIssuedSecret(ctx, "plex-api", "provider", " "); err == nil {
t.Fatal("an empty value was kept")
}
}
// A consumer whose sealing key changed holds a value the mesh made in place of the issued one, and the next ask
// seals the issued value to it again; so does a pair a person accepted over it (novox/hq ADR 0305 §5, the review
// of mesh-controller #239).
func TestAnIssuedValueIsSealedAgainAfterAKeyChangeOrAnAcceptance(t *testing.T) {
inv, ctx := twoNodesWithKeys(t)
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "kometa", Version: "1", Requires: []string{"plex-api"},
Secrets: map[string]string{"plex-api": "/run/plex-api.secret"}}, Source{}); err != nil {
t.Fatal(err)
}
if _, err := inv.SecretFor(ctx, "plex-api", "consumer", "kometa", "provider", ""); err != nil {
t.Fatal(err)
}
if kept, err := inv.KeepIssuedSecret(ctx, "plex-api", "provider", "the-servers-own-token"); err != nil || !kept.Changed {
t.Fatalf("the first ask did not seal: %+v %v", kept, err)
}
// The consumer rejoins with a new key: the mesh remakes the pair, a value the software refuses.
newKey, opens := aSealingKey(t)
cn, _ := inv.NodeByName(ctx, "consumer")
if err := inv.RecordSealingKey(ctx, cn.ID, newKey); err != nil {
t.Fatal(err)
}
remade, err := inv.SecretFor(ctx, "plex-api", "consumer", "kometa", "provider", "")
if err != nil {
t.Fatal(err)
}
if remade.Origin != OriginMade {
t.Fatalf("a value the mesh remade says it came from %q", remade.Origin)
}
again, err := inv.KeepIssuedSecret(ctx, "plex-api", "provider", "the-servers-own-token")
if err != nil {
t.Fatal(err)
}
if !again.Changed || len(again.Sealed) != 1 {
t.Fatalf("the issued value was not sealed again after the key changed: %+v", again)
}
got, _ := inv.SecretFor(ctx, "plex-api", "consumer", "kometa", "provider", "")
if opened, ok := opens(got.ForConsumer); !ok || string(opened) != "the-servers-own-token" {
t.Fatal("the consumer does not hold the issued value under its new key")
}
// A person accepts another value over it: the next ask seals the software's again.
if err := inv.AcceptSecretForPair(ctx, "plex-api", "consumer", "kometa", "provider", "", "a-typed-copy"); err != nil {
t.Fatal(err)
}
after, err := inv.KeepIssuedSecret(ctx, "plex-api", "provider", "the-servers-own-token")
if err != nil {
t.Fatal(err)
}
if !after.Changed {
t.Fatal("an accepted value over the issued one was read as still holding it")
}
}
+142 -48
View File
@@ -111,6 +111,12 @@ type PlanBatch struct {
// a merge never shares a batch with one that waits for mesh-delivery's (ADR 0276, decided during the // a merge never shares a batch with one that waits for mesh-delivery's (ADR 0276, decided during the
// build, 2026-10-10). // build, 2026-10-10).
Own bool `json:"own,omitempty"` Own bool `json:"own,omitempty"`
// Closed says the window was closed by a stop that walks a stopped walk's merges again in this batch: they
// were heard before, and an older commit never waits behind a newer one (novox/hq ADR 0299). Withheld are
// the modules a stopped merge moves: built at the branch, which holds that merge, they would deliver it, so
// the batch's walk does not build them.
Closed bool `json:"closed,omitempty"`
Withheld []string `json:"withheld,omitempty"`
} }
// OwnPath says the record is a batch of merges on the controller's own path. // OwnPath says the record is a batch of merges on the controller's own path.
@@ -187,6 +193,17 @@ type PlanDelivery struct {
// TakenOverBy names the walk a walk folded before it started was taken over by: its merges are that // TakenOverBy names the walk a walk folded before it started was taken over by: its merges are that
// walk's now. // walk's now.
TakenOverBy string `json:"taken_over_by,omitempty"` TakenOverBy string `json:"taken_over_by,omitempty"`
// WalkedAgainBy names, on a walk stopped without some of its merges, the batch that walks its other merges
// again, and LeftOut the merges it left out (novox/hq issue 459, ADR 0299): a delivery of a merge left out
// ends stopped with the walk, one of a merge walked again follows that batch's walk.
WalkedAgainBy string `json:"walked_again_by,omitempty"`
LeftOut []PlanLeftOut `json:"left_out,omitempty"`
// Kept are the modules the stopped walk already sent and nothing walks again: the machines named keep running
// the stopped walk's build (novox/hq ADR 0299).
Kept []PlanKept `json:"kept,omitempty"`
// Withheld are, on a walk cut from a batch a stop walked merges again in, the modules it did not build: a
// stopped merge moves them, and built only as dependents at the branch they would deliver it (ADR 0299).
Withheld []string `json:"withheld,omitempty"`
// Batch is the window of a batch; nil once it is cut into a walk. // Batch is the window of a batch; nil once it is cut into a walk.
Batch *PlanBatch `json:"batch,omitempty"` Batch *PlanBatch `json:"batch,omitempty"`
// Alone says the walk walks one merge on its own commit, after a failed walk carried it in a later one: // Alone says the walk walks one merge on its own commit, after a failed walk carried it in a later one:
@@ -194,6 +211,26 @@ type PlanDelivery struct {
Alone bool `json:"alone,omitempty"` Alone bool `json:"alone,omitempty"`
} }
// PlanLeftOut is a merge a stop left out of the batch that walks a stopped walk's merges again (novox/hq ADR
// 0299): a merge stopped, or a later merge of its repository's branch, which contains the stopped one and so
// cannot be walked without delivering it.
type PlanLeftOut struct {
Repository string `json:"repository"`
Commit string `json:"commit"`
// Contains is the stopped merge a later merge contains, or whose modules it shares; empty for a merge
// stopped itself.
Contains string `json:"contains,omitempty"`
// Shares are the modules it moves that a stopped merge moves too: built at the branch, which holds the stopped
// merge, they would deliver it.
Shares []string `json:"shares,omitempty"`
}
// PlanKept is a module a stopped walk sent and nothing walks again, and the machines it reached.
type PlanKept struct {
Module string `json:"module"`
Machines []string `json:"machines"`
}
// Waiting is whether the walk waits for its delivery's word. // Waiting is whether the walk waits for its delivery's word.
func (p Plan) Waiting() bool { func (p Plan) Waiting() bool {
return p.Delivery != nil && p.Delivery.Awaits != "" && p.Delivery.Go == nil return p.Delivery != nil && p.Delivery.Awaits != "" && p.Delivery.Go == nil
@@ -265,7 +302,10 @@ type PlanGate struct {
Sent map[string]SentDeclaration `json:"sent,omitempty"` Sent map[string]SentDeclaration `json:"sent,omitempty"`
// Passes counts the consecutive judgings that found it healthy, LastPass the newest; a judging that // Passes counts the consecutive judgings that found it healthy, LastPass the newest; a judging that
// does not resets them. // does not resets them.
Passes int `json:"passes,omitempty"` Passes int `json:"passes,omitempty"`
// Wants is how many passes the send needs: a core module's three, a leaf module's one (novox/hq ADR 0282
// decision 4 as ADR 0304 decision 3 changed it). Zero in a gate judged before it was said: three.
Wants int `json:"wants,omitempty"`
LastPass *time.Time `json:"last_pass,omitempty"` LastPass *time.Time `json:"last_pass,omitempty"`
// Last is what the newest judging found wanting, while it still may pass. // Last is what the newest judging found wanting, while it still may pass.
Last string `json:"last,omitempty"` Last string `json:"last,omitempty"`
@@ -305,6 +345,18 @@ type PlanGate struct {
// Readings are the judging's readings with their times (novox/hq ADR 0282 decision 6): every reading that // Readings are the judging's readings with their times (novox/hq ADR 0282 decision 6): every reading that
// counted a pass, and the first that did not after one that did. At most maxReadings, the newest kept. // counted a pass, and the first that did not after one that did. At most maxReadings, the newest kept.
Readings []GateReading `json:"readings,omitempty"` Readings []GateReading `json:"readings,omitempty"`
// Grants is the grants step taken before this gate's send (novox/hq issue 490): the bus's user list sent
// alone to the machine holding the bus, every build there kept, so what the send newly grants is on the
// bus before the gate judges it. Nil when the list did not change, or the gate's machine holds the bus.
Grants *GrantsStep `json:"grants,omitempty"`
}
// GrantsStep is the bus's user list sent to the machine holding the bus ahead of a gated send (novox/hq
// issue 490): to which machine, when, and, when it could not be sent, why — the send went on without it.
type GrantsStep struct {
Node string `json:"node"`
At *time.Time `json:"at,omitempty"`
Failed string `json:"failed,omitempty"`
} }
// GateReading is one reading of a first-node gate. // GateReading is one reading of a first-node gate.
@@ -385,39 +437,28 @@ func (p Plan) Open() bool { return p.State == PlanBuilding || p.State == PlanRol
// with ErrPlanMoved otherwise; and only by a process that may act (ActsUnder), whose epoch it records. // with ErrPlanMoved otherwise; and only by a process that may act (ActsUnder), whose epoch it records.
// On success p's revision and epoch are the ones written, so the caller may save it again. // On success p's revision and epoch are the ones written, so the caller may save it again.
func (i *Inventory) SavePlan(ctx context.Context, p *Plan) error { func (i *Inventory) SavePlan(ctx context.Context, p *Plan) error {
return i.SavePlansMoving(ctx, []*Plan{p}, nil)
}
// MergeMove is a merge given to the plan that answers it now (novox/hq ADR 0276).
type MergeMove struct {
Repository, Commit, Plan string
Alone bool
}
// SavePlansMoving writes plans, as SavePlan does each, and gives merges to the plans that answer them, all in one
// transaction: every write and move, or none (novox/hq ADR 0299 decision 7 — a stop that walks merges again never
// leaves a merge answered by two plans, nor a walk stopped whose merges were not moved). A plan written by
// somebody else since it was read refuses the whole act with ErrPlanMoved. Each plan is said (PlanSaved) once
// the transaction is committed, in order.
func (i *Inventory) SavePlansMoving(ctx context.Context, plans []*Plan, moves []MergeMove) error {
epoch, err := i.actingEpoch(ctx) epoch, err := i.actingEpoch(ctx)
if err != nil { if err != nil {
return fmt.Errorf("the plan for %s %s is not written: %w", p.Repository, p.Commit, err) if len(plans) > 0 {
} return fmt.Errorf("the plan for %s %s is not written: %w", plans[0].Repository, plans[0].Commit, err)
tiers, err := json.Marshal(p.Tiers) }
if err != nil {
return err return err
} }
modules, err := json.Marshal(p.Modules)
if err != nil {
return err
}
var release, delivery, commits, times []byte
if p.Times != nil {
if times, err = json.Marshal(p.Times); err != nil {
return err
}
}
if len(p.Commits) > 0 {
if commits, err = json.Marshal(p.Commits); err != nil {
return err
}
}
if p.Release != nil {
if release, err = json.Marshal(p.Release); err != nil {
return err
}
}
if p.Delivery != nil {
if delivery, err = json.Marshal(p.Delivery); err != nil {
return err
}
}
// **And how long the tier it left took** (novox/hq to-be 45 Phase 0): measured here, where the // **And how long the tier it left took** (novox/hq to-be 45 Phase 0): measured here, where the
// plan moves, in the same transaction as the move, so no save can move a tier unmeasured or // plan moves, in the same transaction as the move, so no save can move a tier unmeasured or
// measure one twice. // measure one twice.
@@ -426,9 +467,77 @@ func (i *Inventory) SavePlan(ctx context.Context, p *Plan) error {
return err return err
} }
defer func() { _ = tx.Rollback(ctx) }() defer func() { _ = tx.Rollback(ctx) }()
type written struct {
revision int64
entered time.Time
}
done := make([]written, len(plans))
for n, p := range plans {
revision, entered, err := writePlan(ctx, tx, p, epoch)
if err != nil {
return err
}
done[n] = written{revision, entered}
}
for _, m := range moves {
if _, err := tx.Exec(ctx,
`update batched_merge set plan_id = nullif($3, ''), alone = $4 where repository = lower($1) and commit_hash = $2`,
m.Repository, m.Commit, m.Plan, m.Alone); err != nil {
return err
}
}
if err := tx.Commit(ctx); err != nil {
return err
}
for n, p := range plans {
p.Revision, p.TierEntered = done[n].revision, done[n].entered
if epoch != nil {
p.Epoch = uint64(*epoch)
} else {
p.Epoch = 0
}
if PlanSaved != nil {
PlanSaved(*p)
}
}
return nil
}
// writePlan writes one plan within a transaction, by compare-and-set on its revision: its new revision, and when
// it entered its tier.
func writePlan(ctx context.Context, tx pgx.Tx, p *Plan, epoch *int64) (int64, time.Time, error) {
tiers, err := json.Marshal(p.Tiers)
if err != nil {
return 0, time.Time{}, err
}
modules, err := json.Marshal(p.Modules)
if err != nil {
return 0, time.Time{}, err
}
var release, delivery, commits, times []byte
if p.Times != nil {
if times, err = json.Marshal(p.Times); err != nil {
return 0, time.Time{}, err
}
}
if len(p.Commits) > 0 {
if commits, err = json.Marshal(p.Commits); err != nil {
return 0, time.Time{}, err
}
}
if p.Release != nil {
if release, err = json.Marshal(p.Release); err != nil {
return 0, time.Time{}, err
}
}
if p.Delivery != nil {
if delivery, err = json.Marshal(p.Delivery); err != nil {
return 0, time.Time{}, err
}
}
entered, err := planTierLeft(ctx, tx, *p, time.Now()) entered, err := planTierLeft(ctx, tx, *p, time.Now())
if err != nil { if err != nil {
return err return 0, time.Time{}, err
} }
var revision int64 var revision int64
err = tx.QueryRow(ctx, err = tx.QueryRow(ctx,
@@ -449,24 +558,9 @@ func (i *Inventory) SavePlan(ctx context.Context, p *Plan) error {
// The row is there and at another revision — moved since this was read, or there already // The row is there and at another revision — moved since this was read, or there already
// when this one is new: either way not this writer's to overwrite. (A plan saved before plans // when this one is new: either way not this writer's to overwrite. (A plan saved before plans
// had revisions is at zero, and its first save here is from a read at zero.) // had revisions is at zero, and its first save here is from a read at zero.)
return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved) return 0, time.Time{}, fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved)
} }
if err != nil { return revision, entered, err
return err
}
if err := tx.Commit(ctx); err != nil {
return err
}
p.Revision, p.TierEntered = revision, entered
if epoch != nil {
p.Epoch = uint64(*epoch)
} else {
p.Epoch = 0
}
if PlanSaved != nil {
PlanSaved(*p)
}
return nil
} }
// short is a commit as a person reads it. // short is a commit as a person reads it.
+6 -3
View File
@@ -58,7 +58,9 @@ const (
// **Remade when either end's sealing key changes.** A node that rejoined generated a new key and // **Remade when either end's sealing key changes.** A node that rejoined generated a new key and
// can no longer open what was sealed to the old one, so keeping the blob would deliver something // can no longer open what was sealed to the old one, so keeping the blob would deliver something
// unreadable for ever. The new secret reaches both ends in the same push, which is the only // unreadable for ever. The new secret reaches both ends in the same push, which is the only
// moment they can be changed together. // moment they can be changed together. A remade value is the mesh's, whatever the row held: its
// origin says made and it carries no stamp, so a value a provider's software issued is sealed to it
// again at the provider's next ask (novox/hq ADR 0305 §5) rather than read as still held.
func (i *Inventory) SecretFor(ctx context.Context, name, consumer, consumerModule, provider, local string) ( func (i *Inventory) SecretFor(ctx context.Context, name, consumer, consumerModule, provider, local string) (
Secret, error) { Secret, error) {
consumerKey, err := i.SealingKeyOf(ctx, consumer) consumerKey, err := i.SealingKeyOf(ctx, consumer)
@@ -121,7 +123,8 @@ func (i *Inventory) SecretFor(ctx context.Context, name, consumer, consumerModul
for_consumer = excluded.for_consumer, for_provider = excluded.for_provider, for_consumer = excluded.for_consumer, for_provider = excluded.for_provider,
consumer_key = excluded.consumer_key, provider_key = excluded.provider_key, consumer_key = excluded.consumer_key, provider_key = excluded.provider_key,
created_at = now(), created_at = now(),
operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key`, operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key,
origin = 'made', generation = null`,
name, consumerNode.ID, consumerModule, providerNode.ID, name, consumerNode.ID, consumerModule, providerNode.ID,
made.ForConsumer, made.ForProvider, made.ConsumerKey, made.ProviderKey, forOperator, operatorKey, local) made.ForConsumer, made.ForProvider, made.ConsumerKey, made.ProviderKey, forOperator, operatorKey, local)
if err != nil { if err != nil {
@@ -202,7 +205,7 @@ func (i *Inventory) AcceptSecretForPair(ctx context.Context, name, consumer, con
on conflict (name, local, consumer, consumer_module, provider) do update set on conflict (name, local, consumer, consumer_module, provider) do update set
for_consumer = excluded.for_consumer, for_provider = excluded.for_provider, for_consumer = excluded.for_consumer, for_provider = excluded.for_provider,
consumer_key = excluded.consumer_key, provider_key = excluded.provider_key, consumer_key = excluded.consumer_key, provider_key = excluded.provider_key,
created_at = now(), origin = excluded.origin, created_at = now(), origin = excluded.origin, generation = null,
operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key`, operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key`,
name, consumerNode.ID, consumerModule, providerNode.ID, name, consumerNode.ID, consumerModule, providerNode.ID,
sealed.ForConsumer, sealed.ForProvider, sealed.ConsumerKey, sealed.ProviderKey, sealed.ForConsumer, sealed.ForProvider, sealed.ConsumerKey, sealed.ProviderKey,
+62 -9
View File
@@ -100,6 +100,23 @@ type Send struct {
CounterRaisedTo int64 CounterRaisedTo int64
// Recorded is false for a refusal of a sequence the mesh has no send for: its sender is unknown. // Recorded is false for a refusal of a sequence the mesh has no send for: its sender is unknown.
Recorded bool Recorded bool
// Answer is what the machine last reported of this send; filled only where a last send is read back
// (LastSend, LastSends), the zero answer elsewhere.
Answer SendAnswer
}
// SendAnswer is what a machine reported of one send, read from its last report when that report is about the
// send's declaration (novox/hq issue 485): whether it applied it, failed part of it or refused it, and why.
// The zero answer is a machine that has not reported on this send — yet, or since a later report replaced it.
type SendAnswer struct {
// Outcome is applied, failed or refused, as the machine reported it; empty when it has not reported on it.
Outcome string
// Refused is the machine's own words when it refused the declaration whole.
Refused string
// Failed is how many of its resources failed, when some did.
Failed int
// At is when the machine reported it; nil when it has not.
At *time.Time
} }
// unknownSender is the sender of a refused sequence the mesh has no record of sending. // unknownSender is the sender of a refused sequence the mesh has no record of sending.
@@ -209,20 +226,56 @@ func scanSends(rows pgx.Rows) ([]Send, error) {
return out, rows.Err() return out, rows.Err()
} }
// LastSend is the last declaration a machine was sent, by its name; false when none was recorded. A refusal // LastSend is the last declaration a machine was sent, by its name, with what the machine answered of it; false
// of a sequence the mesh has no send for is not a send, and is not it. // when none was recorded. A refusal of a sequence the mesh has no send for is not a send, and is not it.
func (i *Inventory) LastSend(ctx context.Context, name string) (Send, bool, error) { func (i *Inventory) LastSend(ctx context.Context, name string) (Send, bool, error) {
rows, err := i.store.Pool().Query(ctx, `select `+sendColumns+` sends, err := i.lastSends(ctx, `n.name = $1`, name)
from declaration_send s join node n on n.id = s.node
where n.name = $1 and s.recorded order by s.id desc limit 1`, name)
if err != nil { if err != nil {
return Send{}, false, err return Send{}, false, err
} }
sends, err := scanSends(rows) s, found := sends[name]
if err != nil || len(sends) == 0 { return s, found, nil
return Send{}, false, err }
// LastSends is every machine's last send, by machine name, each with what the machine answered of it: what
// `status` says per machine (novox/hq issue 485). A machine never sent anything has no entry.
func (i *Inventory) LastSends(ctx context.Context) (map[string]Send, error) {
return i.lastSends(ctx, `true`)
}
// lastSends reads the newest recorded send of each machine matching where, beside the machine's last report
// when that report is about the send (novox/hq issue 485). It names the send's declaration (its digest), and:
// - from an ordered node-engine, one that reports the epoch and sequence it acted on, those are the send's
// own. A periodic report about an identical earlier send names the same digest and may arrive after a later
// send the machine never received; only the sequence tells them apart. It also holds for a fast machine
// whose report lands before the send's row is stamped.
// - from an engine that reports no order, the report came after the send — both times the store's own — so
// a machine sent again a declaration it once applied does not read as having applied the new send.
func (i *Inventory) lastSends(ctx context.Context, where string, args ...any) (map[string]Send, error) {
rows, err := i.store.Pool().Query(ctx, `select distinct on (n.name) `+sendColumns+`,
coalesce(r.outcome, ''), coalesce(r.refused, ''), case when jsonb_typeof(r.failed) = 'array' then jsonb_array_length(r.failed) else 0 end, r.at
from declaration_send s join node n on n.id = s.node
left join node_report r on r.node = s.node and r.declared <> '' and r.declared = s.digest
and case when r.reported_sequence is not null
then r.reported_sequence = s.sequence and coalesce(r.reported_epoch, 0) = coalesce(s.epoch, 0)
else r.at >= s.sent_at end
where s.recorded and `+where+`
order by n.name, s.id desc`, args...)
if err != nil {
return nil, err
} }
return sends[0], true, nil defer rows.Close()
out := map[string]Send{}
for rows.Next() {
var s Send
if err := rows.Scan(&s.Node, &s.NodeName, &s.Sequence, &s.Epoch, &s.Sender, &s.Generation, &s.Digest,
&s.Modules, &s.SentAt, &s.RefusedAt, &s.RefusedApplied, &s.CounterRaisedTo, &s.Recorded,
&s.Answer.Outcome, &s.Answer.Refused, &s.Answer.Failed, &s.Answer.At); err != nil {
return nil, err
}
out[s.NodeName] = s
}
return out, rows.Err()
} }
// RefusedSend keeps a machine's refusal of the send of a sequence for the generation it came from, and // RefusedSend keeps a machine's refusal of the send of a sequence for the generation it came from, and
+141
View File
@@ -205,3 +205,144 @@ func TestWhetherAMachineReadsAGenerationIsItsLatestWord(t *testing.T) {
} }
} }
} }
// **The last send says what the machine answered of it** (novox/hq issue 485): applied, refused with its
// words, or not answered yet — read from the machine's report only when that report is about the send's
// declaration and came after it, so a report about an earlier send is never this one's answer. And every
// machine's last send is read at once, for status, by name; a machine never sent anything has none.
func TestTheLastSendSaysWhatTheMachineAnswered(t *testing.T) {
inv, node := aNodeWithModules(t)
ctx := t.Context()
record, err := inv.NodeByName(ctx, node)
if err != nil {
t.Fatal(err)
}
if _, err := inv.AddNode(ctx, "laptop"); err != nil {
t.Fatal(err)
}
send := func(sequence int64, digest string) {
t.Helper()
if err := inv.RecordSentWith(ctx, record.ID, digest, nil, 57, 40, Send{Sequence: sequence, Epoch: 57,
Sender: "the controller's daemon (pid 7 on anchor)", Generation: 40, Digest: digest,
Modules: []string{"docker", "sudo"}}); err != nil {
t.Fatal(err)
}
}
report := func(d Doing) {
t.Helper()
if _, err := inv.RecordDoing(ctx, record.ID, d); err != nil {
t.Fatal(err)
}
}
answer := func() SendAnswer {
t.Helper()
last, found, err := inv.LastSend(ctx, node)
if err != nil || !found {
t.Fatalf("the last send is not read back: %v, %v", found, err)
}
return last.Answer
}
send(11, "d11")
if a := answer(); a.Outcome != "" || a.At != nil {
t.Fatalf("a send not reported on reads as answered: %+v", a)
}
report(Doing{Outcome: OutcomeApplied, Declared: "d11", Applied: 3})
if a := answer(); a.Outcome != OutcomeApplied || a.At == nil {
t.Fatalf("a send the machine reported applying reads %+v", a)
}
send(12, "d12")
if a := answer(); a.Outcome != "" {
t.Fatalf("the report about the send before is taken for this one's answer: %+v", a)
}
report(Doing{Outcome: OutcomeRefused, Declared: "d12", Refused: "unknown field \"generation\"\nand more"})
if a := answer(); a.Outcome != OutcomeRefused || a.Refused != "unknown field \"generation\"\nand more" {
t.Fatalf("a send the machine refused reads %+v", a)
}
// Sent again a declaration it reported applying before the send: not answered until it reports again.
report(Doing{Outcome: OutcomeApplied, Declared: "d12", Applied: 3})
send(13, "d12")
if a := answer(); a.Outcome != "" {
t.Fatalf("a report from before the send is taken for its answer: %+v", a)
}
all, err := inv.LastSends(ctx)
if err != nil {
t.Fatal(err)
}
if len(all) != 1 || all[node].Sequence != 13 || all[node].Epoch != 57 || all[node].Generation != 40 ||
!slices.Equal(all[node].Modules, []string{"docker", "sudo"}) {
t.Fatalf("every machine's last send reads %+v", all)
}
if _, found := all["laptop"]; found {
t.Error("a machine never sent anything has a last send")
}
}
// **A report about another declaration, or another send of the same one, is not the last send's answer**
// (novox/hq issue 485, review of mesh-controller #234). A report naming an earlier send's digest that arrives
// after a later send is not the later send's answer. And from an ordered node-engine, a periodic report about
// an identical earlier send — the same digest, arriving after the later send — is not its answer either: only
// the report's epoch and sequence say which send it is about.
func TestAReportIsTheLastSendsAnswerOnlyWhenItIsAboutThatSend(t *testing.T) {
inv, node := aNodeWithModules(t)
ctx := t.Context()
record, err := inv.NodeByName(ctx, node)
if err != nil {
t.Fatal(err)
}
send := func(sequence int64, digest string) {
t.Helper()
if err := inv.RecordSentWith(ctx, record.ID, digest, nil, 57, 40, Send{Sequence: sequence, Epoch: 57,
Sender: "the controller's daemon (pid 7 on anchor)", Generation: 40, Digest: digest}); err != nil {
t.Fatal(err)
}
}
ordered := func(epoch, sequence int64, digest string) {
t.Helper()
if _, err := inv.RecordOrderedDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: digest, Applied: 1},
ReportOrder{Epoch: epoch, Sequence: sequence}, func(ReportOrder) bool { return false }); err != nil {
t.Fatal(err)
}
}
answer := func() string {
t.Helper()
last, found, err := inv.LastSend(ctx, node)
if err != nil || !found {
t.Fatalf("the last send is not read back: %v, %v", found, err)
}
return last.Answer.Outcome
}
// An engine that reports no order: the digest decides.
send(11, "d11")
send(12, "d12")
if _, err := inv.RecordDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: "d11"}); err != nil {
t.Fatal(err)
}
if a := answer(); a != "" {
t.Fatalf("a report about d11, after d12 was sent, reads as d12's answer: %q", a)
}
// An ordered engine: the epoch and sequence decide, whatever the digest and the time.
send(20, "d20")
ordered(57, 20, "d20")
if a := answer(); a != OutcomeApplied {
t.Fatalf("an ordered report about sequence 20 is not its answer: %q", a)
}
send(21, "d20")
ordered(57, 20, "d20")
if a := answer(); a != "" {
t.Fatalf("a periodic report about sequence 20, after 21 was sent with the same digest, reads as 21's answer: %q", a)
}
ordered(56, 21, "d20")
if a := answer(); a != "" {
t.Fatalf("a report about sequence 21 of another epoch reads as this send's answer: %q", a)
}
ordered(57, 21, "d20")
if a := answer(); a != OutcomeApplied {
t.Fatalf("an ordered report about sequence 21 is not its answer: %q", a)
}
}
+17 -8
View File
@@ -123,15 +123,24 @@ func TestNatsABuildIsTakenAndItsOutcomeReachesEverybody(t *testing.T) {
t.Fatalf("line %d came back as %s (%v)", want, msg.Data, err) t.Fatalf("line %d came back as %s (%v)", want, msg.Data, err)
} }
} }
// And it is in the stream for a reader who comes later. // And it is in the stream for a reader who comes later. **Waited for, with a bound, rather
info, err := js.Context().StreamInfo(broker.EventsStream, &nats.StreamInfoRequest{SubjectsFilter: BuildLog("b-1")}) // than read once** (novox/hq issue 507): a line is said with a plain publish, so the server
if err != nil { // hands it to a live subscriber and stores it in the stream independently, and under load the
t.Fatal(err) // subscriber above had both lines while the stream still held one, or none.
var held map[string]uint64
for until := time.Now().Add(5 * time.Second); ; {
info, err := js.Context().StreamInfo(broker.EventsStream, &nats.StreamInfoRequest{SubjectsFilter: BuildLog("b-1")})
if err != nil {
t.Fatal(err)
}
held = info.State.Subjects
if held[BuildLog("b-1")] >= 2 || time.Now().After(until) {
break
}
time.Sleep(10 * time.Millisecond)
} }
if info.State.Subjects[BuildLog("b-1")] != 2 { if held[BuildLog("b-1")] != 2 {
t.Fatalf("the stream holds %v under the build's subject, want 2", info.State.Subjects) t.Fatalf("the stream holds %v under the build's subject, want 2", held)
}
if err == nil {
} }
if result.ID != "b-1" || result.Commit != "abc1234" { if result.ID != "b-1" || result.Commit != "abc1234" {
t.Fatalf("the asker got %+v", result) t.Fatalf("the asker got %+v", result)
+19 -1
View File
@@ -173,6 +173,25 @@ func ModuleToolOn(module, tool, node string) string { return ToolSubject(module,
// AskModuleToolOn asks one machine's instance of a module one of its tools and reads its answer. // AskModuleToolOn asks one machine's instance of a module one of its tools and reads its answer.
func AskModuleToolOn(ctx context.Context, conn *nats.Conn, module, tool, node string, args any, func AskModuleToolOn(ctx context.Context, conn *nats.Conn, module, tool, node string, args any,
timeout time.Duration) (Answer, error) {
return askModuleToolAt(ctx, conn, ModuleToolOn(module, tool, node), module, tool, node, args, timeout)
}
// AskModuleToolOnAs is AskModuleToolOn in the caller's own name: on the subject that names its caller (novox/hq
// issue 365), which the bus lets only that user publish, so the tool knows who asks from the subject alone. The
// one way a tool can answer the controller and nobody else (novox/hq ADR 0305 §2).
func AskModuleToolOnAs(ctx context.Context, conn *nats.Conn, caller, module, tool, node string, args any,
timeout time.Duration) (Answer, error) {
subject := broker.CalledSubject(ModuleToolOn(module, tool, node), caller)
if subject == "" {
return Answer{}, fmt.Errorf("%q cannot ask %s.%s on %s in its own name: it is not a bus user's name", caller,
module, tool, node)
}
return askModuleToolAt(ctx, conn, subject, module, tool, node, args, timeout)
}
// askModuleToolAt asks one machine's instance of a module one of its tools on subject and reads its answer.
func askModuleToolAt(ctx context.Context, conn *nats.Conn, subject, module, tool, node string, args any,
timeout time.Duration) (Answer, error) { timeout time.Duration) (Answer, error) {
body, err := json.Marshal(args) body, err := json.Marshal(args)
if err != nil { if err != nil {
@@ -180,7 +199,6 @@ func AskModuleToolOn(ctx context.Context, conn *nats.Conn, module, tool, node st
} }
asking, cancel := context.WithTimeout(ctx, timeout) asking, cancel := context.WithTimeout(ctx, timeout)
defer cancel() defer cancel()
subject := ModuleToolOn(module, tool, node)
refused, stop := refusalsOf(conn, subject) refused, stop := refusalsOf(conn, subject)
defer stop() defer stop()
type replied struct { type replied struct {
+82
View File
@@ -18,6 +18,34 @@
set -eu set -eu
export GOFLAGS=-mod=vendor GOPROXY=off export GOFLAGS=-mod=vendor GOPROXY=off
# judge_store_tests says each listed test of one package as `go test -v` reported it, and counts in $missing each
# that did not pass: $1 the package, $2 go test's exit status, $3 the file holding its output, then the names.
# A result is found anywhere in a line, not only at its start: when a test's last output leaves a line open,
# `go test -v` writes `--- PASS: <name> (…)` on that same line (novox/hq issue 515). A package whose go test
# exits non-zero fails even when each listed test said PASS — a TestMain or a leak checker can fail after them.
judge_store_tests() {
pkg=$1 status=$2 out=$3
shift 3
for name in "$@"; do
line=$(grep -oE -- "--- (PASS|FAIL|SKIP): $name \([^)]*\)" "$out" | tail -n 1 || true)
echo " $pkg ${line:-"--- NOT RUN: $name"}"
case "$line" in "--- PASS: "*) ;; *) missing=$((missing + 1)) ;; esac
done
if [ "$status" -ne 0 ]; then
echo " $pkg FAIL: go test exited $status"
missing=$((missing + 1))
fi
}
# `sh merge-check.sh judge-store-tests <package> <status> <output> <name>…` runs only the judging, to try it on a
# recorded `go test -v` output.
if [ "${1:-}" = judge-store-tests ]; then
shift
missing=0
judge_store_tests "$@"
[ "$missing" -eq 0 ]
exit
fi
unformatted=$(gofmt -l cmd internal examples) unformatted=$(gofmt -l cmd internal examples)
if [ -n "$unformatted" ]; then if [ -n "$unformatted" ]; then
echo "not gofmt'd:" echo "not gofmt'd:"
@@ -32,3 +60,57 @@ else
echo "NOT RACE-CHECKED: the toolchain holds no C compiler; the suite runs without the race detector" echo "NOT RACE-CHECKED: the toolchain holds no C compiler; the suite runs without the race detector"
CGO_ENABLED=0 go test -count=1 -timeout 30m ./... CGO_ENABLED=0 go test -count=1 -timeout 30m ./...
fi fi
# **The store tests listed ran** (novox/hq issue 459): a test that needs the store skips without one, and a skip
# passes unseen. With the store this check is given, every test named in testdata/store-tests must have run against
# it and passed, each said by name; without one, that is said. A pull request adds its own store tests to the list.
listed=$(grep -v '^[[:space:]]*\(#\|$\)' testdata/store-tests || true)
# Each line is exactly `<package> <TestName>`: a line of one field ran nothing and said nothing (novox/hq issue 515).
malformed=$(printf '%s\n' "$listed" | awk 'NF != 2 && NF != 0' || true)
if [ -n "$malformed" ]; then
echo "testdata/store-tests: a line is not <package> <TestName>:"
echo "$malformed"
exit 1
fi
# **A change does not shorten the list it is judged by** (novox/hq issue 515): the list is in the change's own tree,
# so an entry main lists stays listed while its test function exists; removing the function removes the entry.
if ! git rev-parse --verify --quiet origin/main^{commit} >/dev/null; then
echo "origin/main cannot be read, and its testdata/store-tests says which store tests must stay listed"
exit 1
fi
dropped=0
for entry in $(git show origin/main:testdata/store-tests 2>/dev/null | grep -v '^[[:space:]]*\(#\|$\)' |
awk 'NF == 2 {print $1 "|" $2}'); do
pkg=${entry%%|*} name=${entry#*|}
printf '%s\n' "$listed" | awk -v p="$pkg" -v n="$name" '$1 == p && $2 == n {f = 1} END {exit !f}' && continue
if cat "$pkg"/*_test.go 2>/dev/null | grep -q "^func $name("; then
echo "testdata/store-tests: $pkg $name is listed on main and its test still exists, but the change drops it"
dropped=$((dropped + 1))
fi
done
[ "$dropped" -eq 0 ] || exit 1
if [ -z "$listed" ]; then
echo "store tests: none listed in testdata/store-tests"
elif [ -n "${MESH_TEST_POSTGRES:-}" ]; then
echo "store tests, each run against the store:"
missing=0
out=$(mktemp)
trap 'rm -f "$out"' EXIT
for pkg in $(printf '%s\n' "$listed" | awk '{print $1}' | sort -u); do
names=$(printf '%s\n' "$listed" | awk -v p="$pkg" '$1 == p {print $2}')
pattern="^($(printf '%s\n' "$names" | paste -sd '|' -))\$"
status=0
CGO_ENABLED=0 go test -count=1 -v -run "$pattern" "$pkg" >"$out" 2>&1 || status=$?
# shellcheck disable=SC2086 # the names are one word each, split on purpose
judge_store_tests "$pkg" "$status" "$out" $names
done
if [ "$missing" -ne 0 ]; then
echo "$missing store test(s) listed in testdata/store-tests did not run and pass against the store"
exit 1
fi
else
echo "NOT STORE-CHECKED: no MESH_TEST_POSTGRES, so the store tests listed in testdata/store-tests skipped"
fi
BIN
View File
Binary file not shown.
+1
View File
@@ -77,6 +77,7 @@
"drill", "drill",
"warranted", "warranted",
"secret-ask", "secret-ask",
"issued",
"hand-acts", "hand-acts",
"durations", "durations",
"conditions", "conditions",
Binary file not shown.
BIN
View File
Binary file not shown.
+8
View File
@@ -25,3 +25,11 @@ and write the commits here. The SDK is captured at the commit go.mod pins for gi
rm -rf testdata/beside/mesh-sdk && mkdir -p testdata/beside/mesh-sdk rm -rf testdata/beside/mesh-sdk && mkdir -p testdata/beside/mesh-sdk
git -C ../mesh-sdk archive <commit> conformance/events | tar -x -C testdata/beside/mesh-sdk git -C ../mesh-sdk archive <commit> conformance/events | tar -x -C testdata/beside/mesh-sdk
And from mesh-host at the same commit as its line above, the node-engine's placement guard, which
internal/catalogue's engine_guard_test.go holds the controller's resolved-path rules to (novox/hq issue 496),
kept as .captured so no Go tool reads it as this repository's code:
mkdir -p testdata/beside/mesh-host/internal/apply
git -C ../mesh-host show <commit>:internal/apply/placement_guard.go \
> testdata/beside/mesh-host/internal/apply/placement_guard.go.captured
@@ -0,0 +1,549 @@
package apply
// Where the node-engine places nothing and mounts nothing, whoever asks (novox/hq issue 339).
//
// A directory names its path, and the controller resolves part of that path from what was set for the
// module: its `places` setting moves a directory anywhere, with an owner it names, and its `accesses` setting
// says which of the machine's paths are mounted into its container. The engine runs as root, so a path it
// accepts blindly is a path anyone who could change those settings hands to any account: a directory at /etc
// owned by a caller's account gives it /etc, and an access at / mounts the machine's root into a container.
// The controller refuses both where a setting is made; the engine refuses them again where it applies,
// because a guard in one place is a guard one change away from gone. Whatever the declaration says:
//
// 1. **No directory, access or mount source is one of the machine's own roots, or holds one**: /, /etc,
// /usr, /var, /var/lib, /home, /run and the rest of protectedRoots. Modules place directories BELOW /etc
// or /var/lib, never the root itself; owning one is owning everything in it.
// 2. **Nothing is placed in /proc, /sys, /dev, /boot, /root, /var/spool, /opt or the container runtimes' data
// (/var/lib/docker, /var/lib/containers, /var/lib/containerd, and where the runtimes' configuration moves
// them: runtimeDataRoots), nor in the node-engine's
// own trees** (its state, its identity, its installed builds) but by its own module; nothing is mounted from
// those but /proc, /sys and /dev. A mount of a kernel file, a device or the clock is the plumbing
// systemPath names.
// 2a. **An account's .ssh is never an access or a mount**, and is a directory only below its account's home,
// as that account's (rule 4).
// 3. **A directory below /etc, /usr or /run is root's.** The machine's configuration and programs are read
// as root's word; a directory there owned by another account is that account writing root's word. An
// access is never there at all: the operator's data is not the machine's configuration.
// 4. **Below a person's or an agent's home, a directory is that account's.** Root's or another account's
// directory there is one the account does not control in a tree whose every parent it does. A home
// itself may be a module's directory (a backup repository kept as an account's home is one), and as
// every directory the mesh did not make, it is used as found: never chowned or chmodded (applyDirectory).
// 5. **A mount source that is a refused directory or a refused access is refused with it**: the container
// would otherwise bind the very path the engine would not place, and the runtime creates a missing one
// as root.
//
// Each is a failed resource with its reason in the node's report; nothing is touched. Paths are judged as
// declared and again with every link in them resolved, so a link at /srv/x pointing at /etc places nothing.
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"sync"
"time"
"github.com/novox/mesh-host/internal/declaration"
)
// protectedRoots are paths no directory, access or mount source may be, nor hold.
var protectedRoots = []string{"/", "/bin", "/boot", "/dev", "/etc", "/home", "/lib", "/lib32", "/lib64",
"/media", "/mnt", "/opt", "/proc", "/root", "/run", "/sbin", "/srv", "/sys", "/tmp", "/usr", "/usr/bin",
"/usr/lib", "/usr/lib64", "/usr/local", "/usr/local/bin", "/usr/local/lib", "/usr/local/sbin", "/usr/sbin",
"/usr/share", "/var", "/var/cache", "/var/lib", "/var/lib/mesh", "/var/log", "/var/run", "/var/tmp",
"/var/spool"}
// forbiddenBelow are trees nothing is placed in or mounted from: the kernel's, the boot loader's and root's
// home. engineTrees are the node-engine's own, which only its own module places in.
var (
forbiddenBelow = []string{"/proc", "/sys", "/dev", "/boot", "/root", "/var/spool", "/opt", "/var/lib/docker",
"/var/lib/containers", "/var/lib/containerd"}
// forbiddenBelowMount are the trees no container mounts from: a mount of the kernel's files and devices is
// the plumbing a container may need (systemPath); root's home and the boot loader's are no plumbing.
forbiddenBelowMount = []string{"/boot", "/root", "/var/spool", "/opt", "/var/lib/docker", "/var/lib/containers",
"/var/lib/containerd"}
engineTrees = []string{"/var/lib/mesh-host", "/usr/lib/nox-mesh-host"}
// rootsOnly are trees a directory below is root's, and an access is never in.
rootsOnly = []string{"/etc", "/usr", "/run", "/var/run"}
)
// runtimeFiles are where the container runtimes say where they keep their data: dockerd's daemon.json, its
// unit and the unit's drop-ins (an ExecStart with --data-root, or the older -g/--graph, continued over lines,
// quoted, through Environment= or EnvironmentFile=, or a --config-file naming another daemon.json), and podman's
// storage.conf (graphroot, a basic or a literal string). The running runtimes are asked first. containerd keeps its own under /var/lib/containerd, which forbiddenBelow names; a
// containerd configured elsewhere, and podman's rootless stores under each account's home, are not read: the
// first is no runtime this mesh runs, and the second is below a home, which rule 4 already keeps for its
// account. A variable so a test names its own.
type runtimeFiles struct {
daemonJSON string
units []string
dropInDirs []string
storageConf string
}
var runtimeConfigs = runtimeFiles{
daemonJSON: "/etc/docker/daemon.json",
units: []string{"/etc/systemd/system/docker.service", "/usr/lib/systemd/system/docker.service", "/lib/systemd/system/docker.service"},
dropInDirs: []string{"/etc/systemd/system/docker.service.d", "/run/systemd/system/docker.service.d", "/usr/lib/systemd/system/docker.service.d"},
storageConf: "/etc/containers/storage.conf",
}
var (
dataRootFlag = regexp.MustCompile(`(?:--data-root|--graph|-g)(?:=|\s+)(\S+)`)
configFlag = regexp.MustCompile(`--config-file(?:=|\s+)(\S+)`)
graphRoot = regexp.MustCompile(`(?m)^\s*graphroot\s*=\s*(?:"([^"]+)"|'([^']+)')`)
envVar = regexp.MustCompile(`\$\{([A-Za-z_][A-Za-z0-9_]*)\}|\$([A-Za-z_][A-Za-z0-9_]*)`)
)
// runtimeRoots is where the container runtimes keep their data, as the last apply read it (readRuntimeRoots);
// nil until an apply has read it, when the guard reads the files itself.
var (
runtimeRootsMu sync.Mutex
runtimeRoots []string
)
// AskRuntimes is how the engine asks the running container runtimes where they keep their data: set by the engine
// to run the commands, nil in a test, which then reads the files alone. Its own, never the apply's runner, whose
// commands a test reads back as what the apply did.
var AskRuntimes Runner
// refreshRuntimeRoots reads where the runtimes keep their data once, at the start of an apply.
func refreshRuntimeRoots(ctx context.Context) {
roots := readRuntimeRoots(ctx, AskRuntimes)
runtimeRootsMu.Lock()
runtimeRoots = roots
runtimeRootsMu.Unlock()
}
// runtimeDataRoots is every place the container runtimes keep their data, beyond the default trees forbiddenBelow
// names: every container's filesystem is there.
func runtimeDataRoots() []string {
runtimeRootsMu.Lock()
roots := runtimeRoots
runtimeRootsMu.Unlock()
if roots != nil {
return roots
}
return readRuntimeRoots(context.Background(), nil)
}
// readRuntimeRoots asks the running runtimes where they keep their data, when run is given (`docker info`,
// `podman info`), and reads their configuration besides: an answer from a runtime that is running is what it
// really does, and the files say what it will do when it starts again. Both count. A runtime that is not running
// or not installed answers nothing, which is no error.
func readRuntimeRoots(ctx context.Context, run Runner) []string {
seen := map[string]bool{}
var out []string
add := func(p string) {
p = strings.Trim(strings.TrimSpace(p), `"'`)
if !filepath.IsAbs(p) {
return
}
p = filepath.Clean(p)
if !seen[p] {
seen[p] = true
out = append(out, p)
}
}
if run != nil {
// Each runtime has its own ten seconds: one that hangs costs the other nothing.
for _, q := range [][]string{{"docker", "info", "--format", "{{.DockerRootDir}}"},
{"podman", "info", "--format", "{{.Store.GraphRoot}}"}} {
ask, cancel := context.WithTimeout(ctx, 10*time.Second)
if root, err := run(ask, q[0], q[1:]...); err == nil {
add(root)
}
cancel()
}
}
daemonJSON := func(path string) {
raw, err := os.ReadFile(path)
if err != nil {
return
}
var c struct {
DataRoot string `json:"data-root"`
Graph string `json:"graph"`
}
if json.Unmarshal(raw, &c) == nil {
add(c.DataRoot)
add(c.Graph)
}
}
daemonJSON(runtimeConfigs.daemonJSON)
units := append([]string(nil), runtimeConfigs.units...)
for _, dir := range runtimeConfigs.dropInDirs {
matches, _ := filepath.Glob(filepath.Join(dir, "*.conf"))
units = append(units, matches...)
}
// The unit and its drop-ins are one unit to systemd: a variable set in one is seen by an ExecStart in another.
env := map[string]string{}
var execs []string
for _, u := range units {
raw, err := os.ReadFile(u)
if err != nil {
continue
}
e, x := unitLines(string(raw))
for k, v := range e {
env[k] = v
}
execs = append(execs, x...)
}
for _, line := range execs {
line = envVar.ReplaceAllStringFunc(line, func(ref string) string {
m := envVar.FindStringSubmatch(ref)
if v, ok := env[m[1]+m[2]]; ok {
return v
}
return ref
})
for _, m := range dataRootFlag.FindAllStringSubmatch(line, -1) {
add(m[1])
}
for _, m := range configFlag.FindAllStringSubmatch(line, -1) {
daemonJSON(strings.Trim(m[1], `"'`))
}
}
if raw, err := os.ReadFile(runtimeConfigs.storageConf); err == nil {
for _, m := range graphRoot.FindAllStringSubmatch(string(raw), -1) {
add(m[1] + m[2])
}
}
return out
}
// unitLines reads a unit file as systemd does for what matters here: a line ending in a backslash continues on the
// next, Environment= sets variables (quoted or not, several to a line), EnvironmentFile= (a leading - says it may
// be missing) reads KEY=value lines, and every ExecStart line is returned whole.
func unitLines(text string) (map[string]string, []string) {
var joined []string
var cur strings.Builder
for _, line := range strings.Split(text, "\n") {
trimmed := strings.TrimRight(line, " \t")
if strings.HasSuffix(trimmed, "\\") {
cur.WriteString(strings.TrimSuffix(trimmed, "\\") + " ")
continue
}
cur.WriteString(line)
joined = append(joined, cur.String())
cur.Reset()
}
if cur.Len() > 0 {
joined = append(joined, cur.String())
}
env := map[string]string{}
setPairs := func(s string) {
for _, f := range splitQuoted(s) {
if k, v, ok := strings.Cut(f, "="); ok {
env[strings.TrimSpace(k)] = strings.Trim(strings.TrimSpace(v), `"'`)
}
}
}
var execs []string
for _, line := range joined {
l := strings.TrimSpace(line)
switch {
case strings.HasPrefix(l, "Environment="):
setPairs(strings.TrimPrefix(l, "Environment="))
case strings.HasPrefix(l, "EnvironmentFile="):
path := strings.TrimPrefix(strings.TrimSpace(strings.TrimPrefix(l, "EnvironmentFile=")), "-")
if raw, err := os.ReadFile(path); err == nil {
for _, kv := range strings.Split(string(raw), "\n") {
kv = strings.TrimSpace(kv)
if kv == "" || strings.HasPrefix(kv, "#") {
continue
}
setPairs(kv)
}
}
case strings.HasPrefix(l, "ExecStart"):
execs = append(execs, l)
}
}
return env, execs
}
// splitQuoted splits on blanks outside double or single quotes, keeping the quotes' contents whole.
func splitQuoted(s string) []string {
var out []string
var cur strings.Builder
var quote rune
for _, r := range s {
switch {
case quote != 0 && r == quote:
quote = 0
case quote == 0 && (r == '"' || r == '\''):
quote = r
case quote == 0 && (r == ' ' || r == '\t'):
if cur.Len() > 0 {
out = append(out, cur.String())
cur.Reset()
}
default:
cur.WriteRune(r)
}
}
if cur.Len() > 0 {
out = append(out, cur.String())
}
return out
}
// engineModule is the module whose resources may place in the engine's own trees.
const engineModule = "mesh-host"
// passwdFile is the user database homes are read from. A variable so a test names its own.
var passwdFile = "/etc/passwd"
// PlacementRefusedError is a resource the engine will not place, or mount, where it says.
type PlacementRefusedError struct {
Path, Why string
}
func (e *PlacementRefusedError) Error() string {
return fmt.Sprintf("%s is not placed: %s (novox/hq issue 339); nothing was touched", e.Path, e.Why)
}
// homeAccount is a person's or an agent's account and its home.
type homeAccount struct {
Name string
UID int
}
// accountsOfHomes is each person's or agent's home and the account it belongs to, from the user database: an
// account with a uid of 1000 or more, or a home under /home. A variable so a test names its own.
var accountsOfHomes = func() map[string]homeAccount {
f, err := os.Open(passwdFile)
if err != nil {
return nil
}
defer f.Close()
out := map[string]homeAccount{}
sc := bufio.NewScanner(f)
for sc.Scan() {
fields := strings.Split(sc.Text(), ":")
if len(fields) < 6 {
continue
}
uid, err := strconv.Atoi(fields[2])
if err != nil {
continue
}
home := filepath.Clean(fields[5])
if home == "/" || home == "." || home == "" || home == "/nonexistent" {
continue
}
if (uid >= 1000 && uid != 65534) || strings.HasPrefix(home, "/home/") {
out[home] = homeAccount{Name: fields[0], UID: uid}
}
}
return out
}
// below says whether path is strictly below dir.
func below(path, dir string) bool {
if dir == "/" {
return path != "/"
}
return strings.HasPrefix(path, dir+"/")
}
// atOrBelow says whether path is dir or below it.
func atOrBelow(path, dir string) bool { return path == dir || below(path, dir) }
// resolved is a path with every link in it followed, as far as the path exists, and the rest as declared.
func resolved(path string) string {
rest := ""
for p := path; ; p = filepath.Dir(p) {
if real, err := filepath.EvalSymlinks(p); err == nil {
return filepath.Clean(filepath.Join(real, rest))
}
if filepath.Dir(p) == p {
return path
}
rest = filepath.Join(filepath.Base(p), rest)
}
}
// ownedByAccount says whether a declared owner is that account: by name, or by its uid ("1001", "1001:1001").
func ownedByAccount(owner string, a homeAccount) bool {
if owner == a.Name {
return true
}
user, _, _ := strings.Cut(owner, ":")
if uid, err := strconv.Atoi(user); err == nil {
return uid == a.UID
}
return false
}
// rootOwner says whether a declared owner is root: none, "root", or uid 0.
func rootOwner(owner string) bool {
if owner == "" || owner == "root" {
return true
}
user, _, _ := strings.Cut(owner, ":")
return user == "0"
}
// what a guarded path is, for the words of a refusal.
type placing int
const (
placingDirectory placing = iota
placingAccess
placingMount
)
// refusePath says why a path is not placed or mounted; nil when it may be. module is the resource's module,
// owner a directory's declared owner.
func refusePath(path string, kind placing, module, owner string) error {
clean := filepath.Clean(path)
if !filepath.IsAbs(clean) {
return nil // the declaration refuses a relative path already; a named volume is not a path
}
for _, p := range []string{clean, resolved(clean)} {
if err := refuseOne(p, kind, module, owner); err != nil {
if p != clean {
err.Why = fmt.Sprintf("through a link, it is %s, and %s", p, err.Why)
err.Path = clean
}
return err
}
}
return nil
}
func refuseOne(path string, kind placing, module, owner string) *PlacementRefusedError {
for _, root := range protectedRoots {
if path == root || below(root, path) {
return &PlacementRefusedError{Path: path, Why: root + " is one of the machine's own directories, " +
"and owning or mounting it would be owning or mounting everything in it"}
}
}
trees := append([]string(nil), forbiddenBelow...)
if kind == placingMount {
// A mount is the machine's plumbing as often as a module's data — the clock, a kernel file, /dev/null
// (systemPath) — and what a setting can mount at all is a directory or an access, refused above it.
trees = append([]string(nil), forbiddenBelowMount...)
}
// The container runtimes' data, wherever the machine keeps it: every container's filesystem is there.
trees = append(trees, runtimeDataRoots()...)
for _, tree := range trees {
if atOrBelow(path, tree) {
return &PlacementRefusedError{Path: path, Why: "nothing is placed in or mounted from " + tree}
}
}
if module != engineModule {
for _, tree := range engineTrees {
if atOrBelow(path, tree) {
return &PlacementRefusedError{Path: path, Why: tree + " is the node-engine's own, placed in by " +
"its own module alone"}
}
}
}
for _, tree := range rootsOnly {
if !below(path, tree) {
continue
}
switch {
case kind == placingAccess:
return &PlacementRefusedError{Path: path, Why: "an access is the operator's data, and " + tree +
" is the machine's own"}
case kind == placingDirectory && !rootOwner(owner):
return &PlacementRefusedError{Path: path, Why: fmt.Sprintf("a directory below %s is root's, and this "+
"one is declared %s's", tree, owner)}
}
}
ssh := false
for _, part := range strings.Split(path, "/") {
ssh = ssh || part == ".ssh"
}
if ssh && kind != placingDirectory {
return &PlacementRefusedError{Path: path, Why: "it is an account's .ssh, which holds its keys and who may " +
"log in as it, and is never an access or a mount"}
}
if kind != placingDirectory {
return nil
}
homes := accountsOfHomes()
var deepest string
for home := range homes {
if below(path, home) && len(home) > len(deepest) {
deepest = home
}
}
if ssh && deepest == "" {
return &PlacementRefusedError{Path: path, Why: "a .ssh directory is placed only below its account's home, " +
"as that account's"}
}
if deepest != "" {
if a := homes[deepest]; !ownedByAccount(owner, a) {
if owner == "" {
owner = "root"
}
return &PlacementRefusedError{Path: path, Why: fmt.Sprintf("it is below %s's home and declared %s's; "+
"below a home only that account's directories are placed", a.Name, owner)}
}
}
return nil
}
// moduleOfID is the module a resource id names, or "".
func moduleOfID(id string) string {
module, _ := moduleOf(id)
return module
}
// refusedPlaces judges every directory and access of a declaration before anything is applied, and answers
// each refusal by path: what is refused is refused again as a container's mount source.
func refusedPlaces(resources []declaration.Resource) map[string]error {
out := map[string]error{}
for _, r := range resources {
var err error
switch res := r.(type) {
case *declaration.Directory:
err = refusePath(res.Path, placingDirectory, moduleOfID(res.ID), res.Owner)
case *declaration.Access:
err = refusePath(res.Path, placingAccess, moduleOfID(res.ID), "")
default:
continue
}
if err != nil {
out[filepath.Clean(r.Target())] = err
}
}
return out
}
// refuseMounts says why a container's mounts are not made; nil when they may be.
func refuseMounts(c *declaration.Container, refused map[string]error) error {
for _, v := range c.Volumes {
src := mountSource(v)
if !strings.HasPrefix(src, "/") {
continue // a named volume, which the runtime keeps in its own tree
}
src = filepath.Clean(src)
for path, why := range refused {
if atOrBelow(src, path) {
var refusal *PlacementRefusedError
if errors.As(why, &refusal) {
return &PlacementRefusedError{Path: src, Why: "it is mounted from " + path +
", which is refused: " + refusal.Why}
}
return why
}
}
if err := refusePath(src, placingMount, moduleOfID(c.ID), ""); err != nil {
return err
}
}
return nil
}
+14
View File
@@ -0,0 +1,14 @@
# The tests that need the throwaway store and must have run against it (novox/hq issue 459): merge-check.sh runs
# each one named here, by package, and fails unless every one passed — a test that needs the store skips without
# one, and a skip passes unseen. One per line: <package> <TestName>. Lines starting with # are said nothing of.
./cmd/mesh-controller TestAStopWithoutAMergeWalksTheOthersAgain
./cmd/mesh-controller TestAStopWithoutAMergeEndsTheMergesThatContainIt
./cmd/mesh-controller TestAStopWalksTheOthersAgainInTheOpenBatch
./cmd/mesh-controller TestAStopNamesWhatStaysOnTheMachinesAndWhatStillCarriesIt
./cmd/mesh-controller TestAStopWithoutAMergeLeavesItOutOfABatch
./cmd/mesh-controller TestSavingPlansAndMovingMergesIsOneAct
./internal/inventory TestTheLastSendSaysWhatTheMachineAnswered
./internal/inventory TestAReportIsTheLastSendsAnswerOnlyWhenItIsAboutThatSend
./cmd/mesh-controller TestNodeShowAndStatusReadTheLastSendFromTheStore
./internal/inventory TestAnIssuedValueIsSealedToEveryHolderOnceAndNotAgain
./internal/inventory TestAnIssuedValueIsSealedAgainAfterAKeyChangeOrAnAcceptance