Merge remote-tracking branch 'origin/main' into adr-0297-module-logs
This commit is contained in:
@@ -408,6 +408,9 @@ func comparisonOf(module string, c comparison, opts takeOptions) (preview string
|
||||
switch {
|
||||
case sec.Origin == inventory.OriginAccepted:
|
||||
fmt.Fprintf(&b, " %s %s: accepted from a person, carried in as it is\n", what, name)
|
||||
case sec.Origin == inventory.OriginIssued:
|
||||
// Taken from the provider's software (novox/hq ADR 0305): the value the running service uses.
|
||||
fmt.Fprintf(&b, " %s %s: issued by the provider's software, carried in as it is\n", what, name)
|
||||
case opts.Mint[sec.Name]:
|
||||
fmt.Fprintf(&b, " %s %s: minted by the mesh; the service takes the new value, as --mint said\n", what, name)
|
||||
case foundData:
|
||||
|
||||
@@ -367,7 +367,13 @@ func keepBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan,
|
||||
}
|
||||
closes, latest := windowOf(merges, b.Created, window, atMost)
|
||||
b.Delivery.Merges = named
|
||||
b.Delivery.Batch = &inventory.PlanBatch{ClosesAt: closes, AtMost: latest, Behind: behind, Own: b.OwnPath()}
|
||||
var closed bool
|
||||
var withheld []string
|
||||
if was := b.Delivery.Batch; was != nil {
|
||||
closed, withheld = was.Closed, was.Withheld // a stop's, which no look takes back (novox/hq ADR 0299)
|
||||
}
|
||||
b.Delivery.Batch = &inventory.PlanBatch{ClosesAt: closes, AtMost: latest, Behind: behind, Own: b.OwnPath(),
|
||||
Closed: closed, Withheld: withheld}
|
||||
b.State = inventory.PlanAssembling
|
||||
if behind != "" && windowClosed(*b, now) {
|
||||
b.State = inventory.PlanQueued
|
||||
@@ -402,7 +408,7 @@ func windowClosed(b inventory.Plan, now time.Time) bool {
|
||||
return true
|
||||
}
|
||||
w := b.Delivery.Batch
|
||||
return !now.Before(w.ClosesAt) || !now.Before(w.AtMost)
|
||||
return w.Closed || !now.Before(w.ClosesAt) || !now.Before(w.AtMost)
|
||||
}
|
||||
|
||||
// carriedOf is a set of merges as a walk carries them: the latest merge of each repository's branch, and
|
||||
@@ -762,7 +768,18 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
|
||||
}
|
||||
sort.Strings(also)
|
||||
ordered := groupOrderEdges(members, entries, read, edges)
|
||||
var withheld []string
|
||||
plan := planOfMoves(moved, append(append([]inventory.Edge{}, edges...), ordered...))
|
||||
// **Withheld, a stopped merge's modules** (novox/hq ADR 0299): a module is built at its branch, which holds the
|
||||
// merge a stop left out of the walk these merges were walked again from — built here only as a dependent, it
|
||||
// would deliver that merge. One a merge of this batch moves is built: that merge carries the stopped change,
|
||||
// and the stop named it.
|
||||
if batch.Delivery != nil && batch.Delivery.Batch != nil {
|
||||
if out := withhold(&plan, batch.Delivery.Batch.Withheld, moved); len(out) > 0 {
|
||||
withheld = out
|
||||
fmt.Printf(" withheld, a stopped merge's modules built only as dependents: %s\n", strings.Join(out, ", "))
|
||||
}
|
||||
}
|
||||
plan.ID, plan.Revision, plan.Created, plan.Epoch = batch.ID, batch.Revision, now, batch.Epoch
|
||||
plan.Commits = commits
|
||||
newest := newestCommit(commits)
|
||||
@@ -773,6 +790,7 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
|
||||
}
|
||||
plan.Delivery.Merges = named
|
||||
plan.Delivery.Alone = batch.Delivery != nil && batch.Delivery.Alone
|
||||
plan.Delivery.Withheld = withheld
|
||||
// **Its moments and its class** (novox/hq ADR 0282 decision 6): measured, never acted on.
|
||||
plan.Times = walkTimesAtCut(*batch, plan, entries, now)
|
||||
if len(moved) == 0 {
|
||||
@@ -823,6 +841,39 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
|
||||
return nil
|
||||
}
|
||||
|
||||
// withhold takes from a plan every module named that its merges do not move, and the tiers left empty: the
|
||||
// modules taken.
|
||||
func withhold(plan *inventory.Plan, names, moved []string) []string {
|
||||
var out []string
|
||||
for _, n := range names {
|
||||
if _, in := plan.Modules[n]; in && !slices.Contains(moved, n) {
|
||||
delete(plan.Modules, n)
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
var tiers [][]string
|
||||
for _, t := range plan.Tiers {
|
||||
var kept []string
|
||||
for _, n := range t {
|
||||
if !slices.Contains(out, n) {
|
||||
kept = append(kept, n)
|
||||
}
|
||||
}
|
||||
if len(kept) > 0 {
|
||||
tiers = append(tiers, kept)
|
||||
}
|
||||
}
|
||||
if tiers == nil {
|
||||
tiers = [][]string{}
|
||||
}
|
||||
plan.Tiers = tiers
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// walkTimesAtCut is a walk's own moments as it is cut (novox/hq ADR 0282 decision 6): when its batch's window
|
||||
// closed — no merge for the window's length, or its maximum, whichever came first — when it was cut, and its
|
||||
// class. A merge walked alone had no window.
|
||||
|
||||
@@ -835,6 +835,9 @@ type answers struct {
|
||||
// public name on the machine went dark. The holds were correct; they were recorded only in the
|
||||
// machine's own state file, and the one visible symptom was a count that did not add up.
|
||||
untaken map[string]map[string]int
|
||||
// leftOut is, per machine, every module of its set its composition leaves out, and why (novox/hq issue
|
||||
// 380): assigned and not applied, which every push said only in passing. Not well while there is any.
|
||||
leftOut map[string][]leftOutModule
|
||||
// unheld is every module on a machine whose resources are applied through a seat nothing on
|
||||
// that machine holds (novox/hq ADR 0207), with the modules that could hold it. Reported, not
|
||||
// refused, until the switch — and while there is any, the mesh is not all well: the order the
|
||||
@@ -863,6 +866,10 @@ type answers struct {
|
||||
// applied, is not well: an assignment somebody made is not made yet, or will not be.
|
||||
pending []inventory.PendingAssignment
|
||||
pendingUnread string
|
||||
// lastSent is every machine's last recorded send, by name, with what the machine answered of it (novox/hq
|
||||
// issue 485); lastSentUnread why it could not be read.
|
||||
lastSent map[string]inventory.Send
|
||||
lastSentUnread string
|
||||
}
|
||||
|
||||
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
|
||||
|
||||
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
@@ -28,6 +29,14 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
|
||||
for _, e := range entries {
|
||||
byName[e.Manifest.Module] = e
|
||||
}
|
||||
// The machine holding the bus, whose declaration carries every module's grants (novox/hq issue 490).
|
||||
holder := ""
|
||||
for _, e := range entries {
|
||||
if e.Manifest.BusUsers != "" && e.Manifest.ClaimsSeat(catalogue.BrokerSeat) && len(e.On) > 0 {
|
||||
holder = e.On[0]
|
||||
}
|
||||
}
|
||||
var granting []string
|
||||
machines := map[string]*link.MachinePlan{}
|
||||
machine := func(name string) *link.MachinePlan {
|
||||
if machines[name] == nil {
|
||||
@@ -50,6 +59,10 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
|
||||
machine(on).Receives = append(machine(on).Receives, name)
|
||||
}
|
||||
}
|
||||
if !waits && holder != "" && !(len(e.On) == 1 && e.On[0] == holder) && len(e.On) > 0 &&
|
||||
!slices.Contains(granting, name) {
|
||||
granting = append(granting, name)
|
||||
}
|
||||
switch {
|
||||
case (name == "nats" || catalogue.ProvidesBus(e.Manifest)) && len(e.On) > 0:
|
||||
// Said before the merge (novox/hq issue 336): a new bus build holds every send to the bus's machine
|
||||
@@ -76,6 +89,13 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(granting) > 0 {
|
||||
// Said before the merge (novox/hq issue 490): what the walk does when the change alters the bus's
|
||||
// user list. Whether it does is known only once the builds are registered, so it is said as a rule.
|
||||
p.Steps = append(p.Steps, fmt.Sprintf("%s: if this changes what the bus's user list says (a new tool, "+
|
||||
"subject or user), %s is sent that list alone, every build there kept, before %s is judged on its first "+
|
||||
"machine", grantsStepWord, holder, strings.Join(granting, ", ")))
|
||||
}
|
||||
var names []string
|
||||
for name := range machines {
|
||||
names = append(names, name)
|
||||
@@ -88,6 +108,10 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
|
||||
return p
|
||||
}
|
||||
|
||||
// grantsStepWord names the grants step (novox/hq issue 490): the bus's user list sent alone to the machine
|
||||
// holding the bus ahead of a walk's gate. Not the bus step, which replaces the bus itself.
|
||||
const grantsStepWord = "grants step"
|
||||
|
||||
// busUpgradeNeeded is how a change plan says that merging it holds the bus's machine for a person's step.
|
||||
const busUpgradeNeeded = "merging this needs a person's bus upgrade"
|
||||
|
||||
|
||||
+404
-13
@@ -117,28 +117,411 @@ func letGo(ctx context.Context, inv *inventory.Inventory, id, by, why string) (i
|
||||
// stopWalk ends a walk on its delivery's word: failed, said as stopped by whom, why. What it asked still
|
||||
// builds and registers; nothing further is asked or sent.
|
||||
func stopWalk(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, error) {
|
||||
p, _, err := stopWalkWithout(ctx, inv, id, by, why, nil, time.Now().UTC())
|
||||
return p, err
|
||||
}
|
||||
|
||||
// stopAnswer is what a stop that leaves merges out says, as data (novox/hq ADR 0299): the record stopped, the
|
||||
// batch that walks its other merges again with those merges, every merge left out, the modules the stopped walk
|
||||
// already sent that stay on the machines named, the modules the batch's walk withholds, and every merge elsewhere
|
||||
// that will deliver a stopped change when it is walked — a later merge of its repository, or one moving its modules.
|
||||
type stopAnswer struct {
|
||||
Stopped string `json:"stopped"`
|
||||
WalkedAgainBy string `json:"walked_again_by,omitempty"`
|
||||
WalkedAgain []string `json:"walked_again,omitempty"`
|
||||
LeftOut []inventory.PlanLeftOut `json:"left_out,omitempty"`
|
||||
Kept []inventory.PlanKept `json:"kept,omitempty"`
|
||||
Withheld []string `json:"withheld,omitempty"`
|
||||
StillCarriedBy []string `json:"still_carried_by,omitempty"`
|
||||
}
|
||||
|
||||
// stopWalkWithout ends a walk on its delivery's word and, given merges to leave out, walks every other merge it
|
||||
// answered again without them (novox/hq issue 459, ADR 0299): one walk carries every merge of its batch, so ending
|
||||
// it for one group member ended every other delivery on it.
|
||||
//
|
||||
// - Left out are the merges named, each later merge of their repository's branch, which contains one, and each
|
||||
// merge moving a module a left-out merge moves: a module is built at its branch, which holds the stopped merge.
|
||||
// - The others join the open batch of their kind — merges heard while the walk ran wait there, and an older
|
||||
// commit never walks after a newer one — or a new batch when none is open; its window is closed, and its walk
|
||||
// withholds the left-out merges' modules where it would build them only as dependents.
|
||||
// - Given a batch not yet cut, the merges left out leave it for a stopped record of their own, and the batch
|
||||
// goes on without them; with nothing left in it, the batch itself is stopped.
|
||||
// - The stopped record, the batch and every merge moved are written in one transaction (decision 7).
|
||||
//
|
||||
// With nothing left to walk, or no merge named, the walk is only ended. A merge named that the record does not
|
||||
// answer refuses the stop whole.
|
||||
func stopWalkWithout(ctx context.Context, inv *inventory.Inventory, id, by, why string, without []string,
|
||||
now time.Time) (inventory.Plan, stopAnswer, error) {
|
||||
said := stopAnswer{Stopped: id}
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, err
|
||||
return inventory.Plan{}, said, err
|
||||
}
|
||||
defer release()
|
||||
p, err := inv.PlanByID(ctx, id)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, said, err
|
||||
}
|
||||
batch := p.Batch()
|
||||
if !p.Open() && !(batch && len(without) > 0) {
|
||||
return p, said, fmt.Errorf("%s is already %s", p.ID, p.State)
|
||||
}
|
||||
var merges, again []inventory.BatchedMerge
|
||||
if len(without) > 0 {
|
||||
if merges, err = inv.MergesOf(ctx, p.ID); err != nil {
|
||||
return p, said, err
|
||||
}
|
||||
if again, said.LeftOut, err = leaveOut(merges, without); err != nil {
|
||||
return p, said, fmt.Errorf("%s is not stopped: %w", p.ID, err)
|
||||
}
|
||||
}
|
||||
stopped := func(r *inventory.Plan) {
|
||||
if r.Delivery == nil {
|
||||
r.Delivery = &inventory.PlanDelivery{}
|
||||
}
|
||||
r.Delivery.Stopped, r.Delivery.StoppedWhy, r.Delivery.LeftOut = by, why, said.LeftOut
|
||||
r.State = inventory.PlanFailed
|
||||
r.Note = fmt.Sprintf("stopped by %s at tier %d: %s", by, r.Tier, why)
|
||||
}
|
||||
withheld := movesOf(leftOutMerges(merges, said.LeftOut))
|
||||
said.Withheld = withheld
|
||||
if said.StillCarriedBy, err = stillCarried(ctx, inv, p.ID, merges, said.LeftOut, withheld); err != nil {
|
||||
return p, said, err
|
||||
}
|
||||
if batch {
|
||||
return p, said, stopInBatch(ctx, inv, &p, merges, again, &said, stopped, now)
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return p, said, err
|
||||
}
|
||||
running := map[string][]string{}
|
||||
for _, e := range entries {
|
||||
running[e.Manifest.Module] = e.On
|
||||
}
|
||||
said.Kept = keptOf(p, movesOf(again), running)
|
||||
stopped(&p)
|
||||
p.Delivery.Kept = said.Kept
|
||||
if len(again) == 0 {
|
||||
if len(said.LeftOut) > 0 {
|
||||
p.Note += fmt.Sprintf("; left out %s, nothing else to walk", mergesLeftOutWords(said.LeftOut))
|
||||
}
|
||||
return p, said, inv.SavePlan(ctx, &p)
|
||||
}
|
||||
target, err := batchToWalkAgain(ctx, inv, again, now)
|
||||
if err != nil {
|
||||
return p, said, fmt.Errorf("%s is not stopped: %w", p.ID, err)
|
||||
}
|
||||
target.Delivery.Batch.Closed = true
|
||||
target.Delivery.Batch.Withheld = union(target.Delivery.Batch.Withheld, withheld)
|
||||
sort.Strings(target.Delivery.Batch.Withheld)
|
||||
said.WalkedAgainBy = target.ID
|
||||
var moves []inventory.MergeMove
|
||||
for _, m := range again {
|
||||
said.WalkedAgain = append(said.WalkedAgain, m.Repository+"@"+m.Commit)
|
||||
moves = append(moves, inventory.MergeMove{Repository: m.Repository, Commit: m.Commit, Plan: target.ID, Alone: m.Alone})
|
||||
}
|
||||
p.Delivery.WalkedAgainBy = target.ID
|
||||
p.Note += fmt.Sprintf("; its other merges are walked again by %s: %s", target.ID, strings.Join(said.WalkedAgain, ", "))
|
||||
if len(said.LeftOut) > 0 {
|
||||
p.Note += "; left out " + mergesLeftOutWords(said.LeftOut)
|
||||
}
|
||||
if len(said.Kept) > 0 {
|
||||
p.Note += "; " + keptWords(said.Kept)
|
||||
}
|
||||
// The batch first, so no merge names a plan the store does not hold; all of it, or nothing (decision 7).
|
||||
if err := inv.SavePlansMoving(ctx, []*inventory.Plan{&target, &p}, moves); err != nil {
|
||||
return p, said, fmt.Errorf("%s is not stopped, and none of its merges moved: %w", p.ID, err)
|
||||
}
|
||||
// What the batch holds and names now: written again by the cutter's next look should this fail.
|
||||
if err := keepBatch(ctx, inv, &target, now, ""); err != nil {
|
||||
fmt.Printf(" %s holds its merges; its record is written again at the next look: %v\n", target.ID, err)
|
||||
}
|
||||
fmt.Printf("%s stopped by %s; %s walks its other merges again: %s\n", p.ID, by, target.ID,
|
||||
strings.Join(said.WalkedAgain, ", "))
|
||||
return p, said, nil
|
||||
}
|
||||
|
||||
// stopInBatch leaves merges out of a batch not yet cut (ADR 0299): they go to a stopped record of their own, which
|
||||
// names the batch as walking the others, and the batch withholds their modules; a batch left with none is stopped
|
||||
// itself.
|
||||
func stopInBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan, merges, again []inventory.BatchedMerge,
|
||||
said *stopAnswer, stopped func(*inventory.Plan), now time.Time) error {
|
||||
if len(again) == 0 {
|
||||
stopped(b)
|
||||
b.Delivery.Batch = nil
|
||||
b.Note = "a batch whose every merge was left out by a stop: " + b.Note
|
||||
return inv.SavePlan(ctx, b)
|
||||
}
|
||||
left := leftOutMerges(merges, said.LeftOut)
|
||||
first := left[0]
|
||||
r := inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch,
|
||||
Commit: first.Commit, Merged: first.Merged, Created: now, Tiers: [][]string{},
|
||||
Modules: map[string]*inventory.PlanModule{}}
|
||||
if r.ID == b.ID {
|
||||
r.ID += "-left"
|
||||
}
|
||||
stopped(&r)
|
||||
r.Delivery.WalkedAgainBy = b.ID
|
||||
r.Note += fmt.Sprintf("; left out of %s before it was walked: %s; %s walks the others", b.ID,
|
||||
mergesLeftOutWords(said.LeftOut), b.ID)
|
||||
b.Delivery.Batch.Withheld = union(b.Delivery.Batch.Withheld, said.Withheld)
|
||||
sort.Strings(b.Delivery.Batch.Withheld)
|
||||
var moves []inventory.MergeMove
|
||||
for _, m := range left {
|
||||
moves = append(moves, inventory.MergeMove{Repository: m.Repository, Commit: m.Commit, Plan: r.ID, Alone: m.Alone})
|
||||
}
|
||||
for _, m := range again {
|
||||
said.WalkedAgain = append(said.WalkedAgain, m.Repository+"@"+m.Commit)
|
||||
}
|
||||
said.Stopped, said.WalkedAgainBy = r.ID, b.ID
|
||||
if err := inv.SavePlansMoving(ctx, []*inventory.Plan{&r, b}, moves); err != nil {
|
||||
return fmt.Errorf("%s is not changed, and none of its merges moved: %w", b.ID, err)
|
||||
}
|
||||
if err := keepBatch(ctx, inv, b, now, ""); err != nil {
|
||||
fmt.Printf(" %s holds its merges; its record is written again at the next look: %v\n", b.ID, err)
|
||||
}
|
||||
fmt.Printf("%s: %s left out by a stop, kept by %s\n", b.ID, mergesLeftOutWords(said.LeftOut), r.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// batchToWalkAgain is the batch a stopped walk's merges join: the open batch of their kind, or a new one, not yet
|
||||
// written. The kind is read from what the merges moved.
|
||||
func batchToWalkAgain(ctx context.Context, inv *inventory.Inventory, again []inventory.BatchedMerge,
|
||||
now time.Time) (inventory.Plan, error) {
|
||||
own := false
|
||||
for _, n := range movesOf(again) {
|
||||
if _, on := onTheControllersPath[n]; on {
|
||||
own = true
|
||||
}
|
||||
}
|
||||
batches, err := inv.Batches(ctx)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, err
|
||||
}
|
||||
if !p.Open() {
|
||||
return p, fmt.Errorf("%s is already %s", p.ID, p.State)
|
||||
for _, b := range batches {
|
||||
if b.OwnPath() == own {
|
||||
if b.Delivery == nil {
|
||||
b.Delivery = &inventory.PlanDelivery{}
|
||||
}
|
||||
if b.Delivery.Batch == nil {
|
||||
b.Delivery.Batch = &inventory.PlanBatch{Own: own}
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
}
|
||||
if p.Delivery == nil {
|
||||
p.Delivery = &inventory.PlanDelivery{}
|
||||
first := again[0]
|
||||
return inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch,
|
||||
Commit: first.Commit, Merged: first.Merged, Created: now, State: inventory.PlanAssembling,
|
||||
Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{},
|
||||
Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: own}}}, nil
|
||||
}
|
||||
|
||||
// leaveOut splits a walk's merges by the merges named to leave out, each `<repository>@<commit>` (a commit's
|
||||
// first characters will do): the merges walked again, and those left out — each named one; each later merge of
|
||||
// its repository's branch, which contains it; and each merge moving a module a left-out merge moves, which is
|
||||
// built at its branch and so holds the stopped merge — until none more is. A name the walk does not answer is
|
||||
// refused.
|
||||
func leaveOut(merges []inventory.BatchedMerge, without []string) ([]inventory.BatchedMerge, []inventory.PlanLeftOut, error) {
|
||||
var stopped []inventory.BatchedMerge
|
||||
for _, w := range without {
|
||||
repository, commit, ok := strings.Cut(strings.TrimSpace(w), "@")
|
||||
if !ok || repository == "" || len(commit) < 7 {
|
||||
return nil, nil, fmt.Errorf("%q names no merge: <repository>@<commit>", w)
|
||||
}
|
||||
i := slices.IndexFunc(merges, func(m inventory.BatchedMerge) bool {
|
||||
return strings.EqualFold(m.Repository, repository) && strings.HasPrefix(m.Commit, commit)
|
||||
})
|
||||
if i < 0 {
|
||||
return nil, nil, fmt.Errorf("it answers no merge %s@%s", repository, commit)
|
||||
}
|
||||
stopped = append(stopped, merges[i])
|
||||
}
|
||||
p.Delivery.Stopped, p.Delivery.StoppedWhy = by, why
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = fmt.Sprintf("stopped by %s at tier %d: %s", by, p.Tier, why)
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
return p, err
|
||||
same := func(a, b inventory.BatchedMerge) bool {
|
||||
return strings.EqualFold(a.Repository, b.Repository) && a.Commit == b.Commit
|
||||
}
|
||||
return p, nil
|
||||
out := map[int]inventory.PlanLeftOut{}
|
||||
for i, m := range merges {
|
||||
if slices.ContainsFunc(stopped, func(s inventory.BatchedMerge) bool { return same(s, m) }) {
|
||||
out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit}
|
||||
}
|
||||
}
|
||||
for grew := true; grew; {
|
||||
grew = false
|
||||
for i, m := range merges {
|
||||
if _, left := out[i]; left {
|
||||
continue
|
||||
}
|
||||
for j, o := range out {
|
||||
l := merges[j]
|
||||
contains := o.Contains
|
||||
if contains == "" {
|
||||
contains = l.Commit
|
||||
}
|
||||
if strings.EqualFold(l.Repository, m.Repository) && l.Branch == m.Branch && laterMerge(m, l) {
|
||||
out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit, Contains: contains}
|
||||
} else if shared := sharedMoves(m, l); len(shared) > 0 {
|
||||
out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit, Contains: contains,
|
||||
Shares: shared}
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
grew = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
var again []inventory.BatchedMerge
|
||||
var left []inventory.PlanLeftOut
|
||||
for i, m := range merges {
|
||||
if o, is := out[i]; is {
|
||||
left = append(left, o)
|
||||
} else {
|
||||
again = append(again, m)
|
||||
}
|
||||
}
|
||||
sort.SliceStable(left, func(i, j int) bool { return left[i].Contains == "" && left[j].Contains != "" })
|
||||
return again, left, nil
|
||||
}
|
||||
|
||||
// sharedMoves are the modules two kept merges both moved when they were heard.
|
||||
func sharedMoves(a, b inventory.BatchedMerge) []string {
|
||||
var out []string
|
||||
theirs := announcedOf(b).Moves
|
||||
for _, n := range announcedOf(a).Moves {
|
||||
if slices.Contains(theirs, n) && !slices.Contains(out, n) {
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// leftOutMerges are the kept merges a stop left out.
|
||||
func leftOutMerges(merges []inventory.BatchedMerge, out []inventory.PlanLeftOut) []inventory.BatchedMerge {
|
||||
var left []inventory.BatchedMerge
|
||||
for _, m := range merges {
|
||||
if slices.ContainsFunc(out, func(o inventory.PlanLeftOut) bool {
|
||||
return strings.EqualFold(o.Repository, m.Repository) && o.Commit == m.Commit
|
||||
}) {
|
||||
left = append(left, m)
|
||||
}
|
||||
}
|
||||
return left
|
||||
}
|
||||
|
||||
// movesOf is every module kept merges moved when they were heard, sorted.
|
||||
func movesOf(merges []inventory.BatchedMerge) []string {
|
||||
var out []string
|
||||
for _, m := range merges {
|
||||
out = union(out, announcedOf(m).Moves)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// keptOf are the modules a stopped walk sent that no merge walked again moves, and the machines each reached:
|
||||
// those keep running the stopped walk's build. A module sent to its first machines reached those, and the rest it
|
||||
// was sent to after them; one sent with no first machine (its policy sends it to all together, or a later tier is
|
||||
// built by it) reached every machine running it.
|
||||
func keptOf(p inventory.Plan, walkedAgain []string, running map[string][]string) []inventory.PlanKept {
|
||||
var out []inventory.PlanKept
|
||||
for name, m := range p.Modules {
|
||||
if m == nil || slices.Contains(walkedAgain, name) {
|
||||
continue
|
||||
}
|
||||
var machines []string
|
||||
if m.FirstAt != nil || m.SentAt != nil {
|
||||
machines = union(machines, m.First)
|
||||
}
|
||||
if m.SentAt != nil && len(m.First) == 0 {
|
||||
machines = union(machines, running[name])
|
||||
}
|
||||
if m.Gate != nil && m.Gate.Since != nil {
|
||||
machines = union(machines, m.Gate.Machines)
|
||||
}
|
||||
for machine := range m.Rest {
|
||||
machines = union(machines, []string{machine})
|
||||
}
|
||||
if len(machines) == 0 {
|
||||
continue
|
||||
}
|
||||
sort.Strings(machines)
|
||||
out = append(out, inventory.PlanKept{Module: name, Machines: machines})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module })
|
||||
return out
|
||||
}
|
||||
|
||||
// keptWords is what a stopped walk leaves on the machines, as a person reads it.
|
||||
func keptWords(kept []inventory.PlanKept) string {
|
||||
var words []string
|
||||
for _, k := range kept {
|
||||
words = append(words, k.Module+" on "+strings.Join(k.Machines, ", "))
|
||||
}
|
||||
return "left running the stopped walk's build, which nothing walks again: " + strings.Join(words, "; ")
|
||||
}
|
||||
|
||||
// stillCarried are the merges outside the stopped record, in a batch or an open walk, that will deliver a stopped
|
||||
// change when they are walked: a later merge of its repository's branch, which contains it, or one that moves a
|
||||
// module it moves. Holding a merged change back for good is a revert's (ADR 0299).
|
||||
func stillCarried(ctx context.Context, inv *inventory.Inventory, id string, merges []inventory.BatchedMerge,
|
||||
out []inventory.PlanLeftOut, withheld []string) ([]string, error) {
|
||||
var named []inventory.BatchedMerge
|
||||
for _, m := range leftOutMerges(merges, out) {
|
||||
if slices.ContainsFunc(out, func(o inventory.PlanLeftOut) bool { return o.Commit == m.Commit && o.Contains == "" }) {
|
||||
named = append(named, m)
|
||||
}
|
||||
}
|
||||
if len(named) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
open, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
batches, err := inv.Batches(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var carried []string
|
||||
for _, p := range append(open, batches...) {
|
||||
if p.ID == id {
|
||||
continue
|
||||
}
|
||||
ms, err := inv.MergesOf(ctx, p.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, m := range ms {
|
||||
later := slices.ContainsFunc(named, func(s inventory.BatchedMerge) bool {
|
||||
return strings.EqualFold(s.Repository, m.Repository) && s.Branch == m.Branch && laterMerge(m, s)
|
||||
})
|
||||
shares := slices.ContainsFunc(announcedOf(m).Moves, func(n string) bool { return slices.Contains(withheld, n) })
|
||||
if later || shares {
|
||||
carried = append(carried, fmt.Sprintf("%s@%s (in %s)", m.Repository, m.Commit, p.ID))
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(carried)
|
||||
return carried, nil
|
||||
}
|
||||
|
||||
// mergesLeftOutWords is the merges a stop left out, as a person reads them: each that contains a stopped one, or
|
||||
// shares its modules, says so.
|
||||
func mergesLeftOutWords(out []inventory.PlanLeftOut) string {
|
||||
var words []string
|
||||
for _, o := range out {
|
||||
s := o.Repository + "@" + short(o.Commit)
|
||||
switch {
|
||||
case len(o.Shares) > 0:
|
||||
s += " (which moves " + strings.Join(o.Shares, ", ") + " as " + short(o.Contains) + " does)"
|
||||
case o.Contains != "":
|
||||
s += " (which contains " + short(o.Contains) + ")"
|
||||
}
|
||||
words = append(words, s)
|
||||
}
|
||||
return strings.Join(words, ", ")
|
||||
}
|
||||
|
||||
// orderMember is one member of a group, as mesh-delivery says it.
|
||||
@@ -466,6 +849,8 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
group := set.String("group", "", "a delivery group's id")
|
||||
by := set.String("by", catalogue.DeliverySeat, "who says it")
|
||||
why := set.String("why", "", "why")
|
||||
without := set.String("without", "", "merges a stop leaves out, <repository>@<commit>, comma-separated: the walk's "+
|
||||
"other merges are walked again without them (novox/hq ADR 0299)")
|
||||
limit := set.Int("n", 50, "how many ended walks to answer beside the open ones")
|
||||
planID := set.String("plan", "", "one walk")
|
||||
positionals, err := parseAround(set, rest)
|
||||
@@ -552,15 +937,21 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
case "stop":
|
||||
if len(positionals) != 1 || strings.TrimSpace(*why) == "" {
|
||||
return errors.New("delivery stop <plan> --why <text> [--by <who>]")
|
||||
return errors.New("delivery stop <plan> --why <text> [--by <who>] [--without <repository>@<commit>,…]")
|
||||
}
|
||||
var p inventory.Plan
|
||||
var said stopAnswer
|
||||
if err := sayingOnTheBus(ctx, func() (err error) {
|
||||
p, err = stopWalk(ctx, inv, positionals[0], *by, strings.TrimSpace(*why))
|
||||
p, said, err = stopWalkWithout(ctx, inv, positionals[0], *by, strings.TrimSpace(*why), splitList(*without),
|
||||
time.Now().UTC())
|
||||
return err
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if *without != "" {
|
||||
// What it left out and what it walks again, as data: the delivery's owner reads it (ADR 0299).
|
||||
return answer(said)
|
||||
}
|
||||
fmt.Printf("%s stopped by %s at tier %d of %d; what was asked still builds and registers, nothing further "+
|
||||
"is asked or sent\n", p.ID, *by, p.Tier, len(p.Tiers))
|
||||
return nil
|
||||
|
||||
@@ -74,8 +74,12 @@ type probe struct {
|
||||
// probeRegistry is the registry, in to-be 45's order. **The registry is the design's live form**: a
|
||||
// probe added to a design is a row added here.
|
||||
var probeRegistry = []probe{
|
||||
{ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation",
|
||||
From: "issues 236, 263, 275", Kind: "declaration-refused", Raises: []string{kindAwaitingPush}, Phase: 1,
|
||||
// D1 also says every module assigned and left out of a machine's declaration (novox/hq issue 380), from the
|
||||
// resolution it already makes: needs-operator for a setting nobody gave, left-out for any other cause.
|
||||
{ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation; no module " +
|
||||
"assigned to a machine is left out of its declaration unsaid",
|
||||
From: "issues 236, 263, 275, 380", Kind: "declaration-refused",
|
||||
Raises: []string{kindAwaitingPush, kindLeftOut, kindNeedsOperator}, Phase: 1,
|
||||
run: probeDeclarations},
|
||||
{ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6",
|
||||
From: "issue 262", Kind: "resolver-wrong", Phase: 1, run: probeResolvers},
|
||||
@@ -118,8 +122,8 @@ var probeRegistry = []probe{
|
||||
Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries},
|
||||
// The delivery budgets (novox/hq ADR 0282 decision 7): the newest delivery of each class within its budget,
|
||||
// read from mesh-delivery's `times`; a measurement said, never a delivery held.
|
||||
{ID: probeBudgetsID, Asserts: "the newest delivery of a leaf module ran on every machine within five minutes of " +
|
||||
"its merge, and of a core module within ten: mesh-delivery's `times`", From: "ADR 0282",
|
||||
{ID: probeBudgetsID, Asserts: "the newest delivery of a leaf module ran on every machine within three minutes of " +
|
||||
"its merge, and of a core module within ten: mesh-delivery's `times`", From: "ADR 0282, ADR 0304",
|
||||
Kind: kindOverBudget, Phase: 3, run: probeBudgets},
|
||||
// A client of the bus reconnecting in a loop (novox/hq issue 327), from the server's record of closed
|
||||
// connections, which the bus's own module reads.
|
||||
|
||||
@@ -27,8 +27,10 @@ import (
|
||||
// **"Reported applied" is not enough.** ADR 0218 sent a module to one machine first and the rest once
|
||||
// that machine reported it applied. A build that applies and then does nothing, crashes, serves no
|
||||
// tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by
|
||||
// the component's health — the core's health definitions, or a module's own — passing three times over
|
||||
// at least two minutes, within ten minutes of the apply. Only then are the rest sent.
|
||||
// the component's health — the core's health definitions, or a module's own — within ten minutes of the
|
||||
// apply: a core module passing three times over at least two minutes, a leaf module once, as soon as the
|
||||
// node-engine states it healthy (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it). Only then
|
||||
// are the rest sent.
|
||||
//
|
||||
// **A failing gate stops the plan and puts the previous build back there.** The build is marked failed
|
||||
// at its gate — registration refuses it and nothing sends it again on its own — the module's registered
|
||||
@@ -43,11 +45,38 @@ var (
|
||||
gateSettle = 2 * time.Minute
|
||||
// gateBound is how long after the apply the build may take to become healthy.
|
||||
gateBound = 10 * time.Minute
|
||||
// gatePasses is how many consecutive judgings must find it healthy, gateEvery apart at the least.
|
||||
gatePasses = 3
|
||||
gateEvery = 40 * time.Second
|
||||
// gatePasses is how many consecutive judgings must find a core module healthy, gateEvery apart at the
|
||||
// least, after gateSettle. gateLeafPasses is a leaf module's: one, taken once the node-engine states every
|
||||
// resource of it healthy by its own check, with no settle (novox/hq ADR 0282 decision 4 as ADR 0304
|
||||
// decision 3 changed it, issue 524).
|
||||
gatePasses = 3
|
||||
gateLeafPasses = 1
|
||||
gateLeafSettles = false
|
||||
gateEvery = 40 * time.Second
|
||||
)
|
||||
|
||||
// coreModule is whether a module is judged as the core (novox/hq ADR 0282 decision 1): one on the controller's
|
||||
// own path, or one holding the mesh's resolver. Every other module is a leaf.
|
||||
func coreModule(module string, m catalogue.Manifest) bool {
|
||||
if _, own := onTheControllersPath[module]; own {
|
||||
return true
|
||||
}
|
||||
for _, c := range m.Claims {
|
||||
if c.Name == resolverSeat {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return coreComponent(module) != ""
|
||||
}
|
||||
|
||||
// gateWants is how many passes a module needs, and whether it must stay healthy gateSettle first.
|
||||
func gateWants(module string, m catalogue.Manifest) (passes int, settles bool) {
|
||||
if coreModule(module, m) {
|
||||
return gatePasses, true
|
||||
}
|
||||
return gateLeafPasses, gateLeafSettles
|
||||
}
|
||||
|
||||
// gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions
|
||||
// are raised by a plan as it judges, and kept or cleared by the probe on every run.
|
||||
const gateProbe = "DG"
|
||||
@@ -608,8 +637,24 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
}
|
||||
}
|
||||
settled := now.Sub(*g.Since) >= gateSettle
|
||||
// What the send needs is what its most demanding module needs: a send carrying a core module is judged
|
||||
// as the core.
|
||||
wants, settles := 0, false
|
||||
for _, m := range modules {
|
||||
p, s := gateWants(m, shelf[m])
|
||||
wants, settles = max(wants, p), settles || s
|
||||
}
|
||||
if len(modules) == 0 {
|
||||
// A send judged on no machine of its modules (its first machine not running them) has no class to
|
||||
// be judged by: it is judged as the core, never passed at once without a reading.
|
||||
wants, settles = gatePasses, true
|
||||
}
|
||||
g.Wants = wants
|
||||
// A module's own pass is its own class's: a leaf beside a core module that fails keeps its build after
|
||||
// its one reading, and only what did not pass is put back (issue 318's rule, by class since issue 524).
|
||||
passedAlone := func(m string) bool {
|
||||
return settled && reading[m] <= healthPerson && g.Healthy[m] >= gatePasses
|
||||
p, s := gateWants(m, shelf[m])
|
||||
return (settled || !s) && reading[m] <= healthPerson && g.Healthy[m] >= p
|
||||
}
|
||||
// fail decides the send failed: what passed on its own keeps its pass, everything else is put back.
|
||||
fail := func(why string) {
|
||||
@@ -657,7 +702,7 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
g.Passes++
|
||||
g.LastPass, g.Last, g.Failing = &now, "", nil
|
||||
g.Read(now, true, "")
|
||||
if g.Passes >= gatePasses && settled {
|
||||
if g.Passes >= wants && (settled || !settles) {
|
||||
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
|
||||
now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now)
|
||||
}
|
||||
@@ -1340,9 +1385,9 @@ func gateLine(g *inventory.PlanGate) string {
|
||||
if g.Why != "" {
|
||||
line += ": " + g.Why
|
||||
} else if g.Last != "" {
|
||||
line += fmt.Sprintf(" (%d of %d passes; wanting: %s)", g.Passes, gatePasses, g.Last)
|
||||
line += fmt.Sprintf(" (%d of %d passes; wanting: %s)", g.Passes, gateWanted(g), g.Last)
|
||||
} else if g.Verdict == "" {
|
||||
line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gatePasses)
|
||||
line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gateWanted(g))
|
||||
}
|
||||
if g.Verdict == "" && len(g.Waits) > 0 {
|
||||
line += waitsSaid(g.Waits)
|
||||
@@ -1353,6 +1398,10 @@ func gateLine(g *inventory.PlanGate) string {
|
||||
if g.Rollback != "" {
|
||||
line += "; " + g.Rollback
|
||||
}
|
||||
// Before the send it judges (novox/hq issue 490).
|
||||
if said := grantsSaid(g.Grants); said != "" {
|
||||
line += "; " + said
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
@@ -1381,3 +1430,11 @@ func witnessObservation(node string, r lease.Rollback) conditions.Observation {
|
||||
Summary: fmt.Sprintf("the witness on %s judged the %s %s not healthy %s at %s: %s", node, r.Component,
|
||||
short(r.From), what, r.At.UTC().Format(time.RFC3339), r.Why)}
|
||||
}
|
||||
|
||||
// gateWanted is the passes a gate's send needs, three where the gate was judged before it was said.
|
||||
func gateWanted(g *inventory.PlanGate) int {
|
||||
if g.Wants > 0 {
|
||||
return g.Wants
|
||||
}
|
||||
return gatePasses
|
||||
}
|
||||
|
||||
@@ -53,6 +53,7 @@ func healthyContainer(module string) inventory.ResourceHealth {
|
||||
// tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass,
|
||||
// counted over the gate's spacing — before it says its own. The broken one alone is put back.
|
||||
func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
withGateBounds(t, 150*time.Millisecond, 20*time.Millisecond, 10*time.Second)
|
||||
@@ -284,6 +285,7 @@ func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) {
|
||||
// tools are marked and put back in the very judging that found them broken — their registered build is the
|
||||
// one before, and the laptop is sent it — while `app` goes on being judged, recovers, and keeps its own pass.
|
||||
func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
|
||||
@@ -153,6 +153,7 @@ func (g *gateMesh) plan(t *testing.T) inventory.Plan {
|
||||
// registered and sent to it again — and never reaches the second machine; it is marked, said as a
|
||||
// condition and an event, never registered or rolled back again.
|
||||
func TestABuildThatFailsItsGateIsRolledBackOnItsFirstMachineAndGoesNoFurther(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := g.open.inventory
|
||||
@@ -255,7 +256,7 @@ func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
|
||||
t.Fatalf("sent %v", g.sent)
|
||||
}
|
||||
gate := p.Modules["app"].Gate
|
||||
if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gatePasses {
|
||||
if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gateWanted(gate) {
|
||||
t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate)
|
||||
}
|
||||
if v, found, err := g.open.inventory.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
@@ -266,8 +267,52 @@ func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A leaf module passes its gate on one reading, as soon as its first machine is healthy, without waiting out the
|
||||
// core's settle time (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it, issue 524).
|
||||
func TestALeafModulePassesOnOneHealthyReading(t *testing.T) {
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
gateSettle = time.Hour // a core module could not pass for an hour
|
||||
advancePlans(ctx, g.open) // the first machine is sent the new build
|
||||
advancePlans(ctx, g.open) // one healthy judging
|
||||
p := g.plan(t)
|
||||
gate := p.Modules["app"].Gate
|
||||
if gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes != 1 || gate.Wants != 1 {
|
||||
t.Fatalf("a leaf module's gate after one healthy judging: %+v (plan %s: %s)", gate, p.State, p.Note)
|
||||
}
|
||||
if !strings.Contains(gate.Why, "healthy 1 times") {
|
||||
t.Errorf("the verdict says %q", gate.Why)
|
||||
}
|
||||
}
|
||||
|
||||
// Which modules are judged as the core: those on the controller's own path and the resolver's holder, three
|
||||
// readings after the settle time; every other module one (novox/hq ADR 0282 decision 1, ADR 0304 decision 3).
|
||||
func TestAModuleIsJudgedByItsClass(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
module string
|
||||
claims []catalogue.Claim
|
||||
passes int
|
||||
settles bool
|
||||
}{
|
||||
{"app", nil, 1, false},
|
||||
{"nzbget", nil, 1, false},
|
||||
{"mesh-controller", nil, 3, true},
|
||||
{"mesh-host", nil, 3, true},
|
||||
{"node-tools", nil, 3, true},
|
||||
{"nats", nil, 3, true},
|
||||
{catalogue.DeliverySeat, nil, 3, true},
|
||||
{"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, true},
|
||||
} {
|
||||
p, s := gateWants(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims})
|
||||
if p != c.passes || s != c.settles {
|
||||
t.Errorf("%s: %d passes, settles %v; want %d, %v", c.module, p, s, c.passes, c.settles)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A first machine that never becomes healthy fails its gate at the bound, and is put back.
|
||||
func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
advancePlans(ctx, g.open)
|
||||
@@ -593,3 +638,13 @@ func TestAWitnessesVerdictIsAConditionWhileItsReportsSayIt(t *testing.T) {
|
||||
t.Fatalf("a verdict the reports no longer carry is still said: %+v %v", obs, err)
|
||||
}
|
||||
}
|
||||
|
||||
// judgeLeavesAsTheCore makes a test's leaf modules judged as a core module is, three readings after the settle
|
||||
// time, for a test of the gate's other rules (a fault, the bound, a supersession) that needs a judging still
|
||||
// open after a healthy reading.
|
||||
func judgeLeavesAsTheCore(t *testing.T) {
|
||||
t.Helper()
|
||||
wasPasses, wasSettles := gateLeafPasses, gateLeafSettles
|
||||
gateLeafPasses, gateLeafSettles = gatePasses, true
|
||||
t.Cleanup(func() { gateLeafPasses, gateLeafSettles = wasPasses, wasSettles })
|
||||
}
|
||||
|
||||
@@ -202,21 +202,99 @@ func modulesWords(modules []string) string {
|
||||
return strings.Join(modules, ", ")
|
||||
}
|
||||
|
||||
// writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234):
|
||||
// nothing when the mesh has not recorded a send to it.
|
||||
// writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234), and
|
||||
// what it answered (issue 485): nothing when the mesh has not recorded a send to it.
|
||||
func writeLastSend(ctx context.Context, w io.Writer, inv *inventory.Inventory, node string) error {
|
||||
s, found, err := inv.LastSend(ctx, node)
|
||||
if err != nil || !found {
|
||||
return err
|
||||
}
|
||||
generation := "no generation recorded"
|
||||
if s.Generation > 0 {
|
||||
generation = fmt.Sprintf("assignment generation %d", s.Generation)
|
||||
}
|
||||
fmt.Fprintf(w, "%s was last sent sequence %d at %s by %s, composed from %s, naming %s\n", node, s.Sequence, s.SentAt.Local().Format("2006-01-02 15:04:05"), s.Sender, generation, modulesWords(s.Modules))
|
||||
if s.RefusedAt != nil {
|
||||
fmt.Fprintf(w, " and refused it at %s: it had applied generation %d\n",
|
||||
s.RefusedAt.Local().Format("2006-01-02 15:04:05"), s.RefusedApplied)
|
||||
fmt.Fprintf(w, "%s:\n", node)
|
||||
for _, line := range lastSendLines(s) {
|
||||
fmt.Fprintln(w, line)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// lastSendLines is a machine's last send in full, as `node show` and `plan` say it (novox/hq issue 485): when,
|
||||
// by whom, under which lease epoch, from which assignment generation, naming which modules, and what the
|
||||
// machine answered of it — the refusal in its own words.
|
||||
func lastSendLines(s inventory.Send) []string {
|
||||
const stamp = "2006-01-02 15:04:05"
|
||||
epoch := "under no lease epoch"
|
||||
if s.Epoch > 0 {
|
||||
epoch = fmt.Sprintf("under lease epoch %d", s.Epoch)
|
||||
}
|
||||
generation := "no assignment generation recorded"
|
||||
if s.Generation > 0 {
|
||||
generation = fmt.Sprintf("assignment generation %d", s.Generation)
|
||||
}
|
||||
lines := []string{
|
||||
fmt.Sprintf(" last sent sequence %d at %s", s.Sequence, s.SentAt.Local().Format(stamp)),
|
||||
fmt.Sprintf(" by %s", s.Sender),
|
||||
fmt.Sprintf(" %s, composed from %s", epoch, generation),
|
||||
fmt.Sprintf(" naming %s", modulesWords(s.Modules)),
|
||||
}
|
||||
answer := func(words string) { lines = append(lines, " its answer "+words) }
|
||||
switch a := s.Answer; {
|
||||
case s.RefusedAt != nil:
|
||||
// Before the report: a refusal for the generation is kept on the send itself and is the machine's
|
||||
// answer to this send in particular.
|
||||
answer(fmt.Sprintf("refused it at %s: it had applied assignment generation %d, newer than this one",
|
||||
s.RefusedAt.Local().Format(stamp), s.RefusedApplied))
|
||||
case a.Outcome == "" || a.At == nil:
|
||||
answer("the machine has not reported on it yet")
|
||||
case a.Outcome == inventory.OutcomeApplied:
|
||||
answer("applied it at " + a.At.Local().Format(stamp))
|
||||
case a.Outcome == inventory.OutcomeRefused:
|
||||
answer("refused it at " + a.At.Local().Format(stamp) + ", in its words:")
|
||||
for _, line := range strings.Split(strings.TrimRight(a.Refused, "\n"), "\n") {
|
||||
lines = append(lines, " "+strings.TrimSpace(line))
|
||||
}
|
||||
default:
|
||||
answer(fmt.Sprintf("%s at %s: %d resource(s) failed — `status` says which", a.Outcome,
|
||||
a.At.Local().Format(stamp), a.Failed))
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
// sendAnswerWord is what a machine answered of a send, in a word or few, as status says it in one line.
|
||||
func sendAnswerWord(s inventory.Send) string {
|
||||
switch a := s.Answer; {
|
||||
case s.RefusedAt != nil:
|
||||
return fmt.Sprintf("refused: it had applied generation %d", s.RefusedApplied)
|
||||
case a.Outcome == "" || a.At == nil:
|
||||
return "not reported on yet"
|
||||
case a.Outcome == inventory.OutcomeFailed:
|
||||
return fmt.Sprintf("failed (%d resource(s))", a.Failed)
|
||||
default:
|
||||
return a.Outcome
|
||||
}
|
||||
}
|
||||
|
||||
// printLastSends says, one line per machine, when it was last sent a declaration, by whom, from which assignment
|
||||
// generation and what it answered (novox/hq issue 485). `node show <node>` says the send in full.
|
||||
func printLastSends(nodes []inventory.Node, sends map[string]inventory.Send, unread string) {
|
||||
if unread != "" {
|
||||
fmt.Printf("what each machine was last sent could NOT be read: %s\n\n", unread)
|
||||
return
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
return
|
||||
}
|
||||
fmt.Println("last sent, per machine:")
|
||||
for _, n := range nodes {
|
||||
s, found := sends[n.Name]
|
||||
if !found {
|
||||
fmt.Printf(" %-12s no send recorded\n", n.Name)
|
||||
continue
|
||||
}
|
||||
generation := "no generation"
|
||||
if s.Generation > 0 {
|
||||
generation = fmt.Sprintf("generation %d", s.Generation)
|
||||
}
|
||||
fmt.Printf(" %-12s %s by %s, %s — %s\n", n.Name, s.SentAt.Local().Format("2006-01-02 15:04"), s.Sender,
|
||||
generation, sendAnswerWord(s))
|
||||
}
|
||||
fmt.Printf("\n `node show <node>` says a machine's last send in full\n\n")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/identity"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The grants step on a store (novox/hq issue 490): the machine holding the bus is sent its new user list,
|
||||
// and nothing else — not the new build of a module it runs, not a module newly assigned there.
|
||||
//
|
||||
// The restic shape: anchor holds the bus and runs restic at c1, as does laptop; restic's c2 gives it a
|
||||
// health tool. The grants step's send to anchor composes restic at c1, carries the user list that grants
|
||||
// laptop's node-engine c2's tool, and records that anchor still runs c1. A module newly assigned to anchor
|
||||
// makes the step fail, said, rather than install it unjudged.
|
||||
func TestTheGrantsStepSendsTheUserListAndNothingElse(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
start := time.Now().Add(-time.Hour)
|
||||
build := func(module, commit string, at time.Time, manifest map[string]any) link.BuildResult {
|
||||
manifest["module"], manifest["version"] = module, "1"
|
||||
raw, _ := json.Marshal(manifest)
|
||||
return link.BuildResult{ID: link.NewBuildID(at), Repository: "novox/mesh-catalog", Path: "modules/" + module,
|
||||
On: "anchor", Module: module, Commit: commit, Manifest: raw,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
||||
}
|
||||
for _, b := range []link.BuildResult{
|
||||
build("nats", "n1", start, natsFixture()),
|
||||
build("restic", "c1", start.Add(time.Second), resticFixture(false)),
|
||||
build("wallpaper", "w1", start.Add(2*time.Second), wallpaperFixture()),
|
||||
} {
|
||||
if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, a := range [][2]string{{"anchor", "nats"}, {"anchor", "restic"}, {"laptop", "restic"}} {
|
||||
if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: "node." + n, Kind: inventory.BusNode, Node: n}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
wasEpoch := epochForActs
|
||||
epochForActs = func(context.Context) (uint64, error) { return 7, nil }
|
||||
t.Cleanup(func() { epochForActs = wasEpoch })
|
||||
|
||||
// A send to anchor composed as sendToEach composes it: numbered, planned, declared, stamped.
|
||||
compose := func(under context.Context) (catalogue.Resolution, sendable) {
|
||||
t.Helper()
|
||||
numbered, err := allot(under, inv, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, settings, err := planFor(under, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(under, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationWith(under, open, "anchor", plan, settings, gens, Allocating)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
numbered.stamp(&declared)
|
||||
return plan, declared
|
||||
}
|
||||
record := func(declared sendable) {
|
||||
t.Helper()
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := recordSent(ctx, inv, "anchor", body, declared.Builds, declared.Epoch,
|
||||
sentRecordOf(ctx, declared)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// Kept for anchor alone. laptop is never recorded as sent, on purpose: composing anchor resolves laptop
|
||||
// too (who is on the private network), and a step that kept every machine's builds refused anchor's
|
||||
// composition for want of laptop's last send (repo-check of #230 at 0f853e93).
|
||||
kept := keepingEveryBuild(keepingRecorded(ctx), "anchor")
|
||||
|
||||
// What anchor was last sent: everything at the builds of before the merge, as an ordinary send sends it.
|
||||
_, before := compose(keepingRecorded(ctx))
|
||||
record(before)
|
||||
plan, declared := compose(kept)
|
||||
if only, err := grantsOnlyIn(ctx, open, "anchor", plan, declared); err != nil || only != "" {
|
||||
t.Fatalf("with nothing changed, the step reads %q, %v", only, err)
|
||||
}
|
||||
|
||||
// The merge: restic's c2 gives it a health tool.
|
||||
if _, _, err := takeIn(ctx, inv, build("restic", "c2", start.Add(time.Minute),
|
||||
resticFixture(true))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The step's send to anchor: restic as anchor runs it, c1, and the user list granting c2's tool.
|
||||
generation, err := inv.AssignmentGeneration(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, declared = compose(kept)
|
||||
for _, m := range plan.Modules {
|
||||
if m.Module == "restic" && len(m.Tools) > 0 {
|
||||
t.Fatalf("the grants step composed restic's new build on anchor: tools %v", m.Tools)
|
||||
}
|
||||
}
|
||||
if declared.Builds["restic"] != "c1" {
|
||||
t.Fatalf("the step's send records it carries restic %q; want c1, which anchor runs", declared.Builds["restic"])
|
||||
}
|
||||
if !strings.Contains(declared.BusUsers, "mesh.mod.restic.tool.backup_health.laptop") {
|
||||
t.Errorf("the step's declaration does not grant laptop's node-engine restic's new tool:\n%s", declared.BusUsers)
|
||||
}
|
||||
// The generation it carries is the current one (novox/hq issue 234), and the step does not raise it.
|
||||
if declared.composedFrom != generation {
|
||||
t.Errorf("the step's send was composed from generation %d; the mesh is at %d", declared.composedFrom, generation)
|
||||
}
|
||||
// The guard, on the very declaration the send sends: only the user list differs from the last send.
|
||||
if only, err := grantsOnlyIn(ctx, open, "anchor", plan, declared); err != nil || only != "" {
|
||||
t.Fatalf("the step reads as changing more than the user list: %q, %v", only, err)
|
||||
}
|
||||
// And the gate's refusal still reads the step's send: it moves nothing there.
|
||||
if names, err := ungatedIn(kept, open, []string{"anchor"}, ""); err != nil || strings.Join(names, ",") != "anchor" {
|
||||
t.Fatalf("the step's send is refused as a move: %v, %v", names, err)
|
||||
}
|
||||
// Recorded as the send records it: anchor still runs restic c1, its gate still to come there.
|
||||
record(declared)
|
||||
if sent, _, err := inv.SentBuilds(ctx, "anchor"); err != nil || sent["restic"] != "c1" {
|
||||
t.Fatalf("after the step anchor reads as sent restic %q (%v); want c1", sent["restic"], err)
|
||||
}
|
||||
if after, err := inv.AssignmentGeneration(ctx); err != nil || after != generation {
|
||||
t.Fatalf("the step moved the assignment generation from %d to %d (%v)", generation, after, err)
|
||||
}
|
||||
|
||||
// A module newly assigned to anchor: the step would install it, unjudged, so it is refused unsent.
|
||||
if _, err := inv.Assign(ctx, "anchor", "wallpaper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, declared = compose(kept)
|
||||
only, err := grantsOnlyIn(ctx, open, "anchor", plan, declared)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(only, "wallpaper newly assigned") || !strings.Contains(only, "+wallpaper") {
|
||||
t.Fatalf("a new assignment on the bus's machine reads %q; want the step refused, naming it", only)
|
||||
}
|
||||
// And the send itself refuses it, unsent: judged where it is composed, before the bus is dialled.
|
||||
identity.ForTest(t)
|
||||
if _, err := sendToEach(kept, open, []string{"anchor"}); !errors.Is(err, errNotGrantsOnly) ||
|
||||
!strings.Contains(err.Error(), "wallpaper") {
|
||||
t.Fatalf("the grants step's send of a new assignment was not refused by the send: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// fixtureImage is the image every fixture container of the grants step's tests runs.
|
||||
var fixtureImage = "registry.invalid:5000/restic/backup@sha256:" + strings.Repeat("b", 64)
|
||||
|
||||
// resticFixture is restic's manifest in the shape of mesh-catalog #210: at c2 (withTool) its backup judged by
|
||||
// its new tool `backup_health`, beside a check of another kind it does not run itself — a tool check alone
|
||||
// is refused (ADR 0227 rule 8, ADR 0240 rule 2) — and at c1 neither.
|
||||
func resticFixture(withTool bool) map[string]any {
|
||||
backup := map[string]any{"id": "backup", "type": "container", "name": "restic-backup", "image": fixtureImage}
|
||||
measure := map[string]any{"id": "measure", "type": "container", "name": "restic-measure", "image": fixtureImage}
|
||||
m := map[string]any{"resources": []any{backup, measure}}
|
||||
if withTool {
|
||||
backup["health"] = map[string]any{"kind": catalogue.HealthTool, "tool": "backup_health"}
|
||||
measure["health"] = map[string]any{"kind": "runtime"}
|
||||
m["tools"] = []string{"backup_health"}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// natsFixture is the bus's module: it holds mesh-broker and is sent the user list.
|
||||
func natsFixture() map[string]any {
|
||||
return map[string]any{"bus-users": "/var/lib/nats-module/conf/accounts.conf",
|
||||
"claims": []any{map[string]any{"name": catalogue.BrokerSeat, "scope": catalogue.ScopeMesh}}}
|
||||
}
|
||||
|
||||
// wallpaperFixture is a module the bus's machine is newly assigned.
|
||||
func wallpaperFixture() map[string]any {
|
||||
return map[string]any{"resources": []any{
|
||||
map[string]any{"id": "paper", "type": "container", "name": "wallpaper", "image": fixtureImage}}}
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The grants reach the bus before the gate (novox/hq issue 490).
|
||||
//
|
||||
// On 2026-10-11 a merge gave restic, assigned to every machine, a new health tool. Its walk sent restic to
|
||||
// its first machine and judged it there; that machine's node-engine asked the new tool and the bus refused
|
||||
// it, because the grant was in the bus's user list, which only the machine holding the bus carries — and
|
||||
// that machine runs restic too, so it was left out of the send while restic's new build waited there for
|
||||
// the very gate that could not pass. A person pushed it by hand, carrying restic's new build there unjudged.
|
||||
|
||||
// aSend is one send the walk made: to which machines, and whether every build there was kept.
|
||||
type aSend struct {
|
||||
names []string
|
||||
keepsAll bool
|
||||
judged []string
|
||||
}
|
||||
|
||||
// sendsRecorded replaces the walk's send and the reading of the bus's user list for one test: the machine
|
||||
// holding the bus is novox, and its list is behind as behind says.
|
||||
func sendsRecorded(t *testing.T, holder string, behind bool, refuse error) *[]aSend {
|
||||
t.Helper()
|
||||
var sends []aSend
|
||||
wasSend, wasBehind := sendRollout, brokerBehindOf
|
||||
t.Cleanup(func() { sendRollout, brokerBehindOf = wasSend, wasBehind })
|
||||
brokerBehindOf = func(_ context.Context, _ *stores, names []string) (string, bool, error) {
|
||||
if slices.Contains(names, holder) {
|
||||
return holder, false, nil
|
||||
}
|
||||
return holder, behind, nil
|
||||
}
|
||||
sendRollout = func(ctx context.Context, _ *stores, names []string) ([]string, error) {
|
||||
s := aSend{names: append([]string(nil), names...), keepsAll: len(names) == 1 && everyBuildKept(ctx, names[0])}
|
||||
for n := range scopeOf(ctx).judged {
|
||||
s.judged = append(s.judged, n)
|
||||
}
|
||||
sends = append(sends, s)
|
||||
if refuse != nil && s.keepsAll {
|
||||
return nil, refuse
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
return &sends
|
||||
}
|
||||
|
||||
// The restic shape: a new tool on a module on every machine, its first machine ace, the bus on novox.
|
||||
func TestAWalkSendsTheGrantsToTheBusBeforeTheFirstMachineIsJudged(t *testing.T) {
|
||||
sends := sendsRecorded(t, "novox", true, nil)
|
||||
sent, grants, err := sendJudged(t.Context(), nil, "ace")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(*sends) != 2 {
|
||||
t.Fatalf("the walk made %d send(s), want the grants step and then the first machine: %+v", len(*sends), *sends)
|
||||
}
|
||||
first, then := (*sends)[0], (*sends)[1]
|
||||
if strings.Join(first.names, ",") != "novox" || !first.keepsAll || len(first.judged) != 0 {
|
||||
t.Errorf("the first send is %+v, want novox alone, every build there kept, judging nothing", first)
|
||||
}
|
||||
if strings.Join(then.names, ",") != "ace" || then.keepsAll || strings.Join(then.judged, ",") != "ace" {
|
||||
t.Errorf("the second send is %+v, want ace, judged", then)
|
||||
}
|
||||
if strings.Join(sent, ",") != "ace" {
|
||||
t.Errorf("the gate's machines are %v, want ace alone: the bus's machine is not judged", sent)
|
||||
}
|
||||
if grants == nil || grants.Node != "novox" || grants.At == nil || grants.Failed != "" {
|
||||
t.Fatalf("the gate keeps %+v as its grants step", grants)
|
||||
}
|
||||
line := gateLine(&inventory.PlanGate{Machines: sent, Grants: grants})
|
||||
if !strings.Contains(line, "grants step: novox sent the bus's user list first, its builds kept") {
|
||||
t.Errorf("plans says the gate as %q", line)
|
||||
}
|
||||
}
|
||||
|
||||
// Nothing more when there is nothing to carry: the list unchanged, or the first machine holds the bus.
|
||||
func TestAWalkTakesNoGrantsStepWhenTheListIsAlreadyThere(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
node string
|
||||
behind bool
|
||||
}{{"the list unchanged", "ace", false}, {"the first machine holds the bus", "novox", true}} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
sends := sendsRecorded(t, "novox", c.behind, nil)
|
||||
_, grants, err := sendJudged(t.Context(), nil, c.node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(*sends) != 1 || (*sends)[0].keepsAll || grants != nil {
|
||||
t.Errorf("the walk made %+v with grants %+v, want the judged send alone", *sends, grants)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A grants step that cannot be sent is said on the gate, and the walk goes on as it did before.
|
||||
func TestAGrantsStepThatFailsIsSaidAndTheSendGoesOn(t *testing.T) {
|
||||
sends := sendsRecorded(t, "novox", true, errors.New("a bus upgrade waits for a person"))
|
||||
sent, grants, err := sendJudged(t.Context(), nil, "ace")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(sent, ",") != "ace" || len(*sends) != 2 {
|
||||
t.Errorf("the walk made %+v", *sends)
|
||||
}
|
||||
if grants == nil || grants.Failed == "" || grants.At != nil {
|
||||
t.Fatalf("the gate keeps %+v", grants)
|
||||
}
|
||||
if line := gateLine(&inventory.PlanGate{Machines: sent, Grants: grants}); !strings.Contains(line,
|
||||
"grants step to novox FAILED, sent without it: a bus upgrade waits for a person") {
|
||||
t.Errorf("plans says the gate as %q", line)
|
||||
}
|
||||
}
|
||||
|
||||
// A step whose send would change more than the user list is refused unsent by the send (sendToEach): the
|
||||
// gate says it FAILED with what differed, and the walk goes on.
|
||||
func TestAGrantsStepThatWouldCarryMoreIsNotSent(t *testing.T) {
|
||||
refusal := fmt.Errorf("%w: its send would change more than the bus's user list: -postgres.login-n8n", errNotGrantsOnly)
|
||||
sends := sendsRecorded(t, "novox", true, refusal)
|
||||
sent, grants, err := sendJudged(t.Context(), nil, "ace")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(sent, ",") != "ace" || len(*sends) != 2 {
|
||||
t.Errorf("the walk made %+v, want the refused step and then ace's judged send", *sends)
|
||||
}
|
||||
if grants == nil || grants.At != nil || !strings.Contains(grants.Failed, "-postgres.login-n8n") {
|
||||
t.Fatalf("the gate keeps %+v", grants)
|
||||
}
|
||||
}
|
||||
|
||||
// The step's exemption is the list's content alone: its path or mode moving is a change like any other.
|
||||
func TestOnlyTheUserListsContentIsExempt(t *testing.T) {
|
||||
list := "nats.bus-users"
|
||||
file := func(fields map[string]any) sentResource {
|
||||
r := map[string]any{"id": list, "type": "file", "path": "/conf/accounts.conf", "mode": "0600", "content": "a"}
|
||||
for k, v := range fields {
|
||||
r[k] = v
|
||||
}
|
||||
body, _ := json.Marshal(map[string]any{"resources": []any{r}})
|
||||
s, err := summarize(body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return s[0]
|
||||
}
|
||||
was := file(nil)
|
||||
if other := otherThanTheList(diffSent([]sentResource{was}, []sentResource{file(map[string]any{"content": "b"})}), list); len(other) != 0 {
|
||||
t.Errorf("the list's new content reads as more: %v", other)
|
||||
}
|
||||
for _, f := range []map[string]any{{"mode": "0644"}, {"path": "/elsewhere"}, {"content": "b", "owner": "nats"}} {
|
||||
other := otherThanTheList(diffSent([]sentResource{was}, []sentResource{file(f)}), list)
|
||||
if len(other) != 1 || !strings.HasPrefix(other[0], "~"+list) {
|
||||
t.Errorf("the list's resource changed by %v reads %v; want it refused", f, other)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The step keeps the builds of the machine it sends alone: the others its composition resolves are composed
|
||||
// as any send composes them.
|
||||
func TestTheGrantsStepKeepsOneMachinesBuilds(t *testing.T) {
|
||||
ctx := keepingEveryBuild(t.Context(), "novox")
|
||||
if !everyBuildKept(ctx, "novox") || everyBuildKept(ctx, "ace") || everyBuildKept(t.Context(), "novox") {
|
||||
t.Error("the grants step keeps the builds of a machine it does not send")
|
||||
}
|
||||
}
|
||||
|
||||
// The store test's manifests are manifests the controller takes in: checked here, where no store is needed,
|
||||
// so a fixture the module rules refuse fails before the store test is run (repo-check of #230 at 325575b2).
|
||||
func TestTheGrantsStepFixturesAreManifests(t *testing.T) {
|
||||
for name, m := range map[string]map[string]any{"nats": natsFixture(), "restic-c1": resticFixture(false),
|
||||
"restic-c2": resticFixture(true), "wallpaper": wallpaperFixture()} {
|
||||
m["module"], m["version"] = strings.SplitN(name, "-", 2)[0], "1"
|
||||
raw, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := catalogue.ParseManifest(raw); err != nil {
|
||||
t.Errorf("%s: %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The delivery plan says the step before the merge.
|
||||
func TestAChangePlanSaysTheGrantsStep(t *testing.T) {
|
||||
const catalogue_ = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||
entry := func(name string, on ...string) inventory.Entry {
|
||||
e := fromRepo(name, catalogue_, "modules/"+name)
|
||||
e.Source.BuiltFrom = "old"
|
||||
e.On = on
|
||||
return e
|
||||
}
|
||||
nats := entry("nats", "novox")
|
||||
nats.Manifest.BusUsers = "/etc/nats/users.conf"
|
||||
nats.Manifest.Claims = []catalogue.Claim{{Name: catalogue.BrokerSeat, Scope: catalogue.ScopeMesh}}
|
||||
restic := entry("restic", "ace", "novox", "shanks")
|
||||
only := entry("bus-tools", "novox")
|
||||
entries := []inventory.Entry{nats, restic, only}
|
||||
rollsOut := func(string) (inventory.Upgrade, bool) { return inventory.Upgrade{RollOut: true}, true }
|
||||
plan := func(paths ...string) link.ChangePlan {
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: paths}
|
||||
return changePlanOf("novox/mesh-catalog", "main", "head", reachOfMerge(m, entries, nil, nil), entries, rollsOut)
|
||||
}
|
||||
|
||||
text := planText(plan("modules/restic/module.json"))
|
||||
if !strings.Contains(text, "grants step: if this changes what the bus's user list says (a new tool, subject or "+
|
||||
"user), novox is sent that list alone, every build there kept, before restic is judged on its first machine") {
|
||||
t.Errorf("the change plan does not say the grants step:\n%s", text)
|
||||
}
|
||||
// A module only on the bus's machine goes there with the list in its own send: no step of its own.
|
||||
if text := planText(plan("modules/bus-tools/module.json")); strings.Contains(text, "grants step") {
|
||||
t.Errorf("a module on the bus's machine alone reads:\n%s", text)
|
||||
}
|
||||
}
|
||||
@@ -84,6 +84,9 @@ var handActVerbs = []handActVerb{
|
||||
"deliberate test, never a repair", DecidedFor: []string{causeDrill}},
|
||||
// A drill: something broken on purpose to see the mesh raise and clear it. A person's test, never a
|
||||
// repair, however often it is run.
|
||||
// A credential a provider's software issued, taken from the provider and kept for its consumers (novox/hq ADR
|
||||
// 0305): the mesh working as decided, however often the software changes it.
|
||||
{Verb: handActIssued, Decision: "a credential the provider's software issued, kept for its consumers (ADR 0305)"},
|
||||
{Verb: "hand-act drill", Decision: "a drill is a person's deliberate test of the mesh, never a repair"},
|
||||
// A person's decisions by design.
|
||||
{Verb: "retire approve", Decision: "nothing is retired past its bound without a person (ADR 0230)"},
|
||||
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
// another send reaches anchor, unreported; the gate still passes. And a send that moves a judged module
|
||||
// to another build supersedes the judging: no verdict, nothing put back.
|
||||
func TestReplay352(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn)
|
||||
t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging)
|
||||
}
|
||||
@@ -66,10 +67,11 @@ func testANewerSendDoesNotUnreportTheGatesOwn(t *testing.T) {
|
||||
|
||||
// A plan's own first send waits while a release judges the same module on that machine with another build.
|
||||
func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
advancePlans(ctx, b.open) // the release judges app c2 on anchor
|
||||
_, _, err := gatedSend(ctx, b.open, "anchor", []inventory.CarriedMove{{Module: "app", Node: "anchor", From: "c2", To: "c3", Build: "build-app-c3"}})
|
||||
_, _, _, err := gatedSend(ctx, b.open, "anchor", []inventory.CarriedMove{{Module: "app", Node: "anchor", From: "c2", To: "c3", Build: "build-app-c3"}})
|
||||
if !errors.Is(err, errWalkedElsewhere) || !strings.Contains(err.Error(), "release-") {
|
||||
t.Fatalf("a newer build of a judged module was sent under the release's gate: %v", err)
|
||||
}
|
||||
@@ -80,6 +82,7 @@ func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
|
||||
|
||||
// A merge plan's judging is superseded the same way: another send moved its module on the first machine.
|
||||
func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := g.open.inventory
|
||||
|
||||
@@ -0,0 +1,427 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq issue 459, ADR 0299: since the merge window one walk carries every merge of its batch, so a stop of one
|
||||
// group member through `delivery-stop` ended the walk of every other delivery in it. The stop now names the merges
|
||||
// to leave out: the walk is ended, and every other merge it answered is walked again in a fresh batch without them
|
||||
// — the stopped merge, and each later merge of its repository's branch, which contains it and ends with it.
|
||||
|
||||
// endsItsLine ends, as the test ends, a line the controller's output left open (a walk's ask prints " tier N: " and
|
||||
// the asked build the rest, which the tests' fake ask never prints): `go test -v` would put the test's "--- PASS"
|
||||
// after it, and the store-tests step of merge-check.sh, which reads that marker at a line's start, would say the
|
||||
// test never ran.
|
||||
func endsItsLine(t *testing.T) {
|
||||
t.Cleanup(func() { fmt.Println() })
|
||||
}
|
||||
|
||||
// leaved is a kept merge of a repository, made at a moment.
|
||||
func leaved(repository, commit string, made time.Time) inventory.BatchedMerge {
|
||||
return inventory.BatchedMerge{Repository: repository, Branch: "main", Commit: commit, Merged: made, Heard: made}
|
||||
}
|
||||
|
||||
// The stopped merge is left out, and so is each later merge of its repository's branch, naming the merge it
|
||||
// contains; an earlier merge of that repository and every other repository's merge are walked again.
|
||||
func TestLeftOutAreTheStoppedMergeAndTheMergesThatContainIt(t *testing.T) {
|
||||
c1 := leaved("novox/mesh-catalog", "c1aaaaaaaaaa", t0)
|
||||
c2 := leaved("novox/mesh-catalog", "c2bbbbbbbbbb", t0.Add(10*time.Second))
|
||||
c3 := leaved("novox/mesh-catalog", "c3cccccccccc", t0.Add(20*time.Second))
|
||||
o1 := leaved("novox/one", "o1dddddddddd", t0.Add(5*time.Second))
|
||||
again, out, err := leaveOut([]inventory.BatchedMerge{c1, o1, c2, c3}, []string{"novox/Mesh-Catalog@c2bbbbbbbbbb"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var walked []string
|
||||
for _, m := range again {
|
||||
walked = append(walked, m.Commit)
|
||||
}
|
||||
if !reflect.DeepEqual(walked, []string{c1.Commit, o1.Commit}) {
|
||||
t.Fatalf("walked again: %v; want the earlier catalogue merge and the other repository's", walked)
|
||||
}
|
||||
want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: c2.Commit},
|
||||
{Repository: "novox/mesh-catalog", Commit: c3.Commit, Contains: c2.Commit}}
|
||||
if !reflect.DeepEqual(out, want) {
|
||||
t.Fatalf("left out: %+v; want %+v", out, want)
|
||||
}
|
||||
// A merge the walk does not answer is refused, naming it: nothing ends.
|
||||
if _, _, err := leaveOut([]inventory.BatchedMerge{c1, o1}, []string{"novox/mesh-catalog@ffffffffffff"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "ffffffffffff") {
|
||||
t.Fatalf("a merge the walk does not answer was left out: %v", err)
|
||||
}
|
||||
if _, _, err := leaveOut([]inventory.BatchedMerge{c1}, []string{"no-commit"}); err == nil {
|
||||
t.Fatal("a merge without its commit was taken")
|
||||
}
|
||||
}
|
||||
|
||||
// moving is a kept merge that moved modules when it was heard.
|
||||
func moving(m inventory.BatchedMerge, modules ...string) inventory.BatchedMerge {
|
||||
m.Event, _ = json.Marshal(keptMerge{Moves: modules})
|
||||
return m
|
||||
}
|
||||
|
||||
// A merge of another repository that moves a module the stopped merge moves is left out too, naming the modules
|
||||
// it shares: the module is built at its branch, which holds the stopped merge (review of #232). And so on, until
|
||||
// none more is: a merge sharing a module with that one is left out as well.
|
||||
func TestAMergeSharingAStoppedMergesModuleIsLeftOut(t *testing.T) {
|
||||
stopped := moving(leaved("novox/mesh-catalog", "c1aaaaaaaaaa", t0), "app")
|
||||
reader := moving(leaved("novox/one", "o1bbbbbbbbbb", t0.Add(time.Second)), "app", "one")
|
||||
next := moving(leaved("novox/two", "t1cccccccccc", t0.Add(2*time.Second)), "one")
|
||||
free := moving(leaved("novox/three", "h1dddddddddd", t0.Add(3*time.Second)), "three")
|
||||
again, out, err := leaveOut([]inventory.BatchedMerge{stopped, reader, next, free}, []string{"novox/mesh-catalog@c1aaaaaaa"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(again) != 1 || again[0].Commit != free.Commit {
|
||||
t.Fatalf("walked again: %+v; want the merge that shares nothing alone", again)
|
||||
}
|
||||
want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: stopped.Commit},
|
||||
{Repository: "novox/one", Commit: reader.Commit, Contains: stopped.Commit, Shares: []string{"app"}},
|
||||
{Repository: "novox/two", Commit: next.Commit, Contains: stopped.Commit, Shares: []string{"one"}}}
|
||||
if !reflect.DeepEqual(out, want) {
|
||||
t.Fatalf("left out %+v; want %+v", out, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A walk withholds a module a stopped merge moved where it would build it only as a dependent; one a merge of its
|
||||
// own batch moves is built, and empty tiers go.
|
||||
func TestAWalkWithholdsAStoppedMergesDependents(t *testing.T) {
|
||||
plan := inventory.Plan{Tiers: [][]string{{"one"}, {"app"}, {"notes", "app2"}},
|
||||
Modules: map[string]*inventory.PlanModule{"one": {}, "app": {}, "notes": {}, "app2": {}}}
|
||||
got := withhold(&plan, []string{"app", "notes", "absent"}, []string{"notes"})
|
||||
if !reflect.DeepEqual(got, []string{"app"}) || !reflect.DeepEqual(plan.Tiers, [][]string{{"one"}, {"notes", "app2"}}) {
|
||||
t.Fatalf("withheld %v, tiers %v", got, plan.Tiers)
|
||||
}
|
||||
if _, in := plan.Modules["app"]; in {
|
||||
t.Fatal("a withheld module is still in the plan")
|
||||
}
|
||||
}
|
||||
|
||||
// The seat verb passes the merges to leave out to the command.
|
||||
func TestDeliveryStopTakesTheMergesToLeaveOut(t *testing.T) {
|
||||
got, err := argvFor("delivery-stop", map[string]any{"plan": "plan-1", "why": "w", "by": "jochen",
|
||||
"without": "novox/mesh-lab@b1,novox/mesh-lab@b2"})
|
||||
want := []string{"delivery", "stop", "plan-1", "--why", "w", "--by", "mesh-delivery for jochen",
|
||||
"--without", "novox/mesh-lab@b1,novox/mesh-lab@b2"}
|
||||
if err != nil || !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("delivery-stop with without → %v %v, wanted %v", got, err, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A walk of the catalogue's merge and another repository's is stopped without the catalogue's: it ends stopped,
|
||||
// names the batch that walks the other merge again and the merge it left out, and that batch is cut into a walk
|
||||
// that builds the other repository's module alone.
|
||||
func TestAStopWithoutAMergeWalksTheOthersAgain(t *testing.T) {
|
||||
open := windowed(t)
|
||||
endsItsLine(t)
|
||||
asksRecorded(t)
|
||||
ctx := t.Context()
|
||||
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
|
||||
one := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second))
|
||||
hear(t, open, cat, t0.Add(time.Second))
|
||||
hear(t, open, one, t0.Add(6*time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 || !ws[0].Open() {
|
||||
t.Fatalf("no open walk to stop: %+v", ws)
|
||||
}
|
||||
stopped := ws[0]
|
||||
p, said, err := stopWalkWithout(ctx, open.inventory, stopped.ID, "mesh-delivery for jochen", "its group stopped",
|
||||
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.State != inventory.PlanFailed || p.Delivery.Stopped == "" || p.Delivery.WalkedAgainBy == "" ||
|
||||
!reflect.DeepEqual(p.Delivery.LeftOut, []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: cat.Commit}}) {
|
||||
t.Fatalf("the stopped walk reads %s %+v", p.State, p.Delivery)
|
||||
}
|
||||
if said.WalkedAgainBy != p.Delivery.WalkedAgainBy || !reflect.DeepEqual(said.WalkedAgain, []string{"novox/one@" + one.Commit}) {
|
||||
t.Fatalf("the stop answered %+v", said)
|
||||
}
|
||||
again, err := open.inventory.MergesOf(ctx, p.Delivery.WalkedAgainBy)
|
||||
if err != nil || len(again) != 1 || again[0].Commit != one.Commit {
|
||||
t.Fatalf("the fresh batch answers %+v (%v); want the other repository's merge alone", again, err)
|
||||
}
|
||||
if left, _ := open.inventory.MergesOf(ctx, stopped.ID); len(left) != 1 || left[0].Commit != cat.Commit {
|
||||
t.Fatalf("the stopped walk still answers %+v; want the left-out merge alone", left)
|
||||
}
|
||||
cutAt(t, open, t0.Add(4*time.Minute))
|
||||
fresh, err := open.inventory.PlanByID(ctx, p.Delivery.WalkedAgainBy)
|
||||
if err != nil || fresh.Batch() || !fresh.Open() {
|
||||
t.Fatalf("the fresh batch was not cut into a walk: %s %v", fresh.State, err)
|
||||
}
|
||||
if _, in := fresh.Modules["one"]; !in {
|
||||
t.Fatalf("the fresh walk does not build one: %v", fresh.Modules)
|
||||
}
|
||||
if _, in := fresh.Modules["app"]; in {
|
||||
t.Fatalf("the fresh walk builds app, whose merge was left out: %v", fresh.Modules)
|
||||
}
|
||||
if len(fresh.Delivery.Merges) != 1 || fresh.Delivery.Merges[0].Commit != one.Commit {
|
||||
t.Fatalf("the fresh walk answers %+v", fresh.Delivery.Merges)
|
||||
}
|
||||
// A stopped walk starts no search of its own (ADR 0276 decision 3 is for a failed one).
|
||||
cutAt(t, open, t0.Add(5*time.Minute))
|
||||
if alone, _ := open.inventory.AloneMerges(ctx); len(alone) != 0 {
|
||||
t.Fatalf("a merge of the stopped walk waits to be walked alone: %+v", alone)
|
||||
}
|
||||
}
|
||||
|
||||
// A later merge of the stopped merge's repository contains it: it is left out too, named with the merge it
|
||||
// contains, and only the other repository's merge is walked again. With nothing left to walk, the walk is only
|
||||
// ended. A merge the walk does not answer is refused, and nothing ends.
|
||||
func TestAStopWithoutAMergeEndsTheMergesThatContainIt(t *testing.T) {
|
||||
open := windowed(t)
|
||||
endsItsLine(t)
|
||||
asksRecorded(t)
|
||||
ctx := t.Context()
|
||||
first := catalogueMerge("cat1aaaaaaaa", "app", t0)
|
||||
later := catalogueMerge("cat2cccccccc", "notes", t0.Add(10*time.Second))
|
||||
one := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second))
|
||||
hear(t, open, first, t0.Add(time.Second))
|
||||
hear(t, open, one, t0.Add(6*time.Second))
|
||||
hear(t, open, later, t0.Add(11*time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
walk := ws[0]
|
||||
|
||||
if _, _, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "x",
|
||||
[]string{"novox/mesh-catalog@ffffffffffff"}, t0.Add(3*time.Minute)); err == nil {
|
||||
t.Fatal("a stop leaving out a merge the walk does not answer was taken")
|
||||
}
|
||||
if p, _ := open.inventory.PlanByID(ctx, walk.ID); !p.Open() {
|
||||
t.Fatalf("a refused stop ended the walk: %s", p.State)
|
||||
}
|
||||
|
||||
p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped",
|
||||
[]string{"novox/mesh-catalog@" + first.Commit}, t0.Add(3*time.Minute))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: first.Commit},
|
||||
{Repository: "novox/mesh-catalog", Commit: later.Commit, Contains: first.Commit}}
|
||||
if !reflect.DeepEqual(p.Delivery.LeftOut, want) || !reflect.DeepEqual(said.LeftOut, want) {
|
||||
t.Fatalf("left out %+v, said %+v; want %+v", p.Delivery.LeftOut, said.LeftOut, want)
|
||||
}
|
||||
if !strings.Contains(p.Note, later.Commit[:8]) || !strings.Contains(p.Note, p.Delivery.WalkedAgainBy) {
|
||||
t.Fatalf("the stopped walk's note does not name the merge ended with it and the fresh batch: %q", p.Note)
|
||||
}
|
||||
again, _ := open.inventory.MergesOf(ctx, p.Delivery.WalkedAgainBy)
|
||||
if len(again) != 1 || again[0].Commit != one.Commit {
|
||||
t.Fatalf("walked again: %+v; want the other repository's merge alone", again)
|
||||
}
|
||||
|
||||
// Nothing left once the other repository's merge is left out too: the fresh walk is stopped, and nothing more.
|
||||
cutAt(t, open, t0.Add(4*time.Minute))
|
||||
fresh, _ := open.inventory.PlanByID(ctx, p.Delivery.WalkedAgainBy)
|
||||
q, said, err := stopWalkWithout(ctx, open.inventory, fresh.ID, "mesh-delivery for jochen", "that one too",
|
||||
[]string{"novox/one@" + one.Commit}, t0.Add(5*time.Minute))
|
||||
if err != nil || q.State != inventory.PlanFailed || q.Delivery.WalkedAgainBy != "" || said.WalkedAgainBy != "" {
|
||||
t.Fatalf("a stop leaving nothing to walk: %s %+v %+v %v", q.State, q.Delivery, said, err)
|
||||
}
|
||||
if _, bs := walks(t, open); len(bs) != 0 {
|
||||
t.Fatalf("a batch of nothing was made: %+v", bs)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge heard while the walk ran waits in the open batch, at a later commit of the same repository. The merge
|
||||
// walked again joins that batch, never a batch of its own that would walk the older commit after the newer one
|
||||
// (review of #232): its window closes, and its walk is at the newer commit, answering both.
|
||||
func TestAStopWalksTheOthersAgainInTheOpenBatch(t *testing.T) {
|
||||
open := windowed(t)
|
||||
endsItsLine(t)
|
||||
asksRecorded(t)
|
||||
ctx := t.Context()
|
||||
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
|
||||
one1 := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second))
|
||||
hear(t, open, cat, t0.Add(time.Second))
|
||||
hear(t, open, one1, t0.Add(6*time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
walk := ws[0]
|
||||
one2 := repoMerge("one", "one2cccccccc", t0.Add(3*time.Minute))
|
||||
hear(t, open, one2, t0.Add(3*time.Minute))
|
||||
_, bs := walks(t, open)
|
||||
if len(bs) != 1 {
|
||||
t.Fatalf("the merge heard while the walk ran is in %d batches", len(bs))
|
||||
}
|
||||
waiting := bs[0]
|
||||
p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped",
|
||||
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.Delivery.WalkedAgainBy != waiting.ID || said.WalkedAgainBy != waiting.ID {
|
||||
t.Fatalf("walked again by %q (said %q); want the open batch %s", p.Delivery.WalkedAgainBy, said.WalkedAgainBy, waiting.ID)
|
||||
}
|
||||
if _, bs := walks(t, open); len(bs) != 1 {
|
||||
t.Fatalf("%d batches; want the open one alone", len(bs))
|
||||
}
|
||||
b, _ := open.inventory.PlanByID(ctx, waiting.ID)
|
||||
if b.Delivery.Batch == nil || !b.Delivery.Batch.Closed || !slices.Contains(b.Delivery.Batch.Withheld, "app") {
|
||||
t.Fatalf("the open batch reads %+v; want its window closed and app withheld", b.Delivery.Batch)
|
||||
}
|
||||
// Its window closed by the stop, not by the clock: cut at once.
|
||||
cutAt(t, open, t0.Add(3*time.Minute+11*time.Second))
|
||||
fresh, _ := open.inventory.PlanByID(ctx, waiting.ID)
|
||||
if fresh.Batch() || fresh.CommitOf("novox/one") != one2.Commit {
|
||||
t.Fatalf("the batch is %s at %q; want it cut, walking the newer commit", fresh.State, fresh.CommitOf("novox/one"))
|
||||
}
|
||||
var answered []string
|
||||
for _, m := range fresh.Delivery.Merges {
|
||||
answered = append(answered, m.Commit+">"+m.Carried)
|
||||
}
|
||||
if !slices.Contains(answered, one1.Commit+">"+one2.Commit) || !slices.Contains(answered, one2.Commit+">") {
|
||||
t.Fatalf("the walk answers %v; want the earlier merge carried by the later", answered)
|
||||
}
|
||||
}
|
||||
|
||||
// A stop names what the stopped walk already sent and nothing walks again, on which machines; and a later merge of
|
||||
// the stopped merge's repository that waits in a batch, which will deliver the stopped change. A stopped walk
|
||||
// that walked its merges again is refused a retry, saying why.
|
||||
func TestAStopNamesWhatStaysOnTheMachinesAndWhatStillCarriesIt(t *testing.T) {
|
||||
open := windowed(t)
|
||||
endsItsLine(t)
|
||||
asksRecorded(t)
|
||||
ctx := t.Context()
|
||||
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
|
||||
one1 := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second))
|
||||
hear(t, open, cat, t0.Add(time.Second))
|
||||
hear(t, open, one1, t0.Add(6*time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
walk := ws[0]
|
||||
sent := t0.Add(150 * time.Second)
|
||||
for name, m := range walk.Modules {
|
||||
if name == "app" || name == "one" {
|
||||
m.First, m.FirstAt = []string{"anchor"}, &sent
|
||||
}
|
||||
}
|
||||
if err := open.inventory.SavePlan(ctx, &walk); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cat2 := catalogueMerge("cat2dddddddd", "notes", t0.Add(3*time.Minute))
|
||||
hear(t, open, cat2, t0.Add(3*time.Minute))
|
||||
_, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped",
|
||||
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(said.Kept, []inventory.PlanKept{{Module: "app", Machines: []string{"anchor"}}}) {
|
||||
t.Fatalf("kept %+v; want app on anchor alone (one is walked again)", said.Kept)
|
||||
}
|
||||
p, _ := open.inventory.PlanByID(ctx, walk.ID)
|
||||
if !reflect.DeepEqual(p.Delivery.Kept, said.Kept) || !strings.Contains(p.Note, "app on anchor") {
|
||||
t.Fatalf("the stopped walk's record does not name what stays: %+v %q", p.Delivery.Kept, p.Note)
|
||||
}
|
||||
if len(said.StillCarriedBy) != 1 || !strings.Contains(said.StillCarriedBy[0], cat2.Commit) {
|
||||
t.Fatalf("still carried by %v; want the later catalogue merge waiting in the batch", said.StillCarriedBy)
|
||||
}
|
||||
if _, err := retryPlan(ctx, open, walk.ID); err == nil || !strings.Contains(err.Error(), "was stopped without") {
|
||||
t.Fatalf("a walk stopped without some merges was retried, or refused for another reason: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A batch not yet cut is stopped without a merge: the merge goes to a stopped record of its own, the batch goes on
|
||||
// with the others and withholds the stopped merge's modules; with every merge left out, the batch is stopped.
|
||||
func TestAStopWithoutAMergeLeavesItOutOfABatch(t *testing.T) {
|
||||
open := windowed(t)
|
||||
endsItsLine(t)
|
||||
asksRecorded(t)
|
||||
ctx := t.Context()
|
||||
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
|
||||
one := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second))
|
||||
hear(t, open, cat, t0.Add(time.Second))
|
||||
hear(t, open, one, t0.Add(6*time.Second))
|
||||
_, bs := walks(t, open)
|
||||
b := bs[0]
|
||||
r, said, err := stopWalkWithout(ctx, open.inventory, b.ID, "mesh-delivery for jochen", "its group stopped",
|
||||
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(10*time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if said.WalkedAgainBy != b.ID || said.Stopped == b.ID {
|
||||
t.Fatalf("said %+v; want a stopped record of its own, and the batch walking the others", said)
|
||||
}
|
||||
if left, _ := open.inventory.MergesOf(ctx, said.Stopped); len(left) != 1 || left[0].Commit != cat.Commit {
|
||||
t.Fatalf("the stopped record holds %+v", left)
|
||||
}
|
||||
if stopped, _ := open.inventory.PlanByID(ctx, said.Stopped); stopped.State != inventory.PlanFailed ||
|
||||
stopped.Delivery.Stopped == "" || stopped.Delivery.WalkedAgainBy != b.ID {
|
||||
t.Fatalf("the stopped record reads %s %+v", stopped.State, stopped.Delivery)
|
||||
}
|
||||
_ = r
|
||||
cutAt(t, open, t0.Add(3*time.Minute))
|
||||
w, _ := open.inventory.PlanByID(ctx, b.ID)
|
||||
if _, in := w.Modules["app"]; in || w.Batch() {
|
||||
t.Fatalf("the batch's walk is %s and builds %v; want it cut without app", w.State, w.Modules)
|
||||
}
|
||||
// A batch whose every merge is left out is stopped itself.
|
||||
hear(t, open, catalogueMerge("cat3eeeeeeee", "notes", t0.Add(4*time.Minute)), t0.Add(4*time.Minute))
|
||||
_, bs = walks(t, open)
|
||||
last := bs[0]
|
||||
if _, said, err := stopWalkWithout(ctx, open.inventory, last.ID, "mesh-delivery for jochen", "x",
|
||||
[]string{"novox/mesh-catalog@cat3eeeeeeee"}, t0.Add(4*time.Minute+time.Second)); err != nil || said.Stopped != last.ID {
|
||||
t.Fatalf("a batch left with nothing: %+v %v", said, err)
|
||||
}
|
||||
if p, _ := open.inventory.PlanByID(ctx, last.ID); p.State != inventory.PlanFailed || p.Batch() {
|
||||
t.Fatalf("the emptied batch is %s", p.State)
|
||||
}
|
||||
}
|
||||
|
||||
// The stopped walk, the batch and the merges moved are one act (ADR 0299 decision 7): a plan written by somebody
|
||||
// else since it was read refuses all of it, and no merge moves, no plan is written.
|
||||
func TestSavingPlansAndMovingMergesIsOneAct(t *testing.T) {
|
||||
open := windowed(t)
|
||||
endsItsLine(t)
|
||||
ctx := t.Context()
|
||||
hear(t, open, catalogueMerge("cat1aaaaaaaa", "app", t0), t0.Add(time.Second))
|
||||
_, bs := walks(t, open)
|
||||
stale := bs[0]
|
||||
moved := stale
|
||||
moved.Note = "written by somebody else"
|
||||
if err := open.inventory.SavePlan(ctx, &moved); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fresh := inventory.Plan{ID: "plan-fresh", Repository: "novox/mesh-catalog", Branch: "main", Commit: "x", Created: t0,
|
||||
State: inventory.PlanAssembling, Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{}}
|
||||
err := open.inventory.SavePlansMoving(ctx, []*inventory.Plan{&fresh, &stale},
|
||||
[]inventory.MergeMove{{Repository: "novox/mesh-catalog", Commit: "cat1aaaaaaaa", Plan: "plan-fresh"}})
|
||||
if !errors.Is(err, inventory.ErrPlanMoved) {
|
||||
t.Fatalf("a stale plan in the act: %v", err)
|
||||
}
|
||||
if _, err := open.inventory.PlanByID(ctx, "plan-fresh"); err == nil {
|
||||
t.Fatal("the new plan was written though the act was refused")
|
||||
}
|
||||
if m, _, _ := open.inventory.MergeOf(ctx, "novox/mesh-catalog", "cat1aaaaaaaa"); m.Plan != stale.ID {
|
||||
t.Fatalf("the merge moved to %q though the act was refused", m.Plan)
|
||||
}
|
||||
}
|
||||
|
||||
// What a stopped walk leaves on the machines names every machine its sends reached (review of #232): a module sent
|
||||
// with no first machine reached every machine running it; one sent first reached those, and the rest it was sent
|
||||
// to; a module a merge walked again moves is sent again, and is not named.
|
||||
func TestKeptNamesEveryMachineASendReached(t *testing.T) {
|
||||
at := t0
|
||||
p := inventory.Plan{Modules: map[string]*inventory.PlanModule{
|
||||
"together": {SentAt: &at},
|
||||
"first": {First: []string{"anchor"}, FirstAt: &at, Rest: map[string]inventory.SentDeclaration{"laptop": {}}},
|
||||
"again": {SentAt: &at},
|
||||
"unsent": {},
|
||||
}}
|
||||
got := keptOf(p, []string{"again"}, map[string][]string{"together": {"laptop", "anchor"}, "again": {"anchor"}})
|
||||
want := []inventory.PlanKept{{Module: "first", Machines: []string{"anchor", "laptop"}},
|
||||
{Module: "together", Machines: []string{"anchor", "laptop"}}}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("kept %+v; want %+v", got, want)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,272 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/secrets"
|
||||
)
|
||||
|
||||
// `secret issued <provision>`: the credential a provider's software issued, taken from the provider and kept as
|
||||
// the provision's one credential for every consumer (novox/hq ADR 0305, issue 526).
|
||||
//
|
||||
// **The value comes from the provider, never from whoever asks.** A module served by its machine's runtime calls
|
||||
// under the runtime's one bus account, which carries that machine's agents' calls too, so a value handed to a
|
||||
// verb could be anybody's. This line carries none: it asks the provider's own tool, on the provider's machine, in
|
||||
// the controller's own name — the subject that names its caller, which the bus lets the controller alone publish
|
||||
// (issue 365) — with a sealing key made for this one call. The tool answers only the controller, the value sealed
|
||||
// to that key; the controller opens it, seals it to every holder and forgets it, as the desk's value is (ADR 0277
|
||||
// §23). Asking again fetches the same value again, and a value every holder already holds is not sealed anew.
|
||||
//
|
||||
// **What this trusts: the provider's machine.** The answer is taken from whoever answers the provider's tool
|
||||
// subject on the provider's machine first. On that machine its runtime serves the subject for every module it
|
||||
// carries, so anything that runtime runs, or anything else granted that subject there, could answer before the
|
||||
// provider does and hand the controller a value of its own. ADR 0305 accepts that by trusting the provider's
|
||||
// machine, as the provider itself is trusted; the bus keeps every other machine from answering it.
|
||||
|
||||
// handActIssued is the verb a change of an issued value is recorded under: the mesh working as decided.
|
||||
const handActIssued = "secret issued"
|
||||
|
||||
// issuedAsk is how long the provider's tool is given to answer.
|
||||
const issuedAsk = 20 * time.Second
|
||||
|
||||
// issuedProvider is one assignment of a module whose offer says its software issues the provision's credential.
|
||||
type issuedProvider struct {
|
||||
Node, Module, Tool string
|
||||
}
|
||||
|
||||
// issuedTaker is the line's work, with what it reaches handed in: the store, the bus and the sends.
|
||||
type issuedTaker struct {
|
||||
// providers are the assignments to ask, and whether any module offers the provision so at all.
|
||||
providers func(provision string) ([]issuedProvider, bool, error)
|
||||
ask func(p issuedProvider, args map[string]any) (link.Answer, error)
|
||||
keep func(provision, provider, value string) (inventory.IssuedKept, error)
|
||||
record func(act link.HandAct) error
|
||||
send func(machines []string) error
|
||||
}
|
||||
|
||||
// take asks every provider of the provision for its software's value and keeps each; it answers what it did, in
|
||||
// words that never carry the value or its fingerprint.
|
||||
func (t issuedTaker) take(provision string) (string, error) {
|
||||
providers, offered, err := t.providers(provision)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !offered {
|
||||
return "", fmt.Errorf("no module offers %q with a credential its software issues (\"credential\": "+
|
||||
"{\"issued-by\": \"software\", \"tool\": …}, ADR 0305), so there is nothing to take from a provider; "+
|
||||
"nothing was asked", provision)
|
||||
}
|
||||
if len(providers) == 0 {
|
||||
return fmt.Sprintf("no consumer holds %s from a provider whose software issues it yet, so there is nothing "+
|
||||
"to seal; a consumer that binds holds a value the mesh made until the provider next asks", provision), nil
|
||||
}
|
||||
var said []string
|
||||
var failed []string
|
||||
for _, p := range providers {
|
||||
words, err := t.takeFrom(provision, p)
|
||||
if err != nil {
|
||||
failed = append(failed, fmt.Sprintf("%s on %s: %v", p.Module, p.Node, err))
|
||||
continue
|
||||
}
|
||||
said = append(said, words)
|
||||
}
|
||||
out := strings.Join(said, "\n")
|
||||
if len(failed) > 0 {
|
||||
return out, fmt.Errorf("%s was not taken from every provider:\n %s", provision, strings.Join(failed, "\n "))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// takeFrom is one provider: asked, opened, kept, recorded and sent when it changed.
|
||||
func (t issuedTaker) takeFrom(provision string, p issuedProvider) (string, error) {
|
||||
public, private, err := secrets.Keypair()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("no key could be made to take the value: %w", err)
|
||||
}
|
||||
answer, err := t.ask(p, map[string]any{"provision": provision, "seal_to": public})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("its tool %s could not be asked: %w", p.Tool, err)
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return "", fmt.Errorf("its tool %s refused: %s", p.Tool, answer.Error)
|
||||
}
|
||||
var given struct {
|
||||
Sealed string `json:"sealed"`
|
||||
}
|
||||
if err := unmarshalAnswer(answer, &given); err != nil || given.Sealed == "" {
|
||||
return "", fmt.Errorf("its tool %s answered no sealed value", p.Tool)
|
||||
}
|
||||
opened, err := secrets.Open(private, given.Sealed)
|
||||
if err != nil {
|
||||
// Never the value, never what failed to open: only that it was not sealed to this call.
|
||||
return "", fmt.Errorf("its tool %s answered a value not sealed to this call; nothing was taken", p.Tool)
|
||||
}
|
||||
value := strings.TrimSpace(string(opened))
|
||||
for i := range opened {
|
||||
opened[i] = 0
|
||||
}
|
||||
if value == "" {
|
||||
return "", fmt.Errorf("its tool %s answered an empty value; nothing was taken", p.Tool)
|
||||
}
|
||||
kept, err := t.keep(provision, p.Node, value)
|
||||
value = ""
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
holders := make([]string, 0, len(kept.Holders))
|
||||
for _, h := range kept.Holders {
|
||||
holders = append(holders, fmt.Sprintf("%s on %s%s", h.ConsumerModule, h.Consumer, asLocal(h.Local)))
|
||||
}
|
||||
words := ""
|
||||
if len(kept.Unsealable) > 0 {
|
||||
var names []string
|
||||
for _, h := range kept.Unsealable {
|
||||
names = append(names, fmt.Sprintf("%s on %s", h.ConsumerModule, h.Consumer))
|
||||
}
|
||||
words = fmt.Sprintf("\n not sealed to %s: no sealing key on that machine", strings.Join(names, ", "))
|
||||
}
|
||||
if !kept.Changed {
|
||||
return fmt.Sprintf("%s from %s on %s: every holder already holds the value its software issued (%s); "+
|
||||
"nothing was sent%s", provision, p.Module, p.Node, strings.Join(holders, ", "), words), nil
|
||||
}
|
||||
sealedTo := make([]string, 0, len(kept.Sealed))
|
||||
for _, h := range kept.Sealed {
|
||||
sealedTo = append(sealedTo, fmt.Sprintf("%s on %s%s", h.ConsumerModule, h.Consumer, asLocal(h.Local)))
|
||||
}
|
||||
recorded := ""
|
||||
if err := t.record(link.HandAct{Verb: handActIssued, Args: []string{provision},
|
||||
Why: fmt.Sprintf("%s on %s gave the credential its software issued for %s; sealed to %s", p.Module, p.Node,
|
||||
provision, strings.Join(sealedTo, ", ")),
|
||||
Cause: "issued-by-software"}); err != nil {
|
||||
recorded = fmt.Sprintf("\n this was NOT recorded in the hand-act log, and is done anyway: %v", err)
|
||||
}
|
||||
if err := t.send(kept.Machines); err != nil {
|
||||
return "", fmt.Errorf("%s from %s on %s is sealed to %s and could not be sent: %w; `push --behind` sends it",
|
||||
provision, p.Module, p.Node, strings.Join(sealedTo, ", "), err)
|
||||
}
|
||||
return fmt.Sprintf("%s from %s on %s: the value its software issued is sealed to %s; sent %s, and each of "+
|
||||
"them starts again on it%s%s", provision, p.Module, p.Node, strings.Join(sealedTo, ", "),
|
||||
strings.Join(kept.Machines, ", "), words, recorded), nil
|
||||
}
|
||||
|
||||
// issuedProvidersOf are the assignments to ask for a provision: every machine providing it to a pair the mesh
|
||||
// holds, and on it the assigned module whose offer says its software issues the credential (ADR 0305 §1).
|
||||
func issuedProvidersOf(ctx context.Context, inv *inventory.Inventory, provision string) ([]issuedProvider, bool, error) {
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if !offersIssued(shelf, provision) {
|
||||
return nil, false, nil
|
||||
}
|
||||
holders, err := inv.HoldersOf(ctx, provision, "")
|
||||
if err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
nodes := map[string]bool{}
|
||||
for _, h := range holders {
|
||||
nodes[h.Provider] = true
|
||||
}
|
||||
names := make([]string, 0, len(nodes))
|
||||
for n := range nodes {
|
||||
names = append(names, n)
|
||||
}
|
||||
sort.Strings(names)
|
||||
var out []issuedProvider
|
||||
for _, node := range names {
|
||||
assigned, err := inv.Assigned(ctx, node)
|
||||
if err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
out = append(out, issuedOn(shelf, node, assigned, provision)...)
|
||||
}
|
||||
return out, true, nil
|
||||
}
|
||||
|
||||
// offersIssued is whether any module offers the provision with a credential its software issues.
|
||||
func offersIssued(shelf map[string]catalogue.Manifest, provision string) bool {
|
||||
for _, m := range shelf {
|
||||
if _, issued := m.IssuedCredentialOf(provision); issued {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// issuedOn are the modules assigned to one machine whose offer says their software issues the provision's
|
||||
// credential, each with the tool that answers it.
|
||||
func issuedOn(shelf map[string]catalogue.Manifest, node string, assigned []string, provision string) []issuedProvider {
|
||||
var out []issuedProvider
|
||||
for _, module := range assigned {
|
||||
if tool, issued := shelf[module].IssuedCredentialOf(provision); issued {
|
||||
out = append(out, issuedProvider{Node: node, Module: module, Tool: tool})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// issuedRefusesRotation refuses `rotate` of a provision its provider's software issues (ADR 0305 §9): a value the
|
||||
// mesh made would replace one the software holds, and every consumer would hold a dead credential until the
|
||||
// provider next asks.
|
||||
func issuedRefusesRotation(shelf map[string]catalogue.Manifest, provision string) error {
|
||||
if !offersIssued(shelf, provision) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%s's credential is issued by its provider's software, which the mesh cannot make or "+
|
||||
"rotate (ADR 0305): change it in that software, and `secret issued %s` takes the value it holds now. "+
|
||||
"Nothing was rotated", provision, provision)
|
||||
}
|
||||
|
||||
// secretIssued is `secret issued <provision>`, on this controller's stores and bus.
|
||||
func secretIssued(ctx context.Context, args []string) error {
|
||||
if len(args) != 1 || args[0] == "" || strings.HasPrefix(args[0], "-") {
|
||||
return errors.New("secret issued <provision>")
|
||||
}
|
||||
provision := args[0]
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
t := issuedTaker{
|
||||
providers: func(provision string) ([]issuedProvider, bool, error) {
|
||||
return issuedProvidersOf(ctx, open.inventory, provision)
|
||||
},
|
||||
ask: func(p issuedProvider, args map[string]any) (link.Answer, error) {
|
||||
var answer link.Answer
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
var err error
|
||||
answer, err = link.AskModuleToolOnAs(ctx, conn, broker.ControllerName, p.Module, p.Tool, p.Node,
|
||||
args, issuedAsk)
|
||||
return err
|
||||
})
|
||||
return answer, err
|
||||
},
|
||||
keep: func(provision, provider, value string) (inventory.IssuedKept, error) {
|
||||
return open.inventory.KeepIssuedSecret(ctx, provision, provider, value)
|
||||
},
|
||||
record: func(act link.HandAct) error {
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
_, err := link.RecordHandAct(ctx, conn, act)
|
||||
return err
|
||||
})
|
||||
},
|
||||
send: func(machines []string) error { return sendTo(ctx, open, machines) },
|
||||
}
|
||||
words, err := t.take(provision)
|
||||
if words != "" {
|
||||
fmt.Println(words)
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/secrets"
|
||||
)
|
||||
|
||||
// The `issued` verb runs `secret issued <provision>` and nothing else, and that line is the one `secret` line
|
||||
// beside rotate and the desk's that a verb may run (novox/hq ADR 0305 §3, §10, issue 526).
|
||||
func TestTheIssuedVerbRunsTheOneLineThatCarriesNoValue(t *testing.T) {
|
||||
argv, err := argvFor("issued", map[string]any{"provision": "plex-api"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(argv, " ") != "secret issued plex-api" {
|
||||
t.Fatalf("the verb composed %q", argv)
|
||||
}
|
||||
if _, err := argvFor("issued", map[string]any{}); err == nil {
|
||||
t.Fatal("the verb ran with no provision")
|
||||
}
|
||||
if _, err := argvFor("issued", map[string]any{"provision": "plex-api", "secret": "a value"}); err == nil {
|
||||
t.Fatal("the verb took a value beside the provision")
|
||||
}
|
||||
for _, line := range [][]string{
|
||||
{"secret", "issued", "plex-api", "--from", "/tmp/x"},
|
||||
{"secret", "issued", "--from"},
|
||||
{"secret", "issued"},
|
||||
{"secret", "accept", "ace", "kometa", "plex-api", "--provider", "ace"},
|
||||
{"secret", "recover", "ace", "kometa", "plex-api"},
|
||||
} {
|
||||
if err := terminalOnly(line); err == nil {
|
||||
t.Errorf("%q ran through a verb", line)
|
||||
}
|
||||
}
|
||||
if err := terminalOnly([]string{"secret", "issued", "plex-api"}); err != nil {
|
||||
t.Fatalf("the issued line was refused through its verb: %v", err)
|
||||
}
|
||||
found := false
|
||||
for _, v := range catalogue.ControllerVerbs {
|
||||
if v.Name == "issued" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("the controller's seat does not declare the issued verb")
|
||||
}
|
||||
}
|
||||
|
||||
// A provider's tool answering the value sealed to the call's key: what a provider does (ADR 0305 §2).
|
||||
func issuedSealed(t *testing.T, value string) func(issuedProvider, map[string]any) (link.Answer, error) {
|
||||
return func(_ issuedProvider, args map[string]any) (link.Answer, error) {
|
||||
key, _ := args["seal_to"].(string)
|
||||
if key == "" {
|
||||
t.Fatal("the provider was asked without a key to seal to")
|
||||
}
|
||||
if _, carried := args["value"]; carried {
|
||||
t.Fatal("the ask carried a value")
|
||||
}
|
||||
sealed, err := secrets.Seal(key, []byte(value))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(map[string]string{"sealed": sealed})
|
||||
return link.Answer{Result: raw}, nil
|
||||
}
|
||||
}
|
||||
|
||||
type issuedRecorder struct {
|
||||
kept []string
|
||||
acts []link.HandAct
|
||||
sent [][]string
|
||||
again bool // the value is held by every holder already
|
||||
}
|
||||
|
||||
func (r *issuedRecorder) taker(t *testing.T, ask func(issuedProvider, map[string]any) (link.Answer, error)) issuedTaker {
|
||||
return issuedTaker{
|
||||
providers: func(provision string) ([]issuedProvider, bool, error) {
|
||||
return []issuedProvider{{Node: "ace", Module: "plex", Tool: "plex_issued_credential"}}, true, nil
|
||||
},
|
||||
ask: ask,
|
||||
keep: func(provision, provider, value string) (inventory.IssuedKept, error) {
|
||||
r.kept = append(r.kept, value)
|
||||
holders := []inventory.Holder{{Provision: provision, Consumer: "ace", ConsumerModule: "kometa", Provider: provider},
|
||||
{Provision: provision, Consumer: "ace", ConsumerModule: "tautulli", Provider: provider}}
|
||||
if r.again {
|
||||
return inventory.IssuedKept{Holders: holders}, nil
|
||||
}
|
||||
return inventory.IssuedKept{Changed: true, Holders: holders, Sealed: holders, Machines: []string{"ace"}}, nil
|
||||
},
|
||||
record: func(act link.HandAct) error { r.acts = append(r.acts, act); return nil },
|
||||
send: func(machines []string) error { r.sent = append(r.sent, machines); return nil },
|
||||
}
|
||||
}
|
||||
|
||||
// The value the provider's tool sealed to the call is kept, recorded and sent, and the answer never says it.
|
||||
func TestAnIssuedValueIsTakenSealedFromTheProviderAndSent(t *testing.T) {
|
||||
var r issuedRecorder
|
||||
words, err := r.taker(t, issuedSealed(t, "the-servers-own-token")).take("plex-api")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(r.kept) != 1 || r.kept[0] != "the-servers-own-token" {
|
||||
t.Fatalf("the provider's value was not kept: %d kept", len(r.kept))
|
||||
}
|
||||
if len(r.sent) != 1 || strings.Join(r.sent[0], ",") != "ace" {
|
||||
t.Fatalf("the holders' machines were not sent: %v", r.sent)
|
||||
}
|
||||
if len(r.acts) != 1 || r.acts[0].Verb != handActIssued {
|
||||
t.Fatalf("the change was not recorded in the hand-act log: %+v", r.acts)
|
||||
}
|
||||
for _, said := range []string{words, r.acts[0].Why, strings.Join(r.acts[0].Args, " ")} {
|
||||
if strings.Contains(said, "the-servers-own-token") || strings.Contains(said, inventory.IssuedStamp("the-servers-own-token")) {
|
||||
t.Fatalf("the value or its fingerprint was said: %q", said)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(words, "kometa on ace") || !strings.Contains(words, "tautulli on ace") {
|
||||
t.Fatalf("the answer does not name the holders: %q", words)
|
||||
}
|
||||
}
|
||||
|
||||
// The same value again writes nothing, records nothing and sends nobody (ADR 0305 §5).
|
||||
func TestAnIssuedValueEveryHolderHoldsSendsNobody(t *testing.T) {
|
||||
r := issuedRecorder{again: true}
|
||||
words, err := r.taker(t, issuedSealed(t, "the-servers-own-token")).take("plex-api")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(r.sent) != 0 || len(r.acts) != 0 {
|
||||
t.Fatalf("an unchanged value was sent or recorded: %v %v", r.sent, r.acts)
|
||||
}
|
||||
if !strings.Contains(words, "nothing was sent") {
|
||||
t.Fatalf("the answer does not say nothing was sent: %q", words)
|
||||
}
|
||||
}
|
||||
|
||||
// A value not sealed to the call's key is not taken, nor is a refusal, nor an empty answer (ADR 0305 §2).
|
||||
func TestOnlyAValueSealedToTheCallIsTaken(t *testing.T) {
|
||||
other, _, _ := secrets.Keypair()
|
||||
elsewhere, _ := secrets.Seal(other, []byte("somebody's value"))
|
||||
raw, _ := json.Marshal(map[string]string{"sealed": elsewhere})
|
||||
plain, _ := json.Marshal(map[string]string{"value": "the-servers-own-token"})
|
||||
for name, ask := range map[string]func(issuedProvider, map[string]any) (link.Answer, error){
|
||||
"sealed to another key": func(issuedProvider, map[string]any) (link.Answer, error) { return link.Answer{Result: raw}, nil },
|
||||
"answered in the clear": func(issuedProvider, map[string]any) (link.Answer, error) { return link.Answer{Result: plain}, nil },
|
||||
"refused": func(issuedProvider, map[string]any) (link.Answer, error) {
|
||||
return link.Answer{Error: "the controller alone"}, nil
|
||||
},
|
||||
"not reached": func(issuedProvider, map[string]any) (link.Answer, error) {
|
||||
return link.Answer{}, errors.New("503 no responders")
|
||||
},
|
||||
"sealed empty": issuedSealed(t, " "),
|
||||
} {
|
||||
var r issuedRecorder
|
||||
if _, err := r.taker(t, ask).take("plex-api"); err == nil {
|
||||
t.Errorf("%s: taken", name)
|
||||
}
|
||||
if len(r.kept) != 0 || len(r.sent) != 0 {
|
||||
t.Errorf("%s: something was kept or sent", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A provision no offer says its software issues is refused before anything is asked.
|
||||
func TestAProvisionNoOfferSaysIsIssuedIsRefused(t *testing.T) {
|
||||
asked := false
|
||||
taker := issuedTaker{
|
||||
providers: func(string) ([]issuedProvider, bool, error) { return nil, false, nil },
|
||||
ask: func(issuedProvider, map[string]any) (link.Answer, error) {
|
||||
asked = true
|
||||
return link.Answer{}, nil
|
||||
},
|
||||
}
|
||||
if _, err := taker.take("postgres-database"); err == nil || !strings.Contains(err.Error(), "ADR 0305") {
|
||||
t.Fatalf("expected a refusal naming the record, got %v", err)
|
||||
}
|
||||
if asked {
|
||||
t.Fatal("a provider was asked for a provision no offer says its software issues")
|
||||
}
|
||||
}
|
||||
|
||||
// Which assignments are asked: the modules on the providing machine whose offer says so, with their tool.
|
||||
func TestTheProvidersAskedAreTheOffersThatSaySo(t *testing.T) {
|
||||
shelf := map[string]catalogue.Manifest{
|
||||
"plex": {Module: "plex", Provides: []catalogue.Offer{{Name: "plex-api", Credential: &catalogue.OfferCredential{
|
||||
IssuedBy: catalogue.IssuedBySoftware, Tool: "plex_issued_credential"}}}},
|
||||
"jellyfin": {Module: "jellyfin", Provides: []catalogue.Offer{{Name: "plex-api"}}},
|
||||
}
|
||||
got := issuedOn(shelf, "ace", []string{"jellyfin", "plex", "kometa"}, "plex-api")
|
||||
if len(got) != 1 || got[0] != (issuedProvider{Node: "ace", Module: "plex", Tool: "plex_issued_credential"}) {
|
||||
t.Fatalf("the providers asked: %+v", got)
|
||||
}
|
||||
if err := issuedRefusesRotation(shelf, "plex-api"); err == nil || !strings.Contains(err.Error(), "secret issued plex-api") {
|
||||
t.Fatalf("rotate of an issued provision was not refused with the way to take it: %v", err)
|
||||
}
|
||||
if err := issuedRefusesRotation(shelf, "postgres-database"); err != nil {
|
||||
t.Fatalf("rotate of an ordinary provision was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A change of an issued value is the mesh working as decided, never a repair (ADR 0305 §8).
|
||||
func TestAnIssuedValueIsADecisionInTheHandActLog(t *testing.T) {
|
||||
for _, v := range handActVerbs {
|
||||
if v.Verb == handActIssued {
|
||||
if v.Decision == "" {
|
||||
t.Fatal("secret issued counts as a repair")
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("secret issued is not in the hand-act table")
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq issue 485: the mesh records every send — sender, lease epoch, generation, modules — and only `plan`
|
||||
// said it. `status` says per machine when it was last sent a declaration and by whom, in one line; `node show`
|
||||
// says the last send in full and what the machine answered; and `status --json` carries it as fields, since
|
||||
// the store's reader may not read the controller's tables and these verbs are the only window onto them.
|
||||
|
||||
func aLastSend(answer inventory.SendAnswer) inventory.Send {
|
||||
sent := time.Date(2026, 10, 11, 18, 40, 5, 0, time.Local)
|
||||
return inventory.Send{NodeName: "laptop", Sequence: 12, Epoch: 57,
|
||||
Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12",
|
||||
Modules: []string{"docker", "sudo"}, SentAt: sent, Recorded: true, Answer: answer}
|
||||
}
|
||||
|
||||
func TestStatusSaysEachMachinesLastSendInOneLine(t *testing.T) {
|
||||
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.Local)
|
||||
asked := answers{
|
||||
nodes: []inventory.Node{{Name: "anchor", LastSeen: time.Now()}, {Name: "laptop", LastSeen: time.Now()}},
|
||||
lastSent: map[string]inventory.Send{"laptop": aLastSend(inventory.SendAnswer{
|
||||
Outcome: inventory.OutcomeApplied, At: &at})},
|
||||
}
|
||||
shown := printed(t, func() error { return printStatus(asked) })
|
||||
var laptop, anchor []string
|
||||
for _, line := range strings.Split(shown, "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) > 0 && fields[0] == "laptop" {
|
||||
laptop = append(laptop, line)
|
||||
}
|
||||
if len(fields) > 0 && fields[0] == "anchor" {
|
||||
anchor = append(anchor, line)
|
||||
}
|
||||
}
|
||||
if len(laptop) != 1 || !strings.Contains(laptop[0], "2026-10-11 18:40") ||
|
||||
!strings.Contains(laptop[0], "by a one-shot push by jochen at a shell on anchor") ||
|
||||
!strings.Contains(laptop[0], "generation 40") || !strings.Contains(laptop[0], "applied") {
|
||||
t.Fatalf("status does not say laptop's last send in one line:\n%s", shown)
|
||||
}
|
||||
if len(anchor) != 1 || !strings.Contains(anchor[0], "no send recorded") {
|
||||
t.Fatalf("status does not say anchor has no send recorded, in one line:\n%s", shown)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatusSaysTheLastSendsCouldNotBeRead(t *testing.T) {
|
||||
asked := answers{nodes: []inventory.Node{{Name: "laptop", LastSeen: time.Now()}},
|
||||
lastSentUnread: "the store went away"}
|
||||
if shown := printed(t, func() error { return printStatus(asked) }); !strings.Contains(shown, "the store went away") {
|
||||
t.Fatalf("a record of sends that could not be read is not said:\n%s", shown)
|
||||
}
|
||||
if asked.well() {
|
||||
t.Error("a mesh whose sends could not be read is called well")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeShowSaysTheLastSendInFull(t *testing.T) {
|
||||
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.Local)
|
||||
refused := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeRefused,
|
||||
Refused: "unknown field \"generation\"\nsecond line", At: &at})), "\n")
|
||||
for _, want := range []string{"sequence 12", "2026-10-11 18:40:05", "a one-shot push by jochen at a shell on anchor",
|
||||
"lease epoch 57", "generation 40", "docker, sudo", "refused", "unknown field \"generation\""} {
|
||||
if !strings.Contains(refused, want) {
|
||||
t.Errorf("node show's last send does not say %q:\n%s", want, refused)
|
||||
}
|
||||
}
|
||||
if applied := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeApplied,
|
||||
At: &at})), "\n"); !strings.Contains(applied, "applied it at 2026-10-11 18:41:00") {
|
||||
t.Errorf("an applied send is not said applied:\n%s", applied)
|
||||
}
|
||||
if waiting := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{})), "\n"); !strings.Contains(waiting,
|
||||
"not reported on it yet") {
|
||||
t.Errorf("a send not reported on is not said so:\n%s", waiting)
|
||||
}
|
||||
stale := aLastSend(inventory.SendAnswer{})
|
||||
when := at
|
||||
stale.RefusedAt, stale.RefusedApplied = &when, 44
|
||||
if s := strings.Join(lastSendLines(stale), "\n"); !strings.Contains(s, "refused it at 2026-10-11 18:41:00") ||
|
||||
!strings.Contains(s, "generation 44") {
|
||||
t.Errorf("a send refused for its generation is not said so:\n%s", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatusJSONCarriesEachMachinesLastSend(t *testing.T) {
|
||||
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.UTC)
|
||||
asked := answers{nodes: []inventory.Node{{Name: "anchor"}, {Name: "laptop"}},
|
||||
lastSent: map[string]inventory.Send{"laptop": aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeRefused,
|
||||
Refused: "unknown field", At: &at})}}
|
||||
body, err := statusAsJSON(asked)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got struct {
|
||||
LastSent []map[string]any `json:"lastSent"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got.LastSent) != 1 {
|
||||
t.Fatalf("status --json carries %d last sends, want laptop's only:\n%s", len(got.LastSent), body)
|
||||
}
|
||||
s := got.LastSent[0]
|
||||
if s["node"] != "laptop" || s["sequence"] != float64(12) || s["epoch"] != float64(57) ||
|
||||
s["generation"] != float64(40) || s["sender"] != "a one-shot push by jochen at a shell on anchor" ||
|
||||
s["digest"] != "d12" || s["sentAt"] == nil || s["answer"] != "refused" || s["refused"] != "unknown field" ||
|
||||
s["answeredAt"] == nil {
|
||||
t.Fatalf("laptop's last send reads %v", s)
|
||||
}
|
||||
if modules, _ := s["modules"].([]any); len(modules) != 2 || modules[0] != "docker" {
|
||||
t.Fatalf("the modules it named read %v", s["modules"])
|
||||
}
|
||||
}
|
||||
|
||||
// And through the store: a send recorded and refused by the machine is what `node show` and `status --json`
|
||||
// read back.
|
||||
func TestNodeShowAndStatusReadTheLastSendFromTheStore(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
record, err := open.inventory.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSentWith(ctx, record.ID, "d12", nil, 57, 40, inventory.Send{Sequence: 12,
|
||||
Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12",
|
||||
Modules: []string{"docker", "sudo"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeRefused,
|
||||
Declared: "d12", Refused: "unknown field \"generation\""}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
shown := printed(t, func() error { return showNode(ctx, open, "laptop") })
|
||||
for _, want := range []string{"a one-shot push by jochen at a shell on anchor", "lease epoch 57", "generation 40",
|
||||
"docker, sudo", "unknown field \"generation\""} {
|
||||
if !strings.Contains(shown, want) {
|
||||
t.Errorf("node show laptop does not say %q:\n%s", want, shown)
|
||||
}
|
||||
}
|
||||
body := printed(t, func() error { return statusFor(ctx, open, true) })
|
||||
var got struct {
|
||||
LastSent []struct {
|
||||
Node string `json:"node"`
|
||||
Sender string `json:"sender"`
|
||||
Answer string `json:"answer"`
|
||||
} `json:"lastSent"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(body), &got); err != nil {
|
||||
t.Fatalf("%v:\n%s", err, body)
|
||||
}
|
||||
if len(got.LastSent) != 1 || got.LastSent[0].Node != "laptop" || got.LastSent[0].Answer != "refused" ||
|
||||
got.LastSent[0].Sender != "a one-shot push by jochen at a shell on anchor" {
|
||||
t.Fatalf("status --json's last sends read %+v", got.LastSent)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// A module assigned to a machine and left out of its declaration is said (novox/hq issue 380).
|
||||
//
|
||||
// **An omission is a finding, never a refusal of the push** (ADR 0163, rule 6): the machine is sent everything
|
||||
// else, and the module's held things are kept. Until issue 380 the only place it showed was `plan`: nfs-server was
|
||||
// assigned to the home server for days, every send left it out for a setting nobody gave, and the operator believed
|
||||
// it ran. So the self-check (D1) judges every machine as the next send would (Resolution.LeftOutBecause, the send's
|
||||
// own judgement) and raises a condition on that machine for each module it leaves out:
|
||||
//
|
||||
// - a setting nobody gave (catalogue.UnsetSettingError) is the operator's to give: kind needs-operator, naming
|
||||
// the module, the setting and the command that sets it;
|
||||
// - any other cause is said as a module not working is: kind left-out, a warning with the reason as evidence.
|
||||
//
|
||||
// Never urgent and never escalated by age (as ADR 0283 decision 5): nothing the machine ran was undone. It clears
|
||||
// on the first run that no longer finds it — the module composed again, or no longer assigned. `status` and
|
||||
// `node show` list the same modules under "assigned, not applied" with the same reason.
|
||||
//
|
||||
// A module left out because its stored manifest has a key this controller does not know is not raised here: the
|
||||
// catalogue's own unknown-field condition says it once for the whole mesh (ADR 0262); it is still listed.
|
||||
|
||||
const (
|
||||
// probeLeftOutID is the self-check's probe that raises and clears these conditions: D1, which already
|
||||
// resolves every machine (probeDeclarations), so the judgement costs no resolution of its own.
|
||||
probeLeftOutID = "D1"
|
||||
// kindLeftOut is a module left out for any cause but a setting nobody gave; it is also every such condition's
|
||||
// token, whichever its kind, so a cause that changes is the same condition said anew.
|
||||
kindLeftOut = "left-out"
|
||||
)
|
||||
|
||||
// leftOutModule is one module of a machine's set that its declaration leaves out, and why.
|
||||
type leftOutModule struct {
|
||||
Module string
|
||||
// Setting is the setting nobody gave, when that is the cause.
|
||||
Setting string
|
||||
// Why is the declaration's own reason, whole.
|
||||
Why string
|
||||
// Unread is a stored manifest this controller cannot read whole (said by the catalogue's condition).
|
||||
Unread bool
|
||||
}
|
||||
|
||||
// leftOutOf is every module of a machine's resolution that a push would leave out, sorted. Pure: the judgement
|
||||
// the push makes (catalogue.Resolution.LeftOutBecause), nothing allocated.
|
||||
func leftOutOf(plan catalogue.Resolution, settings catalogue.SettingsBy, adopted bool) []leftOutModule {
|
||||
because := plan.LeftOutBecause(settings, adopted)
|
||||
out := make([]leftOutModule, 0, len(because))
|
||||
for module, why := range because {
|
||||
l := leftOutModule{Module: module, Why: oneLine(why.Error())}
|
||||
var unset *catalogue.UnsetSettingError
|
||||
var unread *catalogue.UnreadManifestError
|
||||
switch {
|
||||
case errors.As(why, &unset):
|
||||
l.Setting = unset.Setting
|
||||
case errors.As(why, &unread):
|
||||
l.Unread = true
|
||||
}
|
||||
out = append(out, l)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module })
|
||||
return out
|
||||
}
|
||||
|
||||
// settingCommand is the command that gives a module's setting on one machine.
|
||||
func settingCommand(module, setting, node string) string {
|
||||
return fmt.Sprintf("`settings set %s '{%q: …}' --node %s`", module, setting, node)
|
||||
}
|
||||
|
||||
// reason is why a module is left out, as `status`, `node show` and the condition's summary say it: for a setting
|
||||
// nobody gave, the setting and the command that gives it; otherwise the declaration's own words.
|
||||
func (l leftOutModule) reason(node string) string {
|
||||
if l.Setting != "" {
|
||||
return fmt.Sprintf("nothing sets its setting %q, so every send leaves it out of its declaration — %s sets it", l.Setting,
|
||||
settingCommand(l.Module, l.Setting, node))
|
||||
}
|
||||
return "every send leaves it out of its declaration: " + l.Why
|
||||
}
|
||||
|
||||
// leftOutObservations are the conditions a machine's left-out modules raise, one each.
|
||||
func leftOutObservations(node string, left []leftOutModule) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, l := range left {
|
||||
if l.Unread {
|
||||
continue
|
||||
}
|
||||
out = append(out, leftOutObservation(node, l))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// leftOutObservation is one module left out of one machine's declaration: needs-operator for a setting nobody
|
||||
// gave, left-out otherwise; a warning either way, the operator's to resolve.
|
||||
func leftOutObservation(node string, l leftOutModule) conditions.Observation {
|
||||
o := conditions.Observation{Scope: conditions.ScopeModule, ID: l.Module + "." + node, Token: kindLeftOut,
|
||||
Kind: kindLeftOut, Machine: node, Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s is assigned to %s and not applied: %s", l.Module, node, l.reason(node)),
|
||||
Said: l.Why}
|
||||
w := leftOutWords(l.Module, node, l.Setting)
|
||||
if l.Setting != "" {
|
||||
o.Kind = kindNeedsOperator
|
||||
} else {
|
||||
// The words of why may name a path or an address, which the operator's channel withholds: the
|
||||
// summary sends them to the evidence, and `plan` says them in full.
|
||||
o.Summary = fmt.Sprintf("%s is assigned to %s and not applied: every send leaves it out of its declaration, "+
|
||||
"because what is set for it does not fit its manifest — the evidence and `plan %s` say why", l.Module, node, node)
|
||||
}
|
||||
o.Headline, o.Explanation, o.Needs, o.Resolved = w.Headline, w.Explanation, w.Needs, w.Resolved
|
||||
return o
|
||||
}
|
||||
|
||||
// leftOutWords is what the operator reads of a module left out (ADR 0253): plain, the act named.
|
||||
func leftOutWords(module, node, setting string) words {
|
||||
w := words{
|
||||
Headline: fmt.Sprintf("%s is not applied on %s", module, node),
|
||||
Explanation: fmt.Sprintf("%s is assigned to %s, and every send to %s leaves it out of the declaration because what "+
|
||||
"is set for it does not fit its manifest. Nothing of it changes there; the rest of %s is sent as usual.",
|
||||
module, node, node, node),
|
||||
Needs: fmt.Sprintf("read why in the details, then change what is set for %s or unassign it.", module),
|
||||
Resolved: fmt.Sprintf("%s on %s is no longer left out", module, node),
|
||||
}
|
||||
if setting != "" {
|
||||
w.Explanation = fmt.Sprintf("%s is assigned to %s, and every send to %s leaves it out of the declaration because "+
|
||||
"nothing sets its setting %s. Nothing of it runs there until it is set; the rest of %s is sent as usual.",
|
||||
module, node, node, setting, node)
|
||||
w.Needs = fmt.Sprintf("set %s for %s on %s, or approve it when it is proposed to you.", setting, module, node)
|
||||
// A setting's name that is not plain (a dotted key) is in the summary instead.
|
||||
if _, ok := conditions.PlainWords(w, node); !ok {
|
||||
w.Explanation = fmt.Sprintf("%s is assigned to %s, and every send to %s leaves it out of the declaration because a "+
|
||||
"setting it needs is not set. Nothing of it runs there until it is set; the details name it.",
|
||||
module, node, node)
|
||||
w.Needs = fmt.Sprintf("set what %s needs on %s; the details name the setting.", module, node)
|
||||
}
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// notAppliedLines is a machine's "assigned, not applied" as `node show` prints it: one line a module, with the
|
||||
// reason its condition says.
|
||||
func notAppliedLines(node string, left []leftOutModule) []string {
|
||||
if len(left) == 0 {
|
||||
return nil
|
||||
}
|
||||
lines := []string{"", " assigned, not applied:"}
|
||||
for _, l := range left {
|
||||
lines = append(lines, fmt.Sprintf(" %-22s %s", l.Module, l.reason(node)))
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
// machineNotApplied is one module assigned to a machine and left out of its declaration, in `status --json`.
|
||||
type machineNotApplied struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Setting string `json:"setting,omitempty"`
|
||||
Reason string `json:"reason"`
|
||||
// UnreadManifest is a module left out because this controller cannot read its manifest whole: it raises no
|
||||
// condition of its own on the machine, since the catalogue's unknown-field condition says it once (ADR 0262).
|
||||
UnreadManifest bool `json:"unread-manifest,omitempty"`
|
||||
}
|
||||
|
||||
// notApplied is every machine's left-out modules, in a stated order, as `status --json` carries them.
|
||||
func notApplied(left map[string][]leftOutModule) []machineNotApplied {
|
||||
names := make([]string, 0, len(left))
|
||||
for name := range left {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
var out []machineNotApplied
|
||||
for _, name := range names {
|
||||
for _, l := range left[name] {
|
||||
out = append(out, machineNotApplied{Node: name, Module: l.Module, Setting: l.Setting, Reason: l.reason(name),
|
||||
UnreadManifest: l.Unread})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// printNotApplied is status's "assigned, not applied", every machine's.
|
||||
func printNotApplied(left map[string][]leftOutModule) {
|
||||
rows := notApplied(left)
|
||||
if len(rows) == 0 {
|
||||
return
|
||||
}
|
||||
unread := 0
|
||||
for _, r := range rows {
|
||||
if r.UnreadManifest {
|
||||
unread++
|
||||
}
|
||||
}
|
||||
// Which raise a condition is said, not implied (review of #223): one whose manifest this controller cannot
|
||||
// read is listed here and raises none of its own on the machine — the catalogue's condition says it.
|
||||
raises := "each raises a condition on its machine"
|
||||
switch {
|
||||
case unread == len(rows):
|
||||
raises = "none raises a condition on its machine: this controller cannot read their manifests, which the " +
|
||||
"catalogue's own condition says"
|
||||
case unread > 0:
|
||||
raises += fmt.Sprintf(", except the %d whose manifest this controller cannot read, which the catalogue's own "+
|
||||
"condition says", unread)
|
||||
}
|
||||
fmt.Printf("%d module(s) assigned, not applied — every send leaves them out of their declaration; %s:\n",
|
||||
len(rows), raises)
|
||||
for _, r := range rows {
|
||||
fmt.Printf(" %-12s %-22s %s\n", r.Node, r.Module, r.Reason)
|
||||
}
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
// isLeftOutCondition is whether a condition is a module left out of its declaration, which the machine's health
|
||||
// statements neither raise nor clear: by its token, which every one carries whatever its kind.
|
||||
func isLeftOutCondition(c conditions.Condition) bool {
|
||||
return c.Subject.Scope == conditions.ScopeModule && strings.HasSuffix(c.Key, "."+kindLeftOut)
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// novox/hq issue 380: nfs-server was assigned to the home server for days and every send left it out — "nfs-server
|
||||
// has a file that says ${setting:shares}, and nothing sets shares for it" — and nothing but `plan` said so. The
|
||||
// manifest is the catalogue's own at the commit that added it (mesh-catalog 48fba44), which still says
|
||||
// ${setting:shares}; the reason below is the one the live push printed.
|
||||
|
||||
// leftOutNFS is the resolution of a machine assigned that nfs-server, with the settings given.
|
||||
func leftOutNFS(t *testing.T) catalogue.Resolution {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile("testdata/left-out/nfs-server.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return catalogue.Resolution{Modules: []catalogue.Manifest{m}}
|
||||
}
|
||||
|
||||
// sharesGiven is the operator's setting for it on the machine.
|
||||
func sharesGiven(value string) catalogue.SettingsBy {
|
||||
return catalogue.SettingsBy{"nfs-server": {{From: "anchor", Values: map[string]any{"shares": value}}}}
|
||||
}
|
||||
|
||||
func TestAModuleLeftOutForASettingNobodyGaveNeedsTheOperatorNamingTheSettingAndTheCommand(t *testing.T) {
|
||||
left := leftOutOf(leftOutNFS(t), nil, false)
|
||||
if len(left) != 1 || left[0].Module != "nfs-server" || left[0].Setting != "shares" {
|
||||
t.Fatalf("left out: %+v", left)
|
||||
}
|
||||
if !strings.Contains(left[0].Why, `nothing sets "shares" for it`) {
|
||||
t.Fatalf("the reason is not the push's own: %s", left[0].Why)
|
||||
}
|
||||
obs := leftOutObservations("anchor", left)
|
||||
if len(obs) != 1 {
|
||||
t.Fatalf("raised %+v", obs)
|
||||
}
|
||||
o := obs[0]
|
||||
if o.Key() != "module.nfs-server.anchor.left-out" || o.Kind != kindNeedsOperator || o.Machine != "anchor" ||
|
||||
o.Severity != conditions.Warning || o.Resolver != conditions.ResolverOperator {
|
||||
t.Fatalf("raised %s as %s, %s, by %s, on %q", o.Key(), o.Kind, o.Severity, o.Resolver, o.Machine)
|
||||
}
|
||||
for _, want := range []string{"nfs-server", "anchor", `"shares"`, "`settings set nfs-server '{\"shares\": …}' --node anchor`"} {
|
||||
if !strings.Contains(o.Summary, want) {
|
||||
t.Errorf("the summary does not name %s: %s", want, o.Summary)
|
||||
}
|
||||
}
|
||||
if o.Said != left[0].Why {
|
||||
t.Errorf("the evidence is not the reason the send gives: %s", o.Said)
|
||||
}
|
||||
plainExample(t, o, "nfs-server is not applied on anchor",
|
||||
"Needs you: set shares for nfs-server on anchor, or approve it when it is proposed to you. nfs-server is assigned to "+
|
||||
"anchor, and every send to anchor leaves it out of the declaration because nothing sets its setting shares. "+
|
||||
"Nothing of it runs there until it is set; the rest of anchor is sent as usual.")
|
||||
}
|
||||
|
||||
func TestAModuleLeftOutForAnotherCauseIsAWarningWithTheReasonAsEvidence(t *testing.T) {
|
||||
// A setting stored that its manifest can no longer take: one with a line break (issue 339).
|
||||
left := leftOutOf(leftOutNFS(t), sharesGiven("library=/srv/library\nmedia=/srv/media"), false)
|
||||
if len(left) != 1 || left[0].Setting != "" {
|
||||
t.Fatalf("left out: %+v", left)
|
||||
}
|
||||
obs := leftOutObservations("anchor", left)
|
||||
if len(obs) != 1 {
|
||||
t.Fatalf("raised %+v", obs)
|
||||
}
|
||||
o := obs[0]
|
||||
if o.Key() != "module.nfs-server.anchor.left-out" || o.Kind != kindLeftOut || o.Severity != conditions.Warning {
|
||||
t.Fatalf("raised %s as %s, %s", o.Key(), o.Kind, o.Severity)
|
||||
}
|
||||
if !strings.Contains(o.Said, "holds a line break") || strings.Contains(o.Summary, "/srv/") {
|
||||
t.Fatalf("the reason is not the evidence, or the summary carries it to the channel:\n%s\n%s", o.Summary, o.Said)
|
||||
}
|
||||
plainExample(t, o, "nfs-server is not applied on anchor",
|
||||
"Needs you: read why in the details, then change what is set for nfs-server or unassign it. nfs-server is assigned to "+
|
||||
"anchor, and every send to anchor leaves it out of the declaration because what is set for it does not fit "+
|
||||
"its manifest. Nothing of it changes there; the rest of anchor is sent as usual.")
|
||||
}
|
||||
|
||||
// The self-check raises it, keeps it a warning however long it stands, and clears it when the module composes
|
||||
// again or is no longer assigned; a machine's health statement neither clears nor raises it.
|
||||
func TestALeftOutModulesConditionClearsWhenItComposesAgainOrIsUnassigned(t *testing.T) {
|
||||
plan, settings := leftOutNFS(t), catalogue.SettingsBy(nil)
|
||||
withProbes(t, probe{ID: probeLeftOutID, Asserts: "the test's", Kind: kindLeftOut, Phase: 1,
|
||||
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
|
||||
return leftOutObservations("anchor", leftOutOf(plan, settings, false)), nil
|
||||
}})
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||
defer k.Close(context.Background())
|
||||
d := &doctor{keeper: k, teller: &conditions.Told{}, host: "anchor"}
|
||||
key := "module.nfs-server.anchor.left-out"
|
||||
openOnes := func() map[string]conditions.Condition {
|
||||
t.Helper()
|
||||
open, err := k.Open(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := map[string]conditions.Condition{}
|
||||
for _, c := range open {
|
||||
out[c.Key] = c
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
d.runOnce(t.Context(), "a test")
|
||||
c, raised := openOnes()[key]
|
||||
if !raised || c.Kind != kindNeedsOperator || c.Severity != conditions.Warning {
|
||||
t.Fatalf("a module left out raised %+v", openOnes())
|
||||
}
|
||||
// A statement from the machine that says nothing of it — the module runs nothing there — leaves it open.
|
||||
if err := judgeModuleHealth(t.Context(), nil, k, "anchor", map[string][]inventory.ResourceHealth{}, nil,
|
||||
time.Now().Add(72*time.Hour)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, still := openOnes()[key]; !still {
|
||||
t.Fatal("a health statement that says nothing of the module cleared its left-out condition")
|
||||
}
|
||||
d.runOnce(t.Context(), "a test")
|
||||
if c := openOnes()[key]; c.Severity != conditions.Warning {
|
||||
t.Fatalf("after a health statement and days, it is %+v", openOnes())
|
||||
}
|
||||
|
||||
// The setting given: it composes, and the condition clears.
|
||||
settings = sharesGiven("library=/srv/library")
|
||||
d.runOnce(t.Context(), "a test")
|
||||
if _, still := openOnes()[key]; still {
|
||||
t.Fatalf("composed again, still open: %+v", openOnes())
|
||||
}
|
||||
|
||||
// Left out again, then unassigned: no longer in the machine's set, and it clears.
|
||||
settings = nil
|
||||
d.runOnce(t.Context(), "a test")
|
||||
if _, raised := openOnes()[key]; !raised {
|
||||
t.Fatal("left out again and not raised")
|
||||
}
|
||||
plan = catalogue.Resolution{}
|
||||
d.runOnce(t.Context(), "a test")
|
||||
if _, still := openOnes()[key]; still {
|
||||
t.Fatalf("unassigned, still open: %+v", openOnes())
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheLeftOutProbeIsInTheRegistry(t *testing.T) {
|
||||
for _, p := range probeRegistry {
|
||||
if p.ID == probeLeftOutID {
|
||||
if p.run == nil || !slices.Contains(p.Raises, kindLeftOut) || !slices.Contains(p.Raises, kindNeedsOperator) {
|
||||
t.Fatalf("%+v", p)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("no probe %s: a module left out is said nowhere", probeLeftOutID)
|
||||
}
|
||||
|
||||
// status and node show list it under "assigned, not applied" with the reason the condition says, and status is
|
||||
// not well while there is one.
|
||||
func TestStatusAndNodeListAModuleAssignedAndNotApplied(t *testing.T) {
|
||||
left := map[string][]leftOutModule{"anchor": leftOutOf(leftOutNFS(t), nil, false)}
|
||||
reason := leftOutObservations("anchor", left["anchor"])[0].Summary
|
||||
asked := answers{leftOut: left}
|
||||
if asked.well() {
|
||||
t.Fatal("a mesh with a module assigned and not applied is called well")
|
||||
}
|
||||
shown := printed(t, func() error { return printStatus(asked) })
|
||||
if !strings.Contains(shown, "1 module(s) assigned, not applied") || !strings.Contains(shown, "nfs-server") ||
|
||||
!strings.Contains(shown, "`settings set nfs-server '{\"shares\": …}' --node anchor` sets it") {
|
||||
t.Fatalf("status says:\n%s", shown)
|
||||
}
|
||||
if !strings.Contains(reason, left["anchor"][0].reason("anchor")) {
|
||||
t.Fatalf("status and the condition say different reasons:\n%s\n%s", shown, reason)
|
||||
}
|
||||
body, err := statusAsJSON(asked)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var doc struct {
|
||||
NotApplied []machineNotApplied `json:"not-applied"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &doc); err != nil || len(doc.NotApplied) != 1 ||
|
||||
doc.NotApplied[0].Setting != "shares" || doc.NotApplied[0].Node != "anchor" {
|
||||
t.Fatalf("status --json: %v %s", err, body)
|
||||
}
|
||||
lines := strings.Join(notAppliedLines("anchor", left["anchor"]), "\n")
|
||||
if !strings.Contains(lines, "assigned, not applied:") || !strings.Contains(lines, "nfs-server") ||
|
||||
!strings.Contains(lines, `nothing sets its setting "shares"`) {
|
||||
t.Fatalf("node show says:\n%s", lines)
|
||||
}
|
||||
}
|
||||
|
||||
// The skip is this probe's alone (review of #223): a part waiting for the operator (ADR 0283) keeps its own
|
||||
// needs-operator condition, `module.<m>.<node>.needs-operator`, which still clears on the first statement that no
|
||||
// longer names it — beside a left-out condition on the same machine, which stays.
|
||||
func TestAWaitsNeedsOperatorStillClearsWhenItsStatementStopsNamingItBesideALeftOutOne(t *testing.T) {
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||
defer k.Close(context.Background())
|
||||
left := leftOutObservations("anchor", leftOutOf(leftOutNFS(t), nil, false))
|
||||
if err := k.Reconcile(t.Context(), probeLeftOutID, left); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waiting := map[string][]inventory.ResourceHealth{"notes": {{Module: "notes", Resource: "server", State: link.StateWaiting,
|
||||
Waits: []inventory.Wait{{Setting: "domain", What: "the domain it serves"}}}}}
|
||||
if err := judgeModuleHealth(t.Context(), nil, k, "anchor", waiting, map[string]int{"notes": 2}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitKey, leftKey := needsOperatorKey("notes", "anchor"), "module.nfs-server.anchor.left-out"
|
||||
open := func() map[string]conditions.Condition {
|
||||
t.Helper()
|
||||
list, err := k.Open(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := map[string]conditions.Condition{}
|
||||
for _, c := range list {
|
||||
out[c.Key] = c
|
||||
}
|
||||
return out
|
||||
}
|
||||
if c, raised := open()[waitKey]; !raised || c.Kind != kindNeedsOperator {
|
||||
t.Fatalf("the wait raised %+v", open())
|
||||
}
|
||||
if c := open()[leftKey]; c.Kind != kindNeedsOperator {
|
||||
t.Fatalf("the left-out condition is not open as needs-operator: %+v", open())
|
||||
}
|
||||
// D1 runs again and still finds nfs-server left out: its reconcile clears only what D1 raised, never the wait,
|
||||
// which the machine's statement raised.
|
||||
if err := k.Reconcile(t.Context(), probeLeftOutID, left); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, still := open()[waitKey]; !still {
|
||||
t.Fatalf("D1's reconcile cleared the wait's needs-operator: %+v", open())
|
||||
}
|
||||
if err := judgeModuleHealth(t.Context(), nil, k, "anchor", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, still := open()[waitKey]; still {
|
||||
t.Fatalf("the statement no longer names the wait, and its needs-operator is still open: %+v", open())
|
||||
}
|
||||
if _, still := open()[leftKey]; !still {
|
||||
t.Fatalf("the left-out condition was cleared by a health statement: %+v", open())
|
||||
}
|
||||
}
|
||||
@@ -147,9 +147,11 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
}
|
||||
standing := map[string]conditions.Condition{}
|
||||
for _, c := range open {
|
||||
// A module left out of the composition is the self-check's to raise and clear, never a statement's
|
||||
// (novox/hq issue 380): its needs-operator is not cleared for not being in what the machine runs.
|
||||
if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound ||
|
||||
c.Kind == kindNeedsOperator) &&
|
||||
c.Subject.Machine == node {
|
||||
c.Subject.Machine == node && !isLeftOutCondition(c) {
|
||||
standing[c.Key] = c
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,7 +44,7 @@ func nodeCommand(ctx context.Context, args []string) error {
|
||||
if len(args) != 2 {
|
||||
return errors.New("node show <name>")
|
||||
}
|
||||
return showNode(ctx, inv, args[1])
|
||||
return showNode(ctx, open, args[1])
|
||||
case "add":
|
||||
return addNode(ctx, inv, args[1:])
|
||||
|
||||
@@ -787,7 +787,8 @@ func roughly(d time.Duration) string {
|
||||
//
|
||||
// It is also where "what should it be configured as" is read. The same line that gates an
|
||||
// assignment carries `card1-DP-1`, and a person composing settings for that machine needs it.
|
||||
func showNode(ctx context.Context, inv *inventory.Inventory, name string) error {
|
||||
func showNode(ctx context.Context, stored *stores, name string) error {
|
||||
inv := stored.inventory
|
||||
node, err := inv.NodeByName(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -803,6 +804,19 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
|
||||
if err := showMode(ctx, inv, node); err != nil {
|
||||
return err
|
||||
}
|
||||
// What it was last sent, by whom, under which epoch, from which generation, naming which modules, and what
|
||||
// it answered (novox/hq issue 485): the record of sends is read through this verb, never the store's reader.
|
||||
// Unreadable is said, not taken for none.
|
||||
switch last, found, err := inv.LastSend(ctx, name); {
|
||||
case err != nil:
|
||||
fmt.Printf(" what it was last sent could NOT be read: %v\n", err)
|
||||
case !found:
|
||||
fmt.Printf(" last sent no send recorded since the mesh began keeping them\n")
|
||||
default:
|
||||
for _, line := range lastSendLines(last) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
}
|
||||
// Whom agents run as here, and whether that account can become root without a person (ADR 0266).
|
||||
for _, line := range agentAccountLines(ctx, inv, node) {
|
||||
fmt.Println(line)
|
||||
@@ -889,6 +903,17 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
|
||||
if len(assigned) > 0 {
|
||||
fmt.Printf("\n assigned: %s\n", strings.Join(assigned, ", "))
|
||||
}
|
||||
// And which of them a push leaves out, and why (novox/hq issue 380): judged as the push judges it, the same
|
||||
// reason its condition says. Not computable is said, never read as "all applied".
|
||||
plan, settings, err := planFor(ctx, stored, name)
|
||||
switch {
|
||||
case err != nil:
|
||||
fmt.Printf("\n whether a send leaves any assigned module out is NOT known: %s\n", oneLine(err.Error()))
|
||||
default:
|
||||
for _, line := range notAppliedLines(name, leftOutOf(plan, settings, node.Adopted)) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -17,8 +17,8 @@ import (
|
||||
|
||||
// A delivery over its budget is loud (novox/hq ADR 0282 decision 7, issue 382): the self-check reads
|
||||
// mesh-delivery's `times` and raises the warning `delivery.<class>.over-budget` when the newest delivery of a
|
||||
// class whose delivery time is known took longer than its class's budget — five minutes for a leaf module, ten
|
||||
// for a core module — naming the delivery and its longest phase. It clears when the next delivery of that class
|
||||
// class whose delivery time is known took longer than its class's budget — three minutes for a leaf module (ADR
|
||||
// 0304 decision 4), ten for a core module — naming the delivery and its longest phase. It clears when the next delivery of that class
|
||||
// lands within its budget. Measurement only: the condition says, it never holds or acts on a delivery.
|
||||
|
||||
// probeBudgetsID is the probe that reads the delivery times.
|
||||
@@ -27,8 +27,9 @@ const probeBudgetsID = "D16"
|
||||
// kindOverBudget is what a delivery over its class's budget raises.
|
||||
const kindOverBudget = "over-budget"
|
||||
|
||||
// deliveryBudgets are the budgets by class (ADR 0282 decision 1); the probe raises nothing for another class.
|
||||
var deliveryBudgets = map[string]time.Duration{inventory.ClassLeaf: 5 * time.Minute, inventory.ClassCore: 10 * time.Minute}
|
||||
// deliveryBudgets are the budgets by class (ADR 0282 decision 1, the leaf's as ADR 0304 decision 4 changed it);
|
||||
// the probe raises nothing for another class.
|
||||
var deliveryBudgets = map[string]time.Duration{inventory.ClassLeaf: 3 * time.Minute, inventory.ClassCore: 10 * time.Minute}
|
||||
|
||||
// timesAnswer is what mesh-delivery's `times` answers, as far as the probe reads it.
|
||||
type timesAnswer struct {
|
||||
|
||||
@@ -10,11 +10,12 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A leaf delivery of 5 minutes 10 seconds raises delivery.leaf.over-budget naming its longest phase; one of
|
||||
// exactly five minutes, a core one of nine and a class without a budget raise nothing (ADR 0282 decision 7).
|
||||
// A leaf delivery of 3 minutes 10 seconds raises delivery.leaf.over-budget naming its longest phase; one of
|
||||
// exactly three minutes, a core one of nine and a class without a budget raise nothing (ADR 0282 decision 7,
|
||||
// the leaf's budget as ADR 0304 decision 4 changed it).
|
||||
func TestADeliveryOverItsBudgetIsLoud(t *testing.T) {
|
||||
a := ×Answer{Classes: []timesClass{
|
||||
{Class: inventory.ClassLeaf, Latest: ×Latest{ID: "novox/mesh-catalog@abc", TookMS: (5*time.Minute + 10*time.Second).Milliseconds(),
|
||||
{Class: inventory.ClassLeaf, Latest: ×Latest{ID: "novox/mesh-catalog@abc", TookMS: (3*time.Minute + 10*time.Second).Milliseconds(),
|
||||
Longest: "judgement 2m40s"}},
|
||||
{Class: inventory.ClassCore, Latest: ×Latest{ID: "novox/mesh-controller@def", TookMS: (9 * time.Minute).Milliseconds(),
|
||||
Longest: "build 1m"}},
|
||||
@@ -30,7 +31,7 @@ func TestADeliveryOverItsBudgetIsLoud(t *testing.T) {
|
||||
t.Fatalf("the condition names its delivery and longest phase: %s — %s", o.Key(), o.Summary)
|
||||
}
|
||||
// The next leaf delivery within its budget clears it: the probe raises nothing for the class.
|
||||
a.Classes[0].Latest = ×Latest{ID: "novox/mesh-catalog@ghi", TookMS: (5 * time.Minute).Milliseconds()}
|
||||
a.Classes[0].Latest = ×Latest{ID: "novox/mesh-catalog@ghi", TookMS: (3 * time.Minute).Milliseconds()}
|
||||
if obs := overBudgetObservations(a); len(obs) != 0 {
|
||||
t.Fatalf("a delivery of exactly its budget is within it: %+v", obs)
|
||||
}
|
||||
|
||||
@@ -231,6 +231,14 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
||||
w := needsOperatorWords(orModule(module), node, nil)
|
||||
return w
|
||||
}),
|
||||
kindLeftOut: worded(func(o conditions.Observation) words {
|
||||
// The observation carries its own words (novox/hq issue 380); these are its kind's alone.
|
||||
module := ""
|
||||
if o.Scope == conditions.ScopeModule && o.Machine != "" {
|
||||
module = strings.TrimSuffix(o.ID, "."+o.Machine)
|
||||
}
|
||||
return leftOutWords(orModule(module), machineOr(o, "a machine"), "")
|
||||
}),
|
||||
kindProviderFailing: worded(func(o conditions.Observation) words {
|
||||
thing, consumer := conditions.ThingWords(o), idPart(o, 2)
|
||||
if consumer == "" {
|
||||
|
||||
@@ -278,6 +278,13 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
||||
plans = append(plans, recent...)
|
||||
// **A failed walk whose earlier merges are walked alone is not retried** (novox/hq ADR 0276): the search for
|
||||
// the merge that brought the failure answers them now, and a retried walk would name them twice.
|
||||
// **A walk stopped without some merges is not retried** (novox/hq ADR 0299): the merges it left out were stopped,
|
||||
// and the others are another walk's now.
|
||||
if p.Delivery != nil && (p.Delivery.WalkedAgainBy != "" || len(p.Delivery.LeftOut) > 0) {
|
||||
return "", fmt.Errorf("%s was stopped without %s: those merges were stopped, and %s walks the others; a "+
|
||||
"newer merge, or `rebuild <module>`, builds again", p.ID, mergesLeftOutWords(p.Delivery.LeftOut),
|
||||
orWords(p.Delivery.WalkedAgainBy, "nothing"))
|
||||
}
|
||||
if p.Delivery != nil && len(p.Delivery.Merges) > 0 {
|
||||
kept, err := inv.MergesOf(ctx, p.ID)
|
||||
if err != nil {
|
||||
|
||||
@@ -78,6 +78,11 @@ func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation
|
||||
return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err)
|
||||
}
|
||||
var problems, foreseen []string
|
||||
// Each module the next send leaves out of this machine's declaration (novox/hq issue 380), judged from this
|
||||
// resolution as the send judges it: a finding, never a refusal.
|
||||
if err == nil {
|
||||
out = append(out, leftOutObservations(n.Name, leftOutOf(plan, settings, n.Adopted))...)
|
||||
}
|
||||
if err == nil && gensErr == nil {
|
||||
var declared sendable
|
||||
if declared, err = declarationWith(ctx, open, n.Name, plan, settings, gens, Foreseeing); err == nil {
|
||||
|
||||
@@ -1236,6 +1236,17 @@ func sendToEach(ctx context.Context, open *stores, names []string) ([]string, er
|
||||
continue
|
||||
}
|
||||
numbered.stamp(&declared)
|
||||
// **The grants step sends the user list and nothing else** (novox/hq issue 490): judged on this very
|
||||
// declaration, the one sent, so nothing composed between a check and the send can slip through.
|
||||
if everyBuildKept(ctx, name) {
|
||||
other, err := grantsOnlyIn(ctx, open, name, plan, declared)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if other != "" {
|
||||
return nil, fmt.Errorf("%w: %s", errNotGrantsOnly, other)
|
||||
}
|
||||
}
|
||||
reportLeftOut(name, declared)
|
||||
sending = append(sending, readyNode{name, declared})
|
||||
}
|
||||
|
||||
@@ -69,6 +69,9 @@ type meshStatus struct {
|
||||
// **A document without this said an outage was a well mesh.** Read from what each machine
|
||||
// reported, so it is the machine's account and not the mesh's take-time listing.
|
||||
Untaken []machineUntaken `json:"untaken,omitempty"`
|
||||
// NotApplied is every module assigned to a machine and left out of its composition, with why (novox/hq
|
||||
// issue 380). Absent when every module composes.
|
||||
NotApplied []machineNotApplied `json:"not-applied,omitempty"`
|
||||
// Filtered is every converged machine that is not filtered by the mesh alone (novox/hq ADR
|
||||
// 0168), one entry per rule set the mesh did not write — the found firewall in force again,
|
||||
// or a chain nobody speaks for. Absent when every converged machine is filtered by the mesh
|
||||
@@ -106,6 +109,33 @@ type meshStatus struct {
|
||||
// why they could not be read.
|
||||
Pending []pendingStatus `json:"pending,omitempty"`
|
||||
PendingUnread string `json:"pendingUnread,omitempty"`
|
||||
// LastSent is every machine's last recorded send — who sent it, under which lease epoch, from which
|
||||
// assignment generation, naming which modules — and what the machine answered of it (novox/hq issue 485).
|
||||
// A machine the mesh has recorded no send to is absent. LastSentUnread says why it could not be read.
|
||||
LastSent []machineLastSent `json:"lastSent"`
|
||||
LastSentUnread string `json:"lastSentUnread,omitempty"`
|
||||
}
|
||||
|
||||
// machineLastSent is one machine's last send as status says it.
|
||||
type machineLastSent struct {
|
||||
Node string `json:"node"`
|
||||
Sequence int64 `json:"sequence"`
|
||||
SentAt time.Time `json:"sentAt"`
|
||||
Sender string `json:"sender"`
|
||||
Epoch int64 `json:"epoch"`
|
||||
Generation int64 `json:"generation"`
|
||||
Digest string `json:"digest"`
|
||||
Modules []string `json:"modules"`
|
||||
// Answer is what the machine said of it: applied, failed or refused, as its report has it; empty when it
|
||||
// has not reported on it. Refused is its words when it refused it whole, Failed how many resources failed.
|
||||
Answer string `json:"answer"`
|
||||
Refused string `json:"refused,omitempty"`
|
||||
Failed int `json:"failed,omitempty"`
|
||||
AnsweredAt *time.Time `json:"answeredAt,omitempty"`
|
||||
// RefusedForGeneration is when the machine refused it for coming from an older assignment generation than
|
||||
// it applied, and AppliedGeneration that generation; absent when it did not.
|
||||
RefusedForGeneration *time.Time `json:"refusedForGeneration,omitempty"`
|
||||
AppliedGeneration int64 `json:"appliedGeneration,omitempty"`
|
||||
}
|
||||
|
||||
// pendingStatus is one pending assignment as status says it.
|
||||
@@ -251,6 +281,7 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
||||
}
|
||||
}
|
||||
out.Unheld = asked.unheld
|
||||
out.NotApplied = notApplied(asked.leftOut)
|
||||
out.HandActsThisWeek, out.HandActsUnread = asked.handActs, asked.handActsUnread
|
||||
out.HealsThisWeek, out.HealsUnread = asked.heals, asked.healsUnread
|
||||
// In brief, as `conditions` lists them: status leads with every open condition, and their whole
|
||||
@@ -263,6 +294,7 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
||||
out.Pending = append(out.Pending, pendingStatus{Node: p.Node, Module: p.Module, State: p.State,
|
||||
Build: p.Build, Repository: p.Repository, Path: p.Path, Since: p.Since, Settled: p.Settled, Note: p.Note})
|
||||
}
|
||||
out.LastSent, out.LastSentUnread = lastSentStatus(asked.lastSent), asked.lastSentUnread
|
||||
for name := range asked.refused {
|
||||
out.Unresolved = append(out.Unresolved, machineUnresolved{
|
||||
Node: name, Problem: asked.refused[name]})
|
||||
@@ -319,3 +351,20 @@ func say(value any) error {
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
|
||||
// lastSentStatus is every machine's last send as status --json says it, by machine name (novox/hq issue 485).
|
||||
func lastSentStatus(sends map[string]inventory.Send) []machineLastSent {
|
||||
out := []machineLastSent{}
|
||||
for _, s := range sends {
|
||||
modules := s.Modules
|
||||
if modules == nil {
|
||||
modules = []string{}
|
||||
}
|
||||
out = append(out, machineLastSent{Node: s.NodeName, Sequence: s.Sequence, SentAt: s.SentAt, Sender: s.Sender,
|
||||
Epoch: s.Epoch, Generation: s.Generation, Digest: s.Digest, Modules: modules, Answer: s.Answer.Outcome,
|
||||
Refused: s.Answer.Refused, Failed: s.Answer.Failed, AnsweredAt: s.Answer.At,
|
||||
RefusedForGeneration: s.RefusedAt, AppliedGeneration: s.RefusedApplied})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Node < out[j].Node })
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -61,20 +61,57 @@ func keptExcept(ctx context.Context) (map[string]bool, bool) {
|
||||
return skip, on
|
||||
}
|
||||
|
||||
type keepEveryBuildKey struct{}
|
||||
|
||||
// keepingEveryBuild is a context whose sends compose every module of one machine — recorded or rolling out —
|
||||
// at the build that machine was last sent (novox/hq issue 490): the grants step, which sends the machine
|
||||
// holding the bus its new user list and nothing of any module's new code. That code waits there for a gate
|
||||
// like any other move; the list must not, or the first machine's gate is judged against a bus that refuses
|
||||
// what the change newly grants.
|
||||
//
|
||||
// **That machine alone.** Composing one machine resolves the others too — who is on the private network,
|
||||
// who answers a requirement — on the same context, and those are no part of the step's send: they are
|
||||
// composed as any send composes them. Kept for every machine, a machine whose last send is not known
|
||||
// refused the bus's machine's composition.
|
||||
func keepingEveryBuild(ctx context.Context, node string) context.Context {
|
||||
return context.WithValue(ctx, keepEveryBuildKey{}, node)
|
||||
}
|
||||
|
||||
// everyBuildKept is whether this context keeps every module of this machine at the build it runs.
|
||||
func everyBuildKept(ctx context.Context, node string) bool {
|
||||
on, _ := ctx.Value(keepEveryBuildKey{}).(string)
|
||||
return on != "" && on == node
|
||||
}
|
||||
|
||||
// recordedKept is, for a send under keepingRecorded, every recorded module the machine was last sent a
|
||||
// build of that the mesh's build is not identical to: module → the commit it keeps. Nil when the context
|
||||
// keeps nothing, or when what the machine was last sent is not known (it is then held whole elsewhere —
|
||||
// ADR 0221).
|
||||
//
|
||||
// Under keepingEveryBuild, every module the machine was sent is kept, whatever its policy (novox/hq issue
|
||||
// 490), and a machine whose last send is not known refuses the send: there is nothing to keep it at, and
|
||||
// composing the mesh's builds would be the very move the grants step must not make.
|
||||
func recordedKept(ctx context.Context, open *stores, node string) (map[string]string, error) {
|
||||
every := everyBuildKept(ctx, node)
|
||||
skip, on := keptExcept(ctx)
|
||||
if !on {
|
||||
if !on && !every {
|
||||
return nil, nil
|
||||
}
|
||||
if every {
|
||||
skip = nil
|
||||
}
|
||||
inv := open.inventory
|
||||
sent, known, err := inv.SentBuilds(ctx, node)
|
||||
if err != nil || !known {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !known {
|
||||
if every {
|
||||
return nil, fmt.Errorf("what %s was last sent is not known, so the bus's user list cannot be sent "+
|
||||
"there alone with every build kept (novox/hq issue 490)", node)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -83,7 +120,7 @@ func recordedKept(ctx context.Context, open *stores, node string) (map[string]st
|
||||
out := map[string]string{}
|
||||
for m, was := range sent {
|
||||
now, held := current[m]
|
||||
if !held || now.RollOut || skip[m] || was == "" || sameCommit(was, now.Commit) {
|
||||
if !held || (now.RollOut && !every) || skip[m] || was == "" || sameCommit(was, now.Commit) {
|
||||
continue
|
||||
}
|
||||
if f == nil {
|
||||
@@ -118,6 +155,10 @@ func keepRecorded(ctx context.Context, open *stores, node string, shelf map[stri
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found && everyBuildKept(ctx, node) {
|
||||
return nil, fmt.Errorf("%s runs %s's build %s, which the build records no longer hold: the bus's user "+
|
||||
"list cannot be sent there alone with it kept (novox/hq issue 490)", node, m, short(kept[m]))
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("%s records rather than rolls out, and %s runs its build %s, which the build "+
|
||||
"records no longer hold: this send cannot keep it and does not move it — `push %s` sends the new "+
|
||||
|
||||
+190
-12
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
@@ -253,8 +254,19 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The grants step composes every module at the build its machine was last sent (novox/hq issue 490):
|
||||
// a move it keeps is not made. Read, not assumed, so a move it could not keep is still refused here.
|
||||
var keeps map[string]string
|
||||
if everyBuildKept(ctx, n) {
|
||||
if keeps, err = recordedKept(ctx, open, n); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var waiting []string
|
||||
for _, mv := range moves {
|
||||
if was, kept := keeps[mv.Module]; kept && sameCommit(was, mv.From) {
|
||||
continue
|
||||
}
|
||||
if !scope.judged[n] && !scope.modules[mv.Module] {
|
||||
waiting = append(waiting, fmt.Sprintf("%s %s → %s", mv.Module, short(mv.From), short(mv.To)))
|
||||
}
|
||||
@@ -283,15 +295,18 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
|
||||
// owns are the moves the send exists for — every module of a plan's tier whose first machine this is, in
|
||||
// one send (novox/hq issue 281): a send carries the machine's whole declaration (ADR 0221), so a send per
|
||||
// module was the same declaration sent again and again, each one setting aside the one before.
|
||||
func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, []string, error) {
|
||||
//
|
||||
// And the grants step first, when the send changes what the bus's user list must say (novox/hq issue 490):
|
||||
// answered for the gate to keep, nil when there was none.
|
||||
func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, []string, *inventory.GrantsStep, error) {
|
||||
inv := open.inventory
|
||||
f, err := readMoveFacts(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
moves, err := machineMoves(ctx, open, f, node, true)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
own := func(module string) bool {
|
||||
return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module })
|
||||
@@ -301,7 +316,7 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
|
||||
continue
|
||||
}
|
||||
if id := f.walkedBy(mv.Module, node, mv.To); id != "" {
|
||||
return nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
||||
return nil, nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
||||
mv.Module, short(mv.To), node, id)
|
||||
}
|
||||
}
|
||||
@@ -309,7 +324,7 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
|
||||
// plan, or a release, on this machine is not crossed with a newer build; this send waits for its gate.
|
||||
for _, o := range owns {
|
||||
if id := f.walkedBy(o.Module, node, o.To); id != "" {
|
||||
return nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
||||
return nil, nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
||||
o.Module, short(o.To), node, id)
|
||||
}
|
||||
}
|
||||
@@ -323,22 +338,22 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
|
||||
}
|
||||
}
|
||||
if len(moves) == 0 && len(owns) == 0 {
|
||||
return nil, nil, nil
|
||||
return nil, nil, nil, nil
|
||||
}
|
||||
for i := range moves {
|
||||
if moves[i].Build == "" {
|
||||
if moves[i].Build, err = inv.BuildOf(ctx, moves[i].Module, moves[i].To); err != nil {
|
||||
return nil, nil, err
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(moves, func(i, j int) bool { return moves[i].Module < moves[j].Module })
|
||||
sayRecreations(ctx, open, moves)
|
||||
sent, err := sendRollout(withScope(ctx, sendScope{judged: map[string]bool{node: true}}), open, []string{node})
|
||||
sent, grants, err := sendJudged(ctx, open, node)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
return moves, sent, nil
|
||||
return moves, sent, grants, nil
|
||||
}
|
||||
|
||||
// passCarried keeps a pass as the verdict of every build the gate judged beside its own module.
|
||||
@@ -674,7 +689,7 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
||||
r.Next++
|
||||
continue
|
||||
}
|
||||
moves, sent, err := gatedSend(ctx, open, node, nil)
|
||||
moves, sent, grants, err := gatedSend(ctx, open, node, nil)
|
||||
if errors.Is(err, errWalkedElsewhere) {
|
||||
note := fmt.Sprintf("waiting before %s: %v", node, err)
|
||||
changed := p.Note != note
|
||||
@@ -689,8 +704,12 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
||||
r.Next++
|
||||
continue
|
||||
}
|
||||
r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves, Sent: sentNow(ctx, open.inventory, sent)}
|
||||
r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves, Sent: sentNow(ctx, open.inventory, sent),
|
||||
Grants: grants}
|
||||
p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves))
|
||||
if said := grantsSaid(grants); said != "" {
|
||||
p.Note += "; " + said
|
||||
}
|
||||
if said := recreationsSaid(moves); said != "" {
|
||||
p.Note += "; " + said
|
||||
}
|
||||
@@ -916,3 +935,162 @@ func recreationsSaid(moves []inventory.CarriedMove) string {
|
||||
}
|
||||
return strings.Join(said, "; ")
|
||||
}
|
||||
|
||||
// The grants step (novox/hq issue 490).
|
||||
//
|
||||
// The bus's user list — every module's grants, composed mesh-wide from the catalogue — travels only in the
|
||||
// declaration of the machine holding the bus. A send to any other machine that changed it put that machine
|
||||
// first (issue 249), unless a build waited there for a gate: then ungatedIn left it out, and said so. On
|
||||
// 2026-10-11 a merge gave restic a new health tool; its walk sent restic to its first machine and judged it
|
||||
// there, the machine's node-engine asked the tool and the bus refused it ("this host's grant does not name
|
||||
// …"), because the machine holding the bus also runs restic, whose new build waited for that very gate. The
|
||||
// gate could not pass; a person pushed the bus's machine by hand, which carried restic's new build there
|
||||
// unjudged.
|
||||
//
|
||||
// So before a gated send, when the user list composed now is not the one the machine holding the bus was
|
||||
// last sent, that machine is sent its declaration with **every build kept at the one it runs**: the new
|
||||
// list, and nothing of any module's new code. Said on the gate (`plans`), and, when it cannot be sent, said
|
||||
// with why and passed over: the send goes on as it did before, and its gate says what it finds.
|
||||
|
||||
// brokerBehindOf is brokerBehind: a variable so a test of the walk needs no store.
|
||||
var brokerBehindOf = brokerBehind
|
||||
|
||||
// grantsStep sends the machine holding the bus its user list alone, ahead of a gated send to node, when the
|
||||
// list changed; answers what it did, or nil when there was nothing to send.
|
||||
func grantsStep(ctx context.Context, open *stores, node string) *inventory.GrantsStep {
|
||||
holder, behind, err := brokerBehindOf(ctx, open, []string{node})
|
||||
if err != nil {
|
||||
fmt.Printf("grants step before %s: whether the bus's user list changed could not be read: %v\n", node, err)
|
||||
return &inventory.GrantsStep{Node: "the machine holding the bus", Failed: err.Error()}
|
||||
}
|
||||
if holder == "" || holder == node || !behind {
|
||||
// No bus with a user list, the gate's own machine holds it (its send carries the list first), or the
|
||||
// list is the one already sent.
|
||||
return nil
|
||||
}
|
||||
if _, err := sendRollout(keepingEveryBuild(withScope(ctx, sendScope{}), holder), open, []string{holder}); err != nil {
|
||||
fmt.Printf("grants step: the bus's user list could not be sent to %s ahead of %s, which is sent without "+
|
||||
"it — the bus may refuse what the send newly grants: %v\n", holder, node, err)
|
||||
return &inventory.GrantsStep{Node: holder, Failed: err.Error()}
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
fmt.Printf("grants step: %s sent the bus's user list alone, every build there kept, before %s is judged\n",
|
||||
holder, node)
|
||||
return &inventory.GrantsStep{Node: holder, At: &now}
|
||||
}
|
||||
|
||||
// errNotGrantsOnly is a grants step refused unsent: its declaration would change more than the user list.
|
||||
var errNotGrantsOnly = errors.New("the grants step would change more than the bus's user list, and is not sent")
|
||||
|
||||
// grantsOnlyIn is what a grants step's declaration, composed for sending, would change on the machine holding
|
||||
// the bus besides its user list, against what it was last sent (novox/hq issue 490); empty when nothing else.
|
||||
// Judged by sendToEach on the very declaration it then sends — one composition, judged, then sent or refused
|
||||
// unsent. Kept builds are not the whole of a declaration: a new assignment, a settings change, an assignment
|
||||
// taken away, a provision's logins are composed as the mesh holds them now, and a step that carried any of
|
||||
// it would be the unjudged change this step exists to avoid. Compared resource by resource with the summary
|
||||
// the last send kept (ADR 0217): exact, rather than a list of the cases that can differ. A module there that
|
||||
// was never sent is a new assignment, refused whatever its resources.
|
||||
func grantsOnlyIn(ctx context.Context, open *stores, holder string, plan catalogue.Resolution, declared sendable) (string, error) {
|
||||
inv := open.inventory
|
||||
sent, known, err := inv.SentBuilds(ctx, holder)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !known {
|
||||
return fmt.Sprintf("what %s was last sent is not known", holder), nil
|
||||
}
|
||||
var other []string
|
||||
listID := ""
|
||||
for _, m := range plan.Modules {
|
||||
if _, was := sent[m.Module]; !was {
|
||||
other = append(other, m.Module+" newly assigned")
|
||||
}
|
||||
if m.BusUsers != "" && m.ClaimsSeat(catalogue.BrokerSeat) {
|
||||
listID = m.Module + "." + catalogue.BusUsersID()
|
||||
}
|
||||
}
|
||||
var facts *moveFacts
|
||||
for m, was := range sent {
|
||||
now, carried := declared.Builds[m]
|
||||
if !carried || sameCommit(now, was) {
|
||||
continue
|
||||
}
|
||||
if facts == nil {
|
||||
f, err := readMoveFacts(ctx, inv)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
facts = &f
|
||||
}
|
||||
if !facts.identical(m, was, now) {
|
||||
other = append(other, fmt.Sprintf("%s %s → %s", m, short(was), short(now)))
|
||||
}
|
||||
}
|
||||
for _, f := range declared.foreseen {
|
||||
other = append(other, f+" would be minted")
|
||||
}
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
after, err := summarize(body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
prev, err := inv.SentSummary(ctx, holder)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(prev) == 0 {
|
||||
return fmt.Sprintf("what %s was last sent is not kept for comparison", holder), nil
|
||||
}
|
||||
var before []sentResource
|
||||
if err := json.Unmarshal(prev, &before); err != nil {
|
||||
return "", fmt.Errorf("what %s was last sent is kept in a form this version does not read: %w", holder, err)
|
||||
}
|
||||
other = append(other, otherThanTheList(diffSent(before, after), listID)...)
|
||||
sort.Strings(other)
|
||||
if len(other) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
return "its send would change more than the bus's user list: " + strings.Join(other, "; "), nil
|
||||
}
|
||||
|
||||
// otherThanTheList is what a diff against the last send changes besides the user list's content: every
|
||||
// resource added, removed or changed, but the list's own resource when its content is all that changed —
|
||||
// its path, mode and every other field must be as last sent (novox/hq issue 490).
|
||||
func otherThanTheList(d sentDiff, listID string) []string {
|
||||
var other []string
|
||||
for _, id := range d.Added {
|
||||
other = append(other, "+"+id)
|
||||
}
|
||||
for _, id := range d.Removed {
|
||||
other = append(other, "-"+id)
|
||||
}
|
||||
for _, c := range d.Changed {
|
||||
if c.ID == listID && len(c.Fields) == 1 && c.Fields[0] == "content" {
|
||||
continue
|
||||
}
|
||||
other = append(other, fmt.Sprintf("~%s (%s)", c.ID, strings.Join(c.Fields, ", ")))
|
||||
}
|
||||
return other
|
||||
}
|
||||
|
||||
// sendJudged is a gated send's sends: the grants step when the user list changed, then the machine judged.
|
||||
func sendJudged(ctx context.Context, open *stores, node string) ([]string, *inventory.GrantsStep, error) {
|
||||
grants := grantsStep(ctx, open, node)
|
||||
sent, err := sendRollout(withScope(ctx, sendScope{judged: map[string]bool{node: true}}), open, []string{node})
|
||||
return sent, grants, err
|
||||
}
|
||||
|
||||
// grantsSaid is a grants step as `plans` and a walk's note say it.
|
||||
func grantsSaid(g *inventory.GrantsStep) string {
|
||||
switch {
|
||||
case g == nil:
|
||||
return ""
|
||||
case g.Failed != "":
|
||||
return fmt.Sprintf(grantsStepWord+" to %s FAILED, sent without it: %s", g.Node, g.Failed)
|
||||
default:
|
||||
return fmt.Sprintf(grantsStepWord+": %s sent the bus's user list first, its builds kept", g.Node)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -971,7 +971,7 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
||||
}
|
||||
// A gated send (ADR 0236): everything waiting on the machine goes with the tier, and the gate judges
|
||||
// all of it there.
|
||||
carried, sent, err := gatedSend(ctx, open, node, owns)
|
||||
carried, sent, grants, err := gatedSend(ctx, open, node, owns)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -992,6 +992,7 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
||||
s.GatedBy = ""
|
||||
if m == lead {
|
||||
s.Gate.Carried = carried
|
||||
s.Gate.Grants = grants
|
||||
} else {
|
||||
s.GatedBy = lead
|
||||
}
|
||||
@@ -1001,6 +1002,10 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
||||
strings.Join(sent, ", "))
|
||||
fmt.Printf("%s: tier %d built; sent %d module(s) to %s first in one send (%s), the rest once its gate passes\n",
|
||||
p.ID, p.Tier, len(modules), strings.Join(sent, ", "), strings.Join(modules, ", "))
|
||||
// The grants step taken before it (novox/hq issue 490), said with it.
|
||||
if said := grantsSaid(grants); said != "" {
|
||||
p.Note += "; " + said
|
||||
}
|
||||
// What the send recreates, said with it (novox/hq ADR 0245).
|
||||
if said := recreationsSaid(carried); said != "" {
|
||||
p.Note += "; " + said
|
||||
|
||||
@@ -160,6 +160,7 @@ func (b *backlogMesh) release(t *testing.T) inventory.Plan {
|
||||
// The backlog goes out one machine at a time: the first judged before the second is sent, each build's
|
||||
// pass kept; a rebuild with the same bytes is no move at all; and a send outside a gate is refused.
|
||||
func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
|
||||
@@ -77,6 +77,7 @@ func testAWalkStoppedAfterItsGatePassedKeepsThePass(t *testing.T) {
|
||||
// the first send is older than the wait for a first machine's report. Read before the pass is taken over, the
|
||||
// carried module's step said the first machine never applied it, and the passed build was put back.
|
||||
func TestACarriedModuleTakesItsLeadsPassBeforeItsStepIsRead(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
tm := aTierMesh(t, "m01", "m02")
|
||||
ctx := t.Context()
|
||||
inv := tm.open.inventory
|
||||
|
||||
@@ -55,6 +55,15 @@ func rotateCommand(ctx context.Context, args []string) error {
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
// A credential the provider's software issued is not the mesh's to remake (novox/hq ADR 0305 §9).
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := issuedRefusesRotation(shelf, provision); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
holders, err := inv.HoldersOf(ctx, provision, *only)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -162,6 +162,7 @@ func TestABusRebuiltFromAnUnchangedSourceDemandsNoBusStep(t *testing.T) {
|
||||
// A plan's build made from the source every machine running the module runs is not sent and not
|
||||
// judged: nothing moves. A build of a changed source is sent to its first machine, gated, as before.
|
||||
func TestAPlanSendsNothingForAnUnchangedSource(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
source string
|
||||
|
||||
@@ -410,6 +410,10 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
if b := str("by"); b != "" {
|
||||
argv = append(argv, "--by", catalogue.DeliverySeat+" for "+b)
|
||||
}
|
||||
// The merges to leave out: the walk's other merges are walked again without them (novox/hq ADR 0299).
|
||||
if w := str("without"); w != "" {
|
||||
argv = append(argv, "--without", w)
|
||||
}
|
||||
return argv, nil
|
||||
case "delivery-walks":
|
||||
argv := []string{"delivery", "walks"}
|
||||
@@ -773,6 +777,13 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
return nil, err
|
||||
}
|
||||
return []string{"secret", "ask", str("node"), str("module"), str("secret"), "--at", str("at")}, nil
|
||||
case "issued":
|
||||
// The credential a provider's software issued, taken from the provider (novox/hq ADR 0305): no value in
|
||||
// the arguments, the provision alone.
|
||||
if err := need("provision"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []string{"secret", "issued", str("provision")}, nil
|
||||
case "secret-ask":
|
||||
// A module's own secret asked for, typed by the operator at the desk (novox/hq ADR 0277): never a value
|
||||
// in the arguments. The desk is the module's machine unless at names another.
|
||||
@@ -1579,6 +1590,14 @@ func givenAtTheDesk(argv []string) bool {
|
||||
return argv[5] == "--at" && argv[6] != "" && !strings.HasPrefix(argv[6], "-")
|
||||
}
|
||||
|
||||
// takenFromTheProvider is exactly the line the `issued` verb composes, and nothing beside it: `secret issued
|
||||
// <provision>` (novox/hq ADR 0305 §10). It carries no value — the value comes from the provider's own tool, sealed to
|
||||
// the call — and answers none.
|
||||
func takenFromTheProvider(argv []string) bool {
|
||||
return len(argv) == 3 && argv[0] == "secret" && argv[1] == "issued" && argv[2] != "" &&
|
||||
!strings.HasPrefix(argv[2], "-")
|
||||
}
|
||||
|
||||
// terminalOnly refuses, through any verb, a command that is the operator's at the controller's terminal
|
||||
// alone (novox/hq ADR 0266). **Every `node` subcommand that is not a read**: `node account` and
|
||||
// `node agent-account` above all. Whoever may call a verb includes agents, and an agent that named itself
|
||||
@@ -1595,7 +1614,7 @@ func terminalOnly(argv []string) error {
|
||||
// Of a secret's commands only rotation, which seals the new value to the machine that uses it, and the
|
||||
// `give` verb's own line: an own secret typed by the operator into the desk's hidden prompt, sealed to this
|
||||
// call and then to the module's machine, so no value travels in the verb or its answer (hq ADR 0259 §10).
|
||||
if argv[0] == "secret" && (len(argv) < 2 || argv[1] != "rotate") && !givenAtTheDesk(argv) {
|
||||
if argv[0] == "secret" && (len(argv) < 2 || argv[1] != "rotate") && !givenAtTheDesk(argv) && !takenFromTheProvider(argv) {
|
||||
return terminalRefusal("secret %s is run at the controller's terminal only, never through a verb: accepting, "+
|
||||
"recovering or exporting a secret hands it to whoever asks, and that includes agents (novox/hq ADR "+
|
||||
"0266). Nothing was done", strings.Join(argv[1:], " "))
|
||||
|
||||
@@ -41,6 +41,9 @@ func secretCommand(ctx context.Context, args []string) error {
|
||||
case "accept":
|
||||
case "ask":
|
||||
return secretAsk(ctx, args[1:])
|
||||
case "issued":
|
||||
// The credential a provider's software issued, taken from the provider (novox/hq ADR 0305).
|
||||
return secretIssued(ctx, args[1:])
|
||||
case "rotate":
|
||||
return secretRotate(ctx, args[1:])
|
||||
case "recover":
|
||||
@@ -168,6 +171,7 @@ func secretAsk(ctx context.Context, args []string) error {
|
||||
|
||||
const secretUsage = "secret rotate <node> <module> <name> [--why <text> [--cause <word>]]\n" +
|
||||
"secret ask <node> <module> <name> [--at <machine>]\n" +
|
||||
"secret issued <provision>\n" +
|
||||
"secret accept <node> <module> <name> [--from <file> | --at-desk <machine>] [--provider <node> [--local <name>]]\n" +
|
||||
"secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>] [--provider <node>]\n" +
|
||||
"secret export [--out <file>]"
|
||||
|
||||
@@ -66,7 +66,7 @@ func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
|
||||
}
|
||||
// Both machines' `node show` name it: where the provider runs, and where the consumer is.
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
shown := printed(t, func() error { return showNode(ctx, open.inventory, node) })
|
||||
shown := printed(t, func() error { return showNode(ctx, open, node) })
|
||||
if !strings.Contains(shown, "open condition(s) about this machine") || !strings.Contains(shown, "mesh_laptop_dashboard") {
|
||||
t.Fatalf("node show %s does not name it:\n%s", node, shown)
|
||||
}
|
||||
|
||||
@@ -289,6 +289,10 @@ func printStatus(asked answers) error {
|
||||
fmt.Printf("\n `take <node> <module>` compares what runs against what it declares, and runs it\n\n")
|
||||
}
|
||||
|
||||
// Assigned and not applied (novox/hq issue 380): before what is merely reported, because it reads like work
|
||||
// finished and is none.
|
||||
printNotApplied(asked.leftOut)
|
||||
|
||||
if len(asked.unheld) > 0 {
|
||||
// **Reported, and not refused yet** (novox/hq ADR 0207 §4). Each machine still resolves and
|
||||
// is sent what it would be; this says which of its modules depend on a seat nothing there
|
||||
@@ -337,6 +341,10 @@ func printStatus(asked answers) error {
|
||||
"lists them, and each is in its condition's tried\n\n", asked.heals.Acts, asked.heals.Escalated)
|
||||
}
|
||||
|
||||
// What each machine was last sent and by whom, one line each (novox/hq issue 485). Not a fault, so it does not
|
||||
// break "all well"; a record that could not be read does.
|
||||
printLastSends(nodes, asked.lastSent, asked.lastSentUnread)
|
||||
|
||||
if adopted := adoptedNodes(nodes); len(adopted) > 0 {
|
||||
// Said, because nothing forces the flip: a node left adopted is visible here rather than
|
||||
// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
|
||||
@@ -456,6 +464,13 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
||||
return answers{}, err
|
||||
}
|
||||
plans[n.Name] = planned{plan, settings}
|
||||
// And which of its modules a push leaves out (novox/hq issue 380), judged as the push judges it.
|
||||
if left := leftOutOf(plan, settings, n.Adopted); len(left) > 0 {
|
||||
if out.leftOut == nil {
|
||||
out.leftOut = map[string][]leftOutModule{}
|
||||
}
|
||||
out.leftOut[n.Name] = left
|
||||
}
|
||||
out.unheld = append(out.unheld, plan.Unheld...)
|
||||
// And which of its modules a provider leaves out of its grants, for an identity too long
|
||||
// for what the provision keeps (novox/hq ADR 0225) — judged from the consumer's own
|
||||
@@ -474,6 +489,12 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
||||
out.pendingUnread = err.Error()
|
||||
err = nil
|
||||
}
|
||||
// And what each machine was last sent, by whom (novox/hq issue 485): the record of sends is read through
|
||||
// the controller's verbs alone, the store's reader being shut out of its tables. Unreadable is said.
|
||||
if out.lastSent, err = inv.LastSends(ctx); err != nil {
|
||||
out.lastSentUnread = err.Error()
|
||||
err = nil
|
||||
}
|
||||
out.plans, err = inv.RecentPlans(ctx, 5)
|
||||
if err != nil {
|
||||
return answers{}, err
|
||||
@@ -604,9 +625,10 @@ func untakenModules(ctx context.Context, inv *inventory.Inventory, nodes []inven
|
||||
// read as success for the whole of the edge cut-over outage (novox/hq 04-ISSUES/125).
|
||||
func (a answers) well() bool {
|
||||
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
|
||||
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 &&
|
||||
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 && len(a.leftOut) == 0 &&
|
||||
len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.overflowing) == 0 &&
|
||||
len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending)
|
||||
len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending) &&
|
||||
a.lastSentUnread == ""
|
||||
}
|
||||
|
||||
// pendingOpen is whether any pending assignment is still to be made. One that ended without being made is
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
{
|
||||
"module": "nfs-server",
|
||||
"version": "1",
|
||||
"upgrade": {
|
||||
"policy": "record",
|
||||
"why": "the folders other machines mount: a build that breaks the exports leaves every client's mount hanging or refused, and the gate on this one machine does not see the clients (hq ADR 0236, ADR 0263)"
|
||||
},
|
||||
"capabilities": [
|
||||
"package-manager",
|
||||
"service-manager"
|
||||
],
|
||||
"provides": [
|
||||
{
|
||||
"name": "nfs-share",
|
||||
"scope": "mesh",
|
||||
"identity": false
|
||||
}
|
||||
],
|
||||
"data": {
|
||||
"consumers": {
|
||||
"nfs-share": {
|
||||
"class": "none",
|
||||
"why": "the shared folders are the operator's data (hq ADR 0051): what a client writes lands in them, and they are protected where the operator declares them, never by this module, which keeps nothing of a consumer's"
|
||||
}
|
||||
}
|
||||
},
|
||||
"claims": [
|
||||
{
|
||||
"name": "node-nfs-server",
|
||||
"scope": "node",
|
||||
"serves": [
|
||||
"exports",
|
||||
"clients",
|
||||
"test",
|
||||
"reload",
|
||||
"adopt"
|
||||
]
|
||||
}
|
||||
],
|
||||
"state": [
|
||||
{
|
||||
"name": "exports",
|
||||
"ttl-seconds": 120,
|
||||
"per-machine": true
|
||||
}
|
||||
],
|
||||
"tools": [
|
||||
"nfs_health"
|
||||
],
|
||||
"listens": [
|
||||
{
|
||||
"name": "nfs",
|
||||
"port": 2049,
|
||||
"protocol": "tcp",
|
||||
"from": "mesh",
|
||||
"fixed": true,
|
||||
"why": "the shares, to the mesh's machines only (hq ADR 0263): NFS version 4 alone, which needs no other port, and never the home network, where a device that is not a node could claim any user id"
|
||||
}
|
||||
],
|
||||
"resources": [
|
||||
{
|
||||
"id": "package",
|
||||
"type": "package",
|
||||
"package": "nfs-utils"
|
||||
},
|
||||
{
|
||||
"id": "nfs-conf",
|
||||
"type": "file",
|
||||
"path": "/etc/nfs.conf.d/50-mesh.conf",
|
||||
"mode": "0644",
|
||||
"content": "# Written by the mesh (module nfs-server, novox/hq ADR 0263). Replaced on every push; a drop-in of\n# the operator's that sorts after this one overrides it, and is theirs.\n#\n# NFS version 4 only: a client needs port 2049 and nothing else, so the module opens nothing more\n# than that, to the private network. Version 3 needs rpcbind and mountd, on ports the mesh does not open.\n[nfsd]\nvers2=n\nvers3=n\nvers4=y\nvers4.0=n\nvers4.1=y\nvers4.2=y\n"
|
||||
},
|
||||
{
|
||||
"id": "config-dir",
|
||||
"type": "directory",
|
||||
"path": "/etc/nfs-server",
|
||||
"mode": "0755"
|
||||
},
|
||||
{
|
||||
"id": "config",
|
||||
"type": "file",
|
||||
"path": "/etc/nfs-server/shares.conf",
|
||||
"mode": "0644",
|
||||
"content": "# Written by the mesh (module nfs-server, novox/hq ADR 0263) from this machine's assignment.\n# Replaced on every push; change the `shares` setting, never this file.\n#\n# The shares: name=folder, or name=folder:ro, one share per folder. The module's process exports each\n# to the private network's range below, every client mapped to the folder's owner.\nshares=${setting:shares}\nrange=${machine:mesh-range}\n"
|
||||
},
|
||||
{
|
||||
"id": "run-dir",
|
||||
"type": "directory",
|
||||
"path": "/run/nfs-server",
|
||||
"mode": "0755"
|
||||
},
|
||||
{
|
||||
"id": "server",
|
||||
"type": "service",
|
||||
"unit": "nfs-server.service",
|
||||
"state": "running",
|
||||
"boot": "enabled",
|
||||
"restart-on": [
|
||||
"nfs-conf"
|
||||
],
|
||||
"health": {
|
||||
"kind": "unit"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "exports",
|
||||
"type": "process",
|
||||
"name": "nfs-server-exports",
|
||||
"artifact": "tools",
|
||||
"run": [
|
||||
"./nfs-server",
|
||||
"exports"
|
||||
],
|
||||
"restart-on": [
|
||||
"config"
|
||||
],
|
||||
"health": {
|
||||
"kind": "tool",
|
||||
"tool": "nfs_health",
|
||||
"interval": "60s",
|
||||
"timeout": "10s",
|
||||
"looks": 2,
|
||||
"grace": "90s"
|
||||
}
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/nfs-server",
|
||||
"binary": "nfs-server",
|
||||
"loads": [
|
||||
"nfs-server"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -167,6 +167,7 @@ func (tm *tierMesh) raise(machine, kind, summary string) {
|
||||
// The incident's tier, small: a dozen modules whose first machine is the same are sent there in one
|
||||
// send, judged by one gate, and the rest of their machines are sent once — two sends, not two dozen.
|
||||
func TestATierOfManyModulesIsSentToEachMachineOnce(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
var modules []string
|
||||
for i := 1; i <= 12; i++ {
|
||||
modules = append(modules, fmt.Sprintf("m%02d", i))
|
||||
@@ -268,6 +269,7 @@ func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) {
|
||||
// A machine-level condition about anything else holds the machine back as a whole: everything the send
|
||||
// moved there fails together, once, with that reason — not one module for it.
|
||||
func TestAMachineUnhealthyAsAWholeFailsTheWholeSendTogether(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
tm := aTierMesh(t, "app1", "app2", "app3")
|
||||
ctx := t.Context()
|
||||
inv := tm.open.inventory
|
||||
@@ -330,6 +332,7 @@ func TestTheConditionsAboutAMachineItself(t *testing.T) {
|
||||
// rollback that failed; and `plans retry` asks it again rather than refusing a build that was never
|
||||
// judged (the incident's "NOT put back: no build kept").
|
||||
func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := g.open.inventory
|
||||
@@ -384,6 +387,7 @@ func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) {
|
||||
// them, they are registered back and marked in the judging that finds them broken, the plan goes on judging
|
||||
// the other module to its own pass, and only then fails.
|
||||
func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
for _, order := range [][]string{{broker.RuntimeModule, "app1"}, {"app1", broker.RuntimeModule}} {
|
||||
t.Run("gate kept on "+order[0], func(t *testing.T) {
|
||||
tm := aTierMesh(t, order...)
|
||||
@@ -435,3 +439,38 @@ func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// **A send carrying a core module waits for the core's readings** (issue 524 review): the leaf beside the node
|
||||
// tools has its one reading at once, but the send is judged as the core: not before the settle time, and three
|
||||
// readings.
|
||||
func TestASendCarryingTheCoreIsJudgedAsTheCore(t *testing.T) {
|
||||
tm := aTierMesh(t, broker.RuntimeModule, "app1")
|
||||
ctx := t.Context()
|
||||
// One reading counted per advance, as every test with a gate left open does: a gate judged with no time
|
||||
// between readings would count a pass at every look and never let an advance rest.
|
||||
gateEvery, gateSettle = time.Hour, time.Hour
|
||||
for i := 0; i < 6; i++ {
|
||||
advancePlans(ctx, tm.open)
|
||||
}
|
||||
p := tm.plan(t)
|
||||
lead := p.Modules[broker.RuntimeModule].Gate
|
||||
if lead == nil {
|
||||
lead = p.Modules["app1"].Gate
|
||||
}
|
||||
if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) || !p.Open() || lead == nil || lead.Verdict != "" || lead.Wants != gatePasses {
|
||||
t.Fatalf("passed before the core's settle time: sent %v, plan %s (%s), gate %+v", tm.sent, p.State, p.Note, lead)
|
||||
}
|
||||
gateEvery, gateSettle = 0, 0
|
||||
for i := 0; i < 6; i++ {
|
||||
advancePlans(ctx, tm.open)
|
||||
}
|
||||
p = tm.plan(t)
|
||||
if p.State != inventory.PlanDone || !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"laptop"}}) {
|
||||
t.Fatalf("the plan is %s (%s), sent %v", p.State, p.Note, tm.sent)
|
||||
}
|
||||
for _, m := range []string{broker.RuntimeModule, "app1"} {
|
||||
if g := p.Modules[m].Gate; g != nil && (g.Verdict != inventory.GatePassed || g.Passes < gatePasses) {
|
||||
t.Fatalf("%s's gate: %+v; want passed on %d readings", m, g, gatePasses)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user