diff --git a/.gitignore b/.gitignore index 4d374472..7f67f6c5 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ # A built binary. The lab writes one here when pointed at the repository root by mistake; # built artifacts belong in build/, which is already ignored. /mesh-builder +/mesh-controller +/postgres-provisioner +/redis-provisioner diff --git a/cmd/mesh-controller/adoption.go b/cmd/mesh-controller/adoption.go index be79bfb7..eb93cdee 100644 --- a/cmd/mesh-controller/adoption.go +++ b/cmd/mesh-controller/adoption.go @@ -408,6 +408,9 @@ func comparisonOf(module string, c comparison, opts takeOptions) (preview string switch { case sec.Origin == inventory.OriginAccepted: fmt.Fprintf(&b, " %s %s: accepted from a person, carried in as it is\n", what, name) + case sec.Origin == inventory.OriginIssued: + // Taken from the provider's software (novox/hq ADR 0305): the value the running service uses. + fmt.Fprintf(&b, " %s %s: issued by the provider's software, carried in as it is\n", what, name) case opts.Mint[sec.Name]: fmt.Fprintf(&b, " %s %s: minted by the mesh; the service takes the new value, as --mint said\n", what, name) case foundData: diff --git a/cmd/mesh-controller/batches.go b/cmd/mesh-controller/batches.go index 291a6718..7b4704a7 100644 --- a/cmd/mesh-controller/batches.go +++ b/cmd/mesh-controller/batches.go @@ -367,7 +367,13 @@ func keepBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan, } closes, latest := windowOf(merges, b.Created, window, atMost) b.Delivery.Merges = named - b.Delivery.Batch = &inventory.PlanBatch{ClosesAt: closes, AtMost: latest, Behind: behind, Own: b.OwnPath()} + var closed bool + var withheld []string + if was := b.Delivery.Batch; was != nil { + closed, withheld = was.Closed, was.Withheld // a stop's, which no look takes back (novox/hq ADR 0299) + } + b.Delivery.Batch = &inventory.PlanBatch{ClosesAt: closes, AtMost: latest, Behind: behind, Own: b.OwnPath(), + Closed: closed, Withheld: withheld} b.State = inventory.PlanAssembling if behind != "" && windowClosed(*b, now) { b.State = inventory.PlanQueued @@ -402,7 +408,7 @@ func windowClosed(b inventory.Plan, now time.Time) bool { return true } w := b.Delivery.Batch - return !now.Before(w.ClosesAt) || !now.Before(w.AtMost) + return w.Closed || !now.Before(w.ClosesAt) || !now.Before(w.AtMost) } // carriedOf is a set of merges as a walk carries them: the latest merge of each repository's branch, and @@ -762,7 +768,18 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [ } sort.Strings(also) ordered := groupOrderEdges(members, entries, read, edges) + var withheld []string plan := planOfMoves(moved, append(append([]inventory.Edge{}, edges...), ordered...)) + // **Withheld, a stopped merge's modules** (novox/hq ADR 0299): a module is built at its branch, which holds the + // merge a stop left out of the walk these merges were walked again from — built here only as a dependent, it + // would deliver that merge. One a merge of this batch moves is built: that merge carries the stopped change, + // and the stop named it. + if batch.Delivery != nil && batch.Delivery.Batch != nil { + if out := withhold(&plan, batch.Delivery.Batch.Withheld, moved); len(out) > 0 { + withheld = out + fmt.Printf(" withheld, a stopped merge's modules built only as dependents: %s\n", strings.Join(out, ", ")) + } + } plan.ID, plan.Revision, plan.Created, plan.Epoch = batch.ID, batch.Revision, now, batch.Epoch plan.Commits = commits newest := newestCommit(commits) @@ -773,6 +790,7 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [ } plan.Delivery.Merges = named plan.Delivery.Alone = batch.Delivery != nil && batch.Delivery.Alone + plan.Delivery.Withheld = withheld // **Its moments and its class** (novox/hq ADR 0282 decision 6): measured, never acted on. plan.Times = walkTimesAtCut(*batch, plan, entries, now) if len(moved) == 0 { @@ -823,6 +841,39 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [ return nil } +// withhold takes from a plan every module named that its merges do not move, and the tiers left empty: the +// modules taken. +func withhold(plan *inventory.Plan, names, moved []string) []string { + var out []string + for _, n := range names { + if _, in := plan.Modules[n]; in && !slices.Contains(moved, n) { + delete(plan.Modules, n) + out = append(out, n) + } + } + if len(out) == 0 { + return nil + } + var tiers [][]string + for _, t := range plan.Tiers { + var kept []string + for _, n := range t { + if !slices.Contains(out, n) { + kept = append(kept, n) + } + } + if len(kept) > 0 { + tiers = append(tiers, kept) + } + } + if tiers == nil { + tiers = [][]string{} + } + plan.Tiers = tiers + sort.Strings(out) + return out +} + // walkTimesAtCut is a walk's own moments as it is cut (novox/hq ADR 0282 decision 6): when its batch's window // closed — no merge for the window's length, or its maximum, whichever came first — when it was cut, and its // class. A merge walked alone had no window. diff --git a/cmd/mesh-controller/build.go b/cmd/mesh-controller/build.go index 5b772e91..23cdcfd3 100644 --- a/cmd/mesh-controller/build.go +++ b/cmd/mesh-controller/build.go @@ -835,6 +835,9 @@ type answers struct { // public name on the machine went dark. The holds were correct; they were recorded only in the // machine's own state file, and the one visible symptom was a count that did not add up. untaken map[string]map[string]int + // leftOut is, per machine, every module of its set its composition leaves out, and why (novox/hq issue + // 380): assigned and not applied, which every push said only in passing. Not well while there is any. + leftOut map[string][]leftOutModule // unheld is every module on a machine whose resources are applied through a seat nothing on // that machine holds (novox/hq ADR 0207), with the modules that could hold it. Reported, not // refused, until the switch — and while there is any, the mesh is not all well: the order the @@ -863,6 +866,10 @@ type answers struct { // applied, is not well: an assignment somebody made is not made yet, or will not be. pending []inventory.PendingAssignment pendingUnread string + // lastSent is every machine's last recorded send, by name, with what the machine answered of it (novox/hq + // issue 485); lastSentUnread why it could not be read. + lastSent map[string]inventory.Send + lastSentUnread string } // heldBy is every artifact this mesh has built, for a build that may need one as its base. diff --git a/cmd/mesh-controller/changeplan.go b/cmd/mesh-controller/changeplan.go index 7a983c97..9c84cd19 100644 --- a/cmd/mesh-controller/changeplan.go +++ b/cmd/mesh-controller/changeplan.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "slices" "sort" "strings" @@ -28,6 +29,14 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento for _, e := range entries { byName[e.Manifest.Module] = e } + // The machine holding the bus, whose declaration carries every module's grants (novox/hq issue 490). + holder := "" + for _, e := range entries { + if e.Manifest.BusUsers != "" && e.Manifest.ClaimsSeat(catalogue.BrokerSeat) && len(e.On) > 0 { + holder = e.On[0] + } + } + var granting []string machines := map[string]*link.MachinePlan{} machine := func(name string) *link.MachinePlan { if machines[name] == nil { @@ -50,6 +59,10 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento machine(on).Receives = append(machine(on).Receives, name) } } + if !waits && holder != "" && !(len(e.On) == 1 && e.On[0] == holder) && len(e.On) > 0 && + !slices.Contains(granting, name) { + granting = append(granting, name) + } switch { case (name == "nats" || catalogue.ProvidesBus(e.Manifest)) && len(e.On) > 0: // Said before the merge (novox/hq issue 336): a new bus build holds every send to the bus's machine @@ -76,6 +89,13 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento } } } + if len(granting) > 0 { + // Said before the merge (novox/hq issue 490): what the walk does when the change alters the bus's + // user list. Whether it does is known only once the builds are registered, so it is said as a rule. + p.Steps = append(p.Steps, fmt.Sprintf("%s: if this changes what the bus's user list says (a new tool, "+ + "subject or user), %s is sent that list alone, every build there kept, before %s is judged on its first "+ + "machine", grantsStepWord, holder, strings.Join(granting, ", "))) + } var names []string for name := range machines { names = append(names, name) @@ -88,6 +108,10 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento return p } +// grantsStepWord names the grants step (novox/hq issue 490): the bus's user list sent alone to the machine +// holding the bus ahead of a walk's gate. Not the bus step, which replaces the bus itself. +const grantsStepWord = "grants step" + // busUpgradeNeeded is how a change plan says that merging it holds the bus's machine for a person's step. const busUpgradeNeeded = "merging this needs a person's bus upgrade" diff --git a/cmd/mesh-controller/delivery.go b/cmd/mesh-controller/delivery.go index 0a8a4a88..56e4a077 100644 --- a/cmd/mesh-controller/delivery.go +++ b/cmd/mesh-controller/delivery.go @@ -117,28 +117,411 @@ func letGo(ctx context.Context, inv *inventory.Inventory, id, by, why string) (i // stopWalk ends a walk on its delivery's word: failed, said as stopped by whom, why. What it asked still // builds and registers; nothing further is asked or sent. func stopWalk(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, error) { + p, _, err := stopWalkWithout(ctx, inv, id, by, why, nil, time.Now().UTC()) + return p, err +} + +// stopAnswer is what a stop that leaves merges out says, as data (novox/hq ADR 0299): the record stopped, the +// batch that walks its other merges again with those merges, every merge left out, the modules the stopped walk +// already sent that stay on the machines named, the modules the batch's walk withholds, and every merge elsewhere +// that will deliver a stopped change when it is walked — a later merge of its repository, or one moving its modules. +type stopAnswer struct { + Stopped string `json:"stopped"` + WalkedAgainBy string `json:"walked_again_by,omitempty"` + WalkedAgain []string `json:"walked_again,omitempty"` + LeftOut []inventory.PlanLeftOut `json:"left_out,omitempty"` + Kept []inventory.PlanKept `json:"kept,omitempty"` + Withheld []string `json:"withheld,omitempty"` + StillCarriedBy []string `json:"still_carried_by,omitempty"` +} + +// stopWalkWithout ends a walk on its delivery's word and, given merges to leave out, walks every other merge it +// answered again without them (novox/hq issue 459, ADR 0299): one walk carries every merge of its batch, so ending +// it for one group member ended every other delivery on it. +// +// - Left out are the merges named, each later merge of their repository's branch, which contains one, and each +// merge moving a module a left-out merge moves: a module is built at its branch, which holds the stopped merge. +// - The others join the open batch of their kind — merges heard while the walk ran wait there, and an older +// commit never walks after a newer one — or a new batch when none is open; its window is closed, and its walk +// withholds the left-out merges' modules where it would build them only as dependents. +// - Given a batch not yet cut, the merges left out leave it for a stopped record of their own, and the batch +// goes on without them; with nothing left in it, the batch itself is stopped. +// - The stopped record, the batch and every merge moved are written in one transaction (decision 7). +// +// With nothing left to walk, or no merge named, the walk is only ended. A merge named that the record does not +// answer refuses the stop whole. +func stopWalkWithout(ctx context.Context, inv *inventory.Inventory, id, by, why string, without []string, + now time.Time) (inventory.Plan, stopAnswer, error) { + said := stopAnswer{Stopped: id} release, err := inv.HoldPlans(ctx, true) if err != nil { - return inventory.Plan{}, err + return inventory.Plan{}, said, err } defer release() p, err := inv.PlanByID(ctx, id) + if err != nil { + return inventory.Plan{}, said, err + } + batch := p.Batch() + if !p.Open() && !(batch && len(without) > 0) { + return p, said, fmt.Errorf("%s is already %s", p.ID, p.State) + } + var merges, again []inventory.BatchedMerge + if len(without) > 0 { + if merges, err = inv.MergesOf(ctx, p.ID); err != nil { + return p, said, err + } + if again, said.LeftOut, err = leaveOut(merges, without); err != nil { + return p, said, fmt.Errorf("%s is not stopped: %w", p.ID, err) + } + } + stopped := func(r *inventory.Plan) { + if r.Delivery == nil { + r.Delivery = &inventory.PlanDelivery{} + } + r.Delivery.Stopped, r.Delivery.StoppedWhy, r.Delivery.LeftOut = by, why, said.LeftOut + r.State = inventory.PlanFailed + r.Note = fmt.Sprintf("stopped by %s at tier %d: %s", by, r.Tier, why) + } + withheld := movesOf(leftOutMerges(merges, said.LeftOut)) + said.Withheld = withheld + if said.StillCarriedBy, err = stillCarried(ctx, inv, p.ID, merges, said.LeftOut, withheld); err != nil { + return p, said, err + } + if batch { + return p, said, stopInBatch(ctx, inv, &p, merges, again, &said, stopped, now) + } + entries, err := inv.Catalogued(ctx) + if err != nil { + return p, said, err + } + running := map[string][]string{} + for _, e := range entries { + running[e.Manifest.Module] = e.On + } + said.Kept = keptOf(p, movesOf(again), running) + stopped(&p) + p.Delivery.Kept = said.Kept + if len(again) == 0 { + if len(said.LeftOut) > 0 { + p.Note += fmt.Sprintf("; left out %s, nothing else to walk", mergesLeftOutWords(said.LeftOut)) + } + return p, said, inv.SavePlan(ctx, &p) + } + target, err := batchToWalkAgain(ctx, inv, again, now) + if err != nil { + return p, said, fmt.Errorf("%s is not stopped: %w", p.ID, err) + } + target.Delivery.Batch.Closed = true + target.Delivery.Batch.Withheld = union(target.Delivery.Batch.Withheld, withheld) + sort.Strings(target.Delivery.Batch.Withheld) + said.WalkedAgainBy = target.ID + var moves []inventory.MergeMove + for _, m := range again { + said.WalkedAgain = append(said.WalkedAgain, m.Repository+"@"+m.Commit) + moves = append(moves, inventory.MergeMove{Repository: m.Repository, Commit: m.Commit, Plan: target.ID, Alone: m.Alone}) + } + p.Delivery.WalkedAgainBy = target.ID + p.Note += fmt.Sprintf("; its other merges are walked again by %s: %s", target.ID, strings.Join(said.WalkedAgain, ", ")) + if len(said.LeftOut) > 0 { + p.Note += "; left out " + mergesLeftOutWords(said.LeftOut) + } + if len(said.Kept) > 0 { + p.Note += "; " + keptWords(said.Kept) + } + // The batch first, so no merge names a plan the store does not hold; all of it, or nothing (decision 7). + if err := inv.SavePlansMoving(ctx, []*inventory.Plan{&target, &p}, moves); err != nil { + return p, said, fmt.Errorf("%s is not stopped, and none of its merges moved: %w", p.ID, err) + } + // What the batch holds and names now: written again by the cutter's next look should this fail. + if err := keepBatch(ctx, inv, &target, now, ""); err != nil { + fmt.Printf(" %s holds its merges; its record is written again at the next look: %v\n", target.ID, err) + } + fmt.Printf("%s stopped by %s; %s walks its other merges again: %s\n", p.ID, by, target.ID, + strings.Join(said.WalkedAgain, ", ")) + return p, said, nil +} + +// stopInBatch leaves merges out of a batch not yet cut (ADR 0299): they go to a stopped record of their own, which +// names the batch as walking the others, and the batch withholds their modules; a batch left with none is stopped +// itself. +func stopInBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan, merges, again []inventory.BatchedMerge, + said *stopAnswer, stopped func(*inventory.Plan), now time.Time) error { + if len(again) == 0 { + stopped(b) + b.Delivery.Batch = nil + b.Note = "a batch whose every merge was left out by a stop: " + b.Note + return inv.SavePlan(ctx, b) + } + left := leftOutMerges(merges, said.LeftOut) + first := left[0] + r := inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch, + Commit: first.Commit, Merged: first.Merged, Created: now, Tiers: [][]string{}, + Modules: map[string]*inventory.PlanModule{}} + if r.ID == b.ID { + r.ID += "-left" + } + stopped(&r) + r.Delivery.WalkedAgainBy = b.ID + r.Note += fmt.Sprintf("; left out of %s before it was walked: %s; %s walks the others", b.ID, + mergesLeftOutWords(said.LeftOut), b.ID) + b.Delivery.Batch.Withheld = union(b.Delivery.Batch.Withheld, said.Withheld) + sort.Strings(b.Delivery.Batch.Withheld) + var moves []inventory.MergeMove + for _, m := range left { + moves = append(moves, inventory.MergeMove{Repository: m.Repository, Commit: m.Commit, Plan: r.ID, Alone: m.Alone}) + } + for _, m := range again { + said.WalkedAgain = append(said.WalkedAgain, m.Repository+"@"+m.Commit) + } + said.Stopped, said.WalkedAgainBy = r.ID, b.ID + if err := inv.SavePlansMoving(ctx, []*inventory.Plan{&r, b}, moves); err != nil { + return fmt.Errorf("%s is not changed, and none of its merges moved: %w", b.ID, err) + } + if err := keepBatch(ctx, inv, b, now, ""); err != nil { + fmt.Printf(" %s holds its merges; its record is written again at the next look: %v\n", b.ID, err) + } + fmt.Printf("%s: %s left out by a stop, kept by %s\n", b.ID, mergesLeftOutWords(said.LeftOut), r.ID) + return nil +} + +// batchToWalkAgain is the batch a stopped walk's merges join: the open batch of their kind, or a new one, not yet +// written. The kind is read from what the merges moved. +func batchToWalkAgain(ctx context.Context, inv *inventory.Inventory, again []inventory.BatchedMerge, + now time.Time) (inventory.Plan, error) { + own := false + for _, n := range movesOf(again) { + if _, on := onTheControllersPath[n]; on { + own = true + } + } + batches, err := inv.Batches(ctx) if err != nil { return inventory.Plan{}, err } - if !p.Open() { - return p, fmt.Errorf("%s is already %s", p.ID, p.State) + for _, b := range batches { + if b.OwnPath() == own { + if b.Delivery == nil { + b.Delivery = &inventory.PlanDelivery{} + } + if b.Delivery.Batch == nil { + b.Delivery.Batch = &inventory.PlanBatch{Own: own} + } + return b, nil + } } - if p.Delivery == nil { - p.Delivery = &inventory.PlanDelivery{} + first := again[0] + return inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch, + Commit: first.Commit, Merged: first.Merged, Created: now, State: inventory.PlanAssembling, + Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{}, + Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: own}}}, nil +} + +// leaveOut splits a walk's merges by the merges named to leave out, each `@` (a commit's +// first characters will do): the merges walked again, and those left out — each named one; each later merge of +// its repository's branch, which contains it; and each merge moving a module a left-out merge moves, which is +// built at its branch and so holds the stopped merge — until none more is. A name the walk does not answer is +// refused. +func leaveOut(merges []inventory.BatchedMerge, without []string) ([]inventory.BatchedMerge, []inventory.PlanLeftOut, error) { + var stopped []inventory.BatchedMerge + for _, w := range without { + repository, commit, ok := strings.Cut(strings.TrimSpace(w), "@") + if !ok || repository == "" || len(commit) < 7 { + return nil, nil, fmt.Errorf("%q names no merge: @", w) + } + i := slices.IndexFunc(merges, func(m inventory.BatchedMerge) bool { + return strings.EqualFold(m.Repository, repository) && strings.HasPrefix(m.Commit, commit) + }) + if i < 0 { + return nil, nil, fmt.Errorf("it answers no merge %s@%s", repository, commit) + } + stopped = append(stopped, merges[i]) } - p.Delivery.Stopped, p.Delivery.StoppedWhy = by, why - p.State = inventory.PlanFailed - p.Note = fmt.Sprintf("stopped by %s at tier %d: %s", by, p.Tier, why) - if err := inv.SavePlan(ctx, &p); err != nil { - return p, err + same := func(a, b inventory.BatchedMerge) bool { + return strings.EqualFold(a.Repository, b.Repository) && a.Commit == b.Commit } - return p, nil + out := map[int]inventory.PlanLeftOut{} + for i, m := range merges { + if slices.ContainsFunc(stopped, func(s inventory.BatchedMerge) bool { return same(s, m) }) { + out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit} + } + } + for grew := true; grew; { + grew = false + for i, m := range merges { + if _, left := out[i]; left { + continue + } + for j, o := range out { + l := merges[j] + contains := o.Contains + if contains == "" { + contains = l.Commit + } + if strings.EqualFold(l.Repository, m.Repository) && l.Branch == m.Branch && laterMerge(m, l) { + out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit, Contains: contains} + } else if shared := sharedMoves(m, l); len(shared) > 0 { + out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit, Contains: contains, + Shares: shared} + } else { + continue + } + grew = true + break + } + } + } + var again []inventory.BatchedMerge + var left []inventory.PlanLeftOut + for i, m := range merges { + if o, is := out[i]; is { + left = append(left, o) + } else { + again = append(again, m) + } + } + sort.SliceStable(left, func(i, j int) bool { return left[i].Contains == "" && left[j].Contains != "" }) + return again, left, nil +} + +// sharedMoves are the modules two kept merges both moved when they were heard. +func sharedMoves(a, b inventory.BatchedMerge) []string { + var out []string + theirs := announcedOf(b).Moves + for _, n := range announcedOf(a).Moves { + if slices.Contains(theirs, n) && !slices.Contains(out, n) { + out = append(out, n) + } + } + sort.Strings(out) + return out +} + +// leftOutMerges are the kept merges a stop left out. +func leftOutMerges(merges []inventory.BatchedMerge, out []inventory.PlanLeftOut) []inventory.BatchedMerge { + var left []inventory.BatchedMerge + for _, m := range merges { + if slices.ContainsFunc(out, func(o inventory.PlanLeftOut) bool { + return strings.EqualFold(o.Repository, m.Repository) && o.Commit == m.Commit + }) { + left = append(left, m) + } + } + return left +} + +// movesOf is every module kept merges moved when they were heard, sorted. +func movesOf(merges []inventory.BatchedMerge) []string { + var out []string + for _, m := range merges { + out = union(out, announcedOf(m).Moves) + } + sort.Strings(out) + return out +} + +// keptOf are the modules a stopped walk sent that no merge walked again moves, and the machines each reached: +// those keep running the stopped walk's build. A module sent to its first machines reached those, and the rest it +// was sent to after them; one sent with no first machine (its policy sends it to all together, or a later tier is +// built by it) reached every machine running it. +func keptOf(p inventory.Plan, walkedAgain []string, running map[string][]string) []inventory.PlanKept { + var out []inventory.PlanKept + for name, m := range p.Modules { + if m == nil || slices.Contains(walkedAgain, name) { + continue + } + var machines []string + if m.FirstAt != nil || m.SentAt != nil { + machines = union(machines, m.First) + } + if m.SentAt != nil && len(m.First) == 0 { + machines = union(machines, running[name]) + } + if m.Gate != nil && m.Gate.Since != nil { + machines = union(machines, m.Gate.Machines) + } + for machine := range m.Rest { + machines = union(machines, []string{machine}) + } + if len(machines) == 0 { + continue + } + sort.Strings(machines) + out = append(out, inventory.PlanKept{Module: name, Machines: machines}) + } + sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module }) + return out +} + +// keptWords is what a stopped walk leaves on the machines, as a person reads it. +func keptWords(kept []inventory.PlanKept) string { + var words []string + for _, k := range kept { + words = append(words, k.Module+" on "+strings.Join(k.Machines, ", ")) + } + return "left running the stopped walk's build, which nothing walks again: " + strings.Join(words, "; ") +} + +// stillCarried are the merges outside the stopped record, in a batch or an open walk, that will deliver a stopped +// change when they are walked: a later merge of its repository's branch, which contains it, or one that moves a +// module it moves. Holding a merged change back for good is a revert's (ADR 0299). +func stillCarried(ctx context.Context, inv *inventory.Inventory, id string, merges []inventory.BatchedMerge, + out []inventory.PlanLeftOut, withheld []string) ([]string, error) { + var named []inventory.BatchedMerge + for _, m := range leftOutMerges(merges, out) { + if slices.ContainsFunc(out, func(o inventory.PlanLeftOut) bool { return o.Commit == m.Commit && o.Contains == "" }) { + named = append(named, m) + } + } + if len(named) == 0 { + return nil, nil + } + open, err := inv.OpenPlans(ctx) + if err != nil { + return nil, err + } + batches, err := inv.Batches(ctx) + if err != nil { + return nil, err + } + var carried []string + for _, p := range append(open, batches...) { + if p.ID == id { + continue + } + ms, err := inv.MergesOf(ctx, p.ID) + if err != nil { + return nil, err + } + for _, m := range ms { + later := slices.ContainsFunc(named, func(s inventory.BatchedMerge) bool { + return strings.EqualFold(s.Repository, m.Repository) && s.Branch == m.Branch && laterMerge(m, s) + }) + shares := slices.ContainsFunc(announcedOf(m).Moves, func(n string) bool { return slices.Contains(withheld, n) }) + if later || shares { + carried = append(carried, fmt.Sprintf("%s@%s (in %s)", m.Repository, m.Commit, p.ID)) + } + } + } + sort.Strings(carried) + return carried, nil +} + +// mergesLeftOutWords is the merges a stop left out, as a person reads them: each that contains a stopped one, or +// shares its modules, says so. +func mergesLeftOutWords(out []inventory.PlanLeftOut) string { + var words []string + for _, o := range out { + s := o.Repository + "@" + short(o.Commit) + switch { + case len(o.Shares) > 0: + s += " (which moves " + strings.Join(o.Shares, ", ") + " as " + short(o.Contains) + " does)" + case o.Contains != "": + s += " (which contains " + short(o.Contains) + ")" + } + words = append(words, s) + } + return strings.Join(words, ", ") } // orderMember is one member of a group, as mesh-delivery says it. @@ -466,6 +849,8 @@ func deliveryCommand(ctx context.Context, args []string) error { group := set.String("group", "", "a delivery group's id") by := set.String("by", catalogue.DeliverySeat, "who says it") why := set.String("why", "", "why") + without := set.String("without", "", "merges a stop leaves out, @, comma-separated: the walk's "+ + "other merges are walked again without them (novox/hq ADR 0299)") limit := set.Int("n", 50, "how many ended walks to answer beside the open ones") planID := set.String("plan", "", "one walk") positionals, err := parseAround(set, rest) @@ -552,15 +937,21 @@ func deliveryCommand(ctx context.Context, args []string) error { return nil case "stop": if len(positionals) != 1 || strings.TrimSpace(*why) == "" { - return errors.New("delivery stop --why [--by ]") + return errors.New("delivery stop --why [--by ] [--without @,…]") } var p inventory.Plan + var said stopAnswer if err := sayingOnTheBus(ctx, func() (err error) { - p, err = stopWalk(ctx, inv, positionals[0], *by, strings.TrimSpace(*why)) + p, said, err = stopWalkWithout(ctx, inv, positionals[0], *by, strings.TrimSpace(*why), splitList(*without), + time.Now().UTC()) return err }); err != nil { return err } + if *without != "" { + // What it left out and what it walks again, as data: the delivery's owner reads it (ADR 0299). + return answer(said) + } fmt.Printf("%s stopped by %s at tier %d of %d; what was asked still builds and registers, nothing further "+ "is asked or sent\n", p.ID, *by, p.Tier, len(p.Tiers)) return nil diff --git a/cmd/mesh-controller/doctor.go b/cmd/mesh-controller/doctor.go index dfb74f06..b7f48946 100644 --- a/cmd/mesh-controller/doctor.go +++ b/cmd/mesh-controller/doctor.go @@ -74,8 +74,12 @@ type probe struct { // probeRegistry is the registry, in to-be 45's order. **The registry is the design's live form**: a // probe added to a design is a row added here. var probeRegistry = []probe{ - {ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation", - From: "issues 236, 263, 275", Kind: "declaration-refused", Raises: []string{kindAwaitingPush}, Phase: 1, + // D1 also says every module assigned and left out of a machine's declaration (novox/hq issue 380), from the + // resolution it already makes: needs-operator for a setting nobody gave, left-out for any other cause. + {ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation; no module " + + "assigned to a machine is left out of its declaration unsaid", + From: "issues 236, 263, 275, 380", Kind: "declaration-refused", + Raises: []string{kindAwaitingPush, kindLeftOut, kindNeedsOperator}, Phase: 1, run: probeDeclarations}, {ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6", From: "issue 262", Kind: "resolver-wrong", Phase: 1, run: probeResolvers}, @@ -118,8 +122,8 @@ var probeRegistry = []probe{ Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries}, // The delivery budgets (novox/hq ADR 0282 decision 7): the newest delivery of each class within its budget, // read from mesh-delivery's `times`; a measurement said, never a delivery held. - {ID: probeBudgetsID, Asserts: "the newest delivery of a leaf module ran on every machine within five minutes of " + - "its merge, and of a core module within ten: mesh-delivery's `times`", From: "ADR 0282", + {ID: probeBudgetsID, Asserts: "the newest delivery of a leaf module ran on every machine within three minutes of " + + "its merge, and of a core module within ten: mesh-delivery's `times`", From: "ADR 0282, ADR 0304", Kind: kindOverBudget, Phase: 3, run: probeBudgets}, // A client of the bus reconnecting in a loop (novox/hq issue 327), from the server's record of closed // connections, which the bus's own module reads. diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index 36eefc4d..b4dd98e2 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -27,8 +27,10 @@ import ( // **"Reported applied" is not enough.** ADR 0218 sent a module to one machine first and the rest once // that machine reported it applied. A build that applies and then does nothing, crashes, serves no // tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by -// the component's health — the core's health definitions, or a module's own — passing three times over -// at least two minutes, within ten minutes of the apply. Only then are the rest sent. +// the component's health — the core's health definitions, or a module's own — within ten minutes of the +// apply: a core module passing three times over at least two minutes, a leaf module once, as soon as the +// node-engine states it healthy (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it). Only then +// are the rest sent. // // **A failing gate stops the plan and puts the previous build back there.** The build is marked failed // at its gate — registration refuses it and nothing sends it again on its own — the module's registered @@ -43,11 +45,38 @@ var ( gateSettle = 2 * time.Minute // gateBound is how long after the apply the build may take to become healthy. gateBound = 10 * time.Minute - // gatePasses is how many consecutive judgings must find it healthy, gateEvery apart at the least. - gatePasses = 3 - gateEvery = 40 * time.Second + // gatePasses is how many consecutive judgings must find a core module healthy, gateEvery apart at the + // least, after gateSettle. gateLeafPasses is a leaf module's: one, taken once the node-engine states every + // resource of it healthy by its own check, with no settle (novox/hq ADR 0282 decision 4 as ADR 0304 + // decision 3 changed it, issue 524). + gatePasses = 3 + gateLeafPasses = 1 + gateLeafSettles = false + gateEvery = 40 * time.Second ) +// coreModule is whether a module is judged as the core (novox/hq ADR 0282 decision 1): one on the controller's +// own path, or one holding the mesh's resolver. Every other module is a leaf. +func coreModule(module string, m catalogue.Manifest) bool { + if _, own := onTheControllersPath[module]; own { + return true + } + for _, c := range m.Claims { + if c.Name == resolverSeat { + return true + } + } + return coreComponent(module) != "" +} + +// gateWants is how many passes a module needs, and whether it must stay healthy gateSettle first. +func gateWants(module string, m catalogue.Manifest) (passes int, settles bool) { + if coreModule(module, m) { + return gatePasses, true + } + return gateLeafPasses, gateLeafSettles +} + // gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions // are raised by a plan as it judges, and kept or cleared by the probe on every run. const gateProbe = "DG" @@ -608,8 +637,24 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs } } settled := now.Sub(*g.Since) >= gateSettle + // What the send needs is what its most demanding module needs: a send carrying a core module is judged + // as the core. + wants, settles := 0, false + for _, m := range modules { + p, s := gateWants(m, shelf[m]) + wants, settles = max(wants, p), settles || s + } + if len(modules) == 0 { + // A send judged on no machine of its modules (its first machine not running them) has no class to + // be judged by: it is judged as the core, never passed at once without a reading. + wants, settles = gatePasses, true + } + g.Wants = wants + // A module's own pass is its own class's: a leaf beside a core module that fails keeps its build after + // its one reading, and only what did not pass is put back (issue 318's rule, by class since issue 524). passedAlone := func(m string) bool { - return settled && reading[m] <= healthPerson && g.Healthy[m] >= gatePasses + p, s := gateWants(m, shelf[m]) + return (settled || !s) && reading[m] <= healthPerson && g.Healthy[m] >= p } // fail decides the send failed: what passed on its own keeps its pass, everything else is put back. fail := func(why string) { @@ -657,7 +702,7 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs g.Passes++ g.LastPass, g.Last, g.Failing = &now, "", nil g.Read(now, true, "") - if g.Passes >= gatePasses && settled { + if g.Passes >= wants && (settled || !settles) { decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes, now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now) } @@ -1340,9 +1385,9 @@ func gateLine(g *inventory.PlanGate) string { if g.Why != "" { line += ": " + g.Why } else if g.Last != "" { - line += fmt.Sprintf(" (%d of %d passes; wanting: %s)", g.Passes, gatePasses, g.Last) + line += fmt.Sprintf(" (%d of %d passes; wanting: %s)", g.Passes, gateWanted(g), g.Last) } else if g.Verdict == "" { - line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gatePasses) + line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gateWanted(g)) } if g.Verdict == "" && len(g.Waits) > 0 { line += waitsSaid(g.Waits) @@ -1353,6 +1398,10 @@ func gateLine(g *inventory.PlanGate) string { if g.Rollback != "" { line += "; " + g.Rollback } + // Before the send it judges (novox/hq issue 490). + if said := grantsSaid(g.Grants); said != "" { + line += "; " + said + } return line } @@ -1381,3 +1430,11 @@ func witnessObservation(node string, r lease.Rollback) conditions.Observation { Summary: fmt.Sprintf("the witness on %s judged the %s %s not healthy %s at %s: %s", node, r.Component, short(r.From), what, r.At.UTC().Format(time.RFC3339), r.Why)} } + +// gateWanted is the passes a gate's send needs, three where the gate was judged before it was said. +func gateWanted(g *inventory.PlanGate) int { + if g.Wants > 0 { + return g.Wants + } + return gatePasses +} diff --git a/cmd/mesh-controller/gate_followup_test.go b/cmd/mesh-controller/gate_followup_test.go index 54022686..35929eb5 100644 --- a/cmd/mesh-controller/gate_followup_test.go +++ b/cmd/mesh-controller/gate_followup_test.go @@ -53,6 +53,7 @@ func healthyContainer(module string) inventory.ResourceHealth { // tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass, // counted over the gate's spacing — before it says its own. The broken one alone is put back. func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) { + judgeLeavesAsTheCore(t) open := aMesh(t) ctx := t.Context() withGateBounds(t, 150*time.Millisecond, 20*time.Millisecond, 10*time.Second) @@ -284,6 +285,7 @@ func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) { // tools are marked and put back in the very judging that found them broken — their registered build is the // one before, and the laptop is sent it — while `app` goes on being judged, recovers, and keeps its own pass. func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testing.T) { + judgeLeavesAsTheCore(t) open := aMesh(t) ctx := t.Context() inv := open.inventory diff --git a/cmd/mesh-controller/gate_test.go b/cmd/mesh-controller/gate_test.go index 8fe19c58..ced2ccce 100644 --- a/cmd/mesh-controller/gate_test.go +++ b/cmd/mesh-controller/gate_test.go @@ -153,6 +153,7 @@ func (g *gateMesh) plan(t *testing.T) inventory.Plan { // registered and sent to it again — and never reaches the second machine; it is marked, said as a // condition and an event, never registered or rolled back again. func TestABuildThatFailsItsGateIsRolledBackOnItsFirstMachineAndGoesNoFurther(t *testing.T) { + judgeLeavesAsTheCore(t) g := aGateMesh(t) ctx := t.Context() inv := g.open.inventory @@ -255,7 +256,7 @@ func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) { t.Fatalf("sent %v", g.sent) } gate := p.Modules["app"].Gate - if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gatePasses { + if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gateWanted(gate) { t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate) } if v, found, err := g.open.inventory.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed { @@ -266,8 +267,52 @@ func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) { } } +// A leaf module passes its gate on one reading, as soon as its first machine is healthy, without waiting out the +// core's settle time (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it, issue 524). +func TestALeafModulePassesOnOneHealthyReading(t *testing.T) { + g := aGateMesh(t) + ctx := t.Context() + gateSettle = time.Hour // a core module could not pass for an hour + advancePlans(ctx, g.open) // the first machine is sent the new build + advancePlans(ctx, g.open) // one healthy judging + p := g.plan(t) + gate := p.Modules["app"].Gate + if gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes != 1 || gate.Wants != 1 { + t.Fatalf("a leaf module's gate after one healthy judging: %+v (plan %s: %s)", gate, p.State, p.Note) + } + if !strings.Contains(gate.Why, "healthy 1 times") { + t.Errorf("the verdict says %q", gate.Why) + } +} + +// Which modules are judged as the core: those on the controller's own path and the resolver's holder, three +// readings after the settle time; every other module one (novox/hq ADR 0282 decision 1, ADR 0304 decision 3). +func TestAModuleIsJudgedByItsClass(t *testing.T) { + for _, c := range []struct { + module string + claims []catalogue.Claim + passes int + settles bool + }{ + {"app", nil, 1, false}, + {"nzbget", nil, 1, false}, + {"mesh-controller", nil, 3, true}, + {"mesh-host", nil, 3, true}, + {"node-tools", nil, 3, true}, + {"nats", nil, 3, true}, + {catalogue.DeliverySeat, nil, 3, true}, + {"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, true}, + } { + p, s := gateWants(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims}) + if p != c.passes || s != c.settles { + t.Errorf("%s: %d passes, settles %v; want %d, %v", c.module, p, s, c.passes, c.settles) + } + } +} + // A first machine that never becomes healthy fails its gate at the bound, and is put back. func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) { + judgeLeavesAsTheCore(t) g := aGateMesh(t) ctx := t.Context() advancePlans(ctx, g.open) @@ -593,3 +638,13 @@ func TestAWitnessesVerdictIsAConditionWhileItsReportsSayIt(t *testing.T) { t.Fatalf("a verdict the reports no longer carry is still said: %+v %v", obs, err) } } + +// judgeLeavesAsTheCore makes a test's leaf modules judged as a core module is, three readings after the settle +// time, for a test of the gate's other rules (a fault, the bound, a supersession) that needs a judging still +// open after a healthy reading. +func judgeLeavesAsTheCore(t *testing.T) { + t.Helper() + wasPasses, wasSettles := gateLeafPasses, gateLeafSettles + gateLeafPasses, gateLeafSettles = gatePasses, true + t.Cleanup(func() { gateLeafPasses, gateLeafSettles = wasPasses, wasSettles }) +} diff --git a/cmd/mesh-controller/generation.go b/cmd/mesh-controller/generation.go index 15f79cbb..abf9ff0e 100644 --- a/cmd/mesh-controller/generation.go +++ b/cmd/mesh-controller/generation.go @@ -202,21 +202,99 @@ func modulesWords(modules []string) string { return strings.Join(modules, ", ") } -// writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234): -// nothing when the mesh has not recorded a send to it. +// writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234), and +// what it answered (issue 485): nothing when the mesh has not recorded a send to it. func writeLastSend(ctx context.Context, w io.Writer, inv *inventory.Inventory, node string) error { s, found, err := inv.LastSend(ctx, node) if err != nil || !found { return err } - generation := "no generation recorded" - if s.Generation > 0 { - generation = fmt.Sprintf("assignment generation %d", s.Generation) - } - fmt.Fprintf(w, "%s was last sent sequence %d at %s by %s, composed from %s, naming %s\n", node, s.Sequence, s.SentAt.Local().Format("2006-01-02 15:04:05"), s.Sender, generation, modulesWords(s.Modules)) - if s.RefusedAt != nil { - fmt.Fprintf(w, " and refused it at %s: it had applied generation %d\n", - s.RefusedAt.Local().Format("2006-01-02 15:04:05"), s.RefusedApplied) + fmt.Fprintf(w, "%s:\n", node) + for _, line := range lastSendLines(s) { + fmt.Fprintln(w, line) } return nil } + +// lastSendLines is a machine's last send in full, as `node show` and `plan` say it (novox/hq issue 485): when, +// by whom, under which lease epoch, from which assignment generation, naming which modules, and what the +// machine answered of it — the refusal in its own words. +func lastSendLines(s inventory.Send) []string { + const stamp = "2006-01-02 15:04:05" + epoch := "under no lease epoch" + if s.Epoch > 0 { + epoch = fmt.Sprintf("under lease epoch %d", s.Epoch) + } + generation := "no assignment generation recorded" + if s.Generation > 0 { + generation = fmt.Sprintf("assignment generation %d", s.Generation) + } + lines := []string{ + fmt.Sprintf(" last sent sequence %d at %s", s.Sequence, s.SentAt.Local().Format(stamp)), + fmt.Sprintf(" by %s", s.Sender), + fmt.Sprintf(" %s, composed from %s", epoch, generation), + fmt.Sprintf(" naming %s", modulesWords(s.Modules)), + } + answer := func(words string) { lines = append(lines, " its answer "+words) } + switch a := s.Answer; { + case s.RefusedAt != nil: + // Before the report: a refusal for the generation is kept on the send itself and is the machine's + // answer to this send in particular. + answer(fmt.Sprintf("refused it at %s: it had applied assignment generation %d, newer than this one", + s.RefusedAt.Local().Format(stamp), s.RefusedApplied)) + case a.Outcome == "" || a.At == nil: + answer("the machine has not reported on it yet") + case a.Outcome == inventory.OutcomeApplied: + answer("applied it at " + a.At.Local().Format(stamp)) + case a.Outcome == inventory.OutcomeRefused: + answer("refused it at " + a.At.Local().Format(stamp) + ", in its words:") + for _, line := range strings.Split(strings.TrimRight(a.Refused, "\n"), "\n") { + lines = append(lines, " "+strings.TrimSpace(line)) + } + default: + answer(fmt.Sprintf("%s at %s: %d resource(s) failed — `status` says which", a.Outcome, + a.At.Local().Format(stamp), a.Failed)) + } + return lines +} + +// sendAnswerWord is what a machine answered of a send, in a word or few, as status says it in one line. +func sendAnswerWord(s inventory.Send) string { + switch a := s.Answer; { + case s.RefusedAt != nil: + return fmt.Sprintf("refused: it had applied generation %d", s.RefusedApplied) + case a.Outcome == "" || a.At == nil: + return "not reported on yet" + case a.Outcome == inventory.OutcomeFailed: + return fmt.Sprintf("failed (%d resource(s))", a.Failed) + default: + return a.Outcome + } +} + +// printLastSends says, one line per machine, when it was last sent a declaration, by whom, from which assignment +// generation and what it answered (novox/hq issue 485). `node show ` says the send in full. +func printLastSends(nodes []inventory.Node, sends map[string]inventory.Send, unread string) { + if unread != "" { + fmt.Printf("what each machine was last sent could NOT be read: %s\n\n", unread) + return + } + if len(nodes) == 0 { + return + } + fmt.Println("last sent, per machine:") + for _, n := range nodes { + s, found := sends[n.Name] + if !found { + fmt.Printf(" %-12s no send recorded\n", n.Name) + continue + } + generation := "no generation" + if s.Generation > 0 { + generation = fmt.Sprintf("generation %d", s.Generation) + } + fmt.Printf(" %-12s %s by %s, %s — %s\n", n.Name, s.SentAt.Local().Format("2006-01-02 15:04"), s.Sender, + generation, sendAnswerWord(s)) + } + fmt.Printf("\n `node show ` says a machine's last send in full\n\n") +} diff --git a/cmd/mesh-controller/grants_step_store_test.go b/cmd/mesh-controller/grants_step_store_test.go new file mode 100644 index 00000000..095a84f1 --- /dev/null +++ b/cmd/mesh-controller/grants_step_store_test.go @@ -0,0 +1,198 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/identity" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// The grants step on a store (novox/hq issue 490): the machine holding the bus is sent its new user list, +// and nothing else — not the new build of a module it runs, not a module newly assigned there. +// +// The restic shape: anchor holds the bus and runs restic at c1, as does laptop; restic's c2 gives it a +// health tool. The grants step's send to anchor composes restic at c1, carries the user list that grants +// laptop's node-engine c2's tool, and records that anchor still runs c1. A module newly assigned to anchor +// makes the step fail, said, rather than install it unjudged. +func TestTheGrantsStepSendsTheUserListAndNothingElse(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + start := time.Now().Add(-time.Hour) + build := func(module, commit string, at time.Time, manifest map[string]any) link.BuildResult { + manifest["module"], manifest["version"] = module, "1" + raw, _ := json.Marshal(manifest) + return link.BuildResult{ID: link.NewBuildID(at), Repository: "novox/mesh-catalog", Path: "modules/" + module, + On: "anchor", Module: module, Commit: commit, Manifest: raw, + Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}} + } + for _, b := range []link.BuildResult{ + build("nats", "n1", start, natsFixture()), + build("restic", "c1", start.Add(time.Second), resticFixture(false)), + build("wallpaper", "w1", start.Add(2*time.Second), wallpaperFixture()), + } { + if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil { + t.Fatal(err) + } + } + for _, a := range [][2]string{{"anchor", "nats"}, {"anchor", "restic"}, {"laptop", "restic"}} { + if _, err := inv.Assign(ctx, a[0], a[1]); err != nil { + t.Fatal(err) + } + } + for _, n := range []string{"anchor", "laptop"} { + if _, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: "node." + n, Kind: inventory.BusNode, Node: n}); err != nil { + t.Fatal(err) + } + } + + wasEpoch := epochForActs + epochForActs = func(context.Context) (uint64, error) { return 7, nil } + t.Cleanup(func() { epochForActs = wasEpoch }) + + // A send to anchor composed as sendToEach composes it: numbered, planned, declared, stamped. + compose := func(under context.Context) (catalogue.Resolution, sendable) { + t.Helper() + numbered, err := allot(under, inv, "anchor") + if err != nil { + t.Fatal(err) + } + plan, settings, err := planFor(under, open, "anchor") + if err != nil { + t.Fatal(err) + } + gens, err := generators(under, open) + if err != nil { + t.Fatal(err) + } + declared, err := declarationWith(under, open, "anchor", plan, settings, gens, Allocating) + if err != nil { + t.Fatal(err) + } + numbered.stamp(&declared) + return plan, declared + } + record := func(declared sendable) { + t.Helper() + body, err := declared.Body() + if err != nil { + t.Fatal(err) + } + if _, err := recordSent(ctx, inv, "anchor", body, declared.Builds, declared.Epoch, + sentRecordOf(ctx, declared)); err != nil { + t.Fatal(err) + } + } + // Kept for anchor alone. laptop is never recorded as sent, on purpose: composing anchor resolves laptop + // too (who is on the private network), and a step that kept every machine's builds refused anchor's + // composition for want of laptop's last send (repo-check of #230 at 0f853e93). + kept := keepingEveryBuild(keepingRecorded(ctx), "anchor") + + // What anchor was last sent: everything at the builds of before the merge, as an ordinary send sends it. + _, before := compose(keepingRecorded(ctx)) + record(before) + plan, declared := compose(kept) + if only, err := grantsOnlyIn(ctx, open, "anchor", plan, declared); err != nil || only != "" { + t.Fatalf("with nothing changed, the step reads %q, %v", only, err) + } + + // The merge: restic's c2 gives it a health tool. + if _, _, err := takeIn(ctx, inv, build("restic", "c2", start.Add(time.Minute), + resticFixture(true))); err != nil { + t.Fatal(err) + } + + // The step's send to anchor: restic as anchor runs it, c1, and the user list granting c2's tool. + generation, err := inv.AssignmentGeneration(ctx) + if err != nil { + t.Fatal(err) + } + plan, declared = compose(kept) + for _, m := range plan.Modules { + if m.Module == "restic" && len(m.Tools) > 0 { + t.Fatalf("the grants step composed restic's new build on anchor: tools %v", m.Tools) + } + } + if declared.Builds["restic"] != "c1" { + t.Fatalf("the step's send records it carries restic %q; want c1, which anchor runs", declared.Builds["restic"]) + } + if !strings.Contains(declared.BusUsers, "mesh.mod.restic.tool.backup_health.laptop") { + t.Errorf("the step's declaration does not grant laptop's node-engine restic's new tool:\n%s", declared.BusUsers) + } + // The generation it carries is the current one (novox/hq issue 234), and the step does not raise it. + if declared.composedFrom != generation { + t.Errorf("the step's send was composed from generation %d; the mesh is at %d", declared.composedFrom, generation) + } + // The guard, on the very declaration the send sends: only the user list differs from the last send. + if only, err := grantsOnlyIn(ctx, open, "anchor", plan, declared); err != nil || only != "" { + t.Fatalf("the step reads as changing more than the user list: %q, %v", only, err) + } + // And the gate's refusal still reads the step's send: it moves nothing there. + if names, err := ungatedIn(kept, open, []string{"anchor"}, ""); err != nil || strings.Join(names, ",") != "anchor" { + t.Fatalf("the step's send is refused as a move: %v, %v", names, err) + } + // Recorded as the send records it: anchor still runs restic c1, its gate still to come there. + record(declared) + if sent, _, err := inv.SentBuilds(ctx, "anchor"); err != nil || sent["restic"] != "c1" { + t.Fatalf("after the step anchor reads as sent restic %q (%v); want c1", sent["restic"], err) + } + if after, err := inv.AssignmentGeneration(ctx); err != nil || after != generation { + t.Fatalf("the step moved the assignment generation from %d to %d (%v)", generation, after, err) + } + + // A module newly assigned to anchor: the step would install it, unjudged, so it is refused unsent. + if _, err := inv.Assign(ctx, "anchor", "wallpaper"); err != nil { + t.Fatal(err) + } + plan, declared = compose(kept) + only, err := grantsOnlyIn(ctx, open, "anchor", plan, declared) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(only, "wallpaper newly assigned") || !strings.Contains(only, "+wallpaper") { + t.Fatalf("a new assignment on the bus's machine reads %q; want the step refused, naming it", only) + } + // And the send itself refuses it, unsent: judged where it is composed, before the bus is dialled. + identity.ForTest(t) + if _, err := sendToEach(kept, open, []string{"anchor"}); !errors.Is(err, errNotGrantsOnly) || + !strings.Contains(err.Error(), "wallpaper") { + t.Fatalf("the grants step's send of a new assignment was not refused by the send: %v", err) + } +} + +// fixtureImage is the image every fixture container of the grants step's tests runs. +var fixtureImage = "registry.invalid:5000/restic/backup@sha256:" + strings.Repeat("b", 64) + +// resticFixture is restic's manifest in the shape of mesh-catalog #210: at c2 (withTool) its backup judged by +// its new tool `backup_health`, beside a check of another kind it does not run itself — a tool check alone +// is refused (ADR 0227 rule 8, ADR 0240 rule 2) — and at c1 neither. +func resticFixture(withTool bool) map[string]any { + backup := map[string]any{"id": "backup", "type": "container", "name": "restic-backup", "image": fixtureImage} + measure := map[string]any{"id": "measure", "type": "container", "name": "restic-measure", "image": fixtureImage} + m := map[string]any{"resources": []any{backup, measure}} + if withTool { + backup["health"] = map[string]any{"kind": catalogue.HealthTool, "tool": "backup_health"} + measure["health"] = map[string]any{"kind": "runtime"} + m["tools"] = []string{"backup_health"} + } + return m +} + +// natsFixture is the bus's module: it holds mesh-broker and is sent the user list. +func natsFixture() map[string]any { + return map[string]any{"bus-users": "/var/lib/nats-module/conf/accounts.conf", + "claims": []any{map[string]any{"name": catalogue.BrokerSeat, "scope": catalogue.ScopeMesh}}} +} + +// wallpaperFixture is a module the bus's machine is newly assigned. +func wallpaperFixture() map[string]any { + return map[string]any{"resources": []any{ + map[string]any{"id": "paper", "type": "container", "name": "wallpaper", "image": fixtureImage}}} +} diff --git a/cmd/mesh-controller/grants_step_test.go b/cmd/mesh-controller/grants_step_test.go new file mode 100644 index 00000000..70e275b2 --- /dev/null +++ b/cmd/mesh-controller/grants_step_test.go @@ -0,0 +1,226 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// The grants reach the bus before the gate (novox/hq issue 490). +// +// On 2026-10-11 a merge gave restic, assigned to every machine, a new health tool. Its walk sent restic to +// its first machine and judged it there; that machine's node-engine asked the new tool and the bus refused +// it, because the grant was in the bus's user list, which only the machine holding the bus carries — and +// that machine runs restic too, so it was left out of the send while restic's new build waited there for +// the very gate that could not pass. A person pushed it by hand, carrying restic's new build there unjudged. + +// aSend is one send the walk made: to which machines, and whether every build there was kept. +type aSend struct { + names []string + keepsAll bool + judged []string +} + +// sendsRecorded replaces the walk's send and the reading of the bus's user list for one test: the machine +// holding the bus is novox, and its list is behind as behind says. +func sendsRecorded(t *testing.T, holder string, behind bool, refuse error) *[]aSend { + t.Helper() + var sends []aSend + wasSend, wasBehind := sendRollout, brokerBehindOf + t.Cleanup(func() { sendRollout, brokerBehindOf = wasSend, wasBehind }) + brokerBehindOf = func(_ context.Context, _ *stores, names []string) (string, bool, error) { + if slices.Contains(names, holder) { + return holder, false, nil + } + return holder, behind, nil + } + sendRollout = func(ctx context.Context, _ *stores, names []string) ([]string, error) { + s := aSend{names: append([]string(nil), names...), keepsAll: len(names) == 1 && everyBuildKept(ctx, names[0])} + for n := range scopeOf(ctx).judged { + s.judged = append(s.judged, n) + } + sends = append(sends, s) + if refuse != nil && s.keepsAll { + return nil, refuse + } + return names, nil + } + return &sends +} + +// The restic shape: a new tool on a module on every machine, its first machine ace, the bus on novox. +func TestAWalkSendsTheGrantsToTheBusBeforeTheFirstMachineIsJudged(t *testing.T) { + sends := sendsRecorded(t, "novox", true, nil) + sent, grants, err := sendJudged(t.Context(), nil, "ace") + if err != nil { + t.Fatal(err) + } + if len(*sends) != 2 { + t.Fatalf("the walk made %d send(s), want the grants step and then the first machine: %+v", len(*sends), *sends) + } + first, then := (*sends)[0], (*sends)[1] + if strings.Join(first.names, ",") != "novox" || !first.keepsAll || len(first.judged) != 0 { + t.Errorf("the first send is %+v, want novox alone, every build there kept, judging nothing", first) + } + if strings.Join(then.names, ",") != "ace" || then.keepsAll || strings.Join(then.judged, ",") != "ace" { + t.Errorf("the second send is %+v, want ace, judged", then) + } + if strings.Join(sent, ",") != "ace" { + t.Errorf("the gate's machines are %v, want ace alone: the bus's machine is not judged", sent) + } + if grants == nil || grants.Node != "novox" || grants.At == nil || grants.Failed != "" { + t.Fatalf("the gate keeps %+v as its grants step", grants) + } + line := gateLine(&inventory.PlanGate{Machines: sent, Grants: grants}) + if !strings.Contains(line, "grants step: novox sent the bus's user list first, its builds kept") { + t.Errorf("plans says the gate as %q", line) + } +} + +// Nothing more when there is nothing to carry: the list unchanged, or the first machine holds the bus. +func TestAWalkTakesNoGrantsStepWhenTheListIsAlreadyThere(t *testing.T) { + for _, c := range []struct { + name string + node string + behind bool + }{{"the list unchanged", "ace", false}, {"the first machine holds the bus", "novox", true}} { + t.Run(c.name, func(t *testing.T) { + sends := sendsRecorded(t, "novox", c.behind, nil) + _, grants, err := sendJudged(t.Context(), nil, c.node) + if err != nil { + t.Fatal(err) + } + if len(*sends) != 1 || (*sends)[0].keepsAll || grants != nil { + t.Errorf("the walk made %+v with grants %+v, want the judged send alone", *sends, grants) + } + }) + } +} + +// A grants step that cannot be sent is said on the gate, and the walk goes on as it did before. +func TestAGrantsStepThatFailsIsSaidAndTheSendGoesOn(t *testing.T) { + sends := sendsRecorded(t, "novox", true, errors.New("a bus upgrade waits for a person")) + sent, grants, err := sendJudged(t.Context(), nil, "ace") + if err != nil { + t.Fatal(err) + } + if strings.Join(sent, ",") != "ace" || len(*sends) != 2 { + t.Errorf("the walk made %+v", *sends) + } + if grants == nil || grants.Failed == "" || grants.At != nil { + t.Fatalf("the gate keeps %+v", grants) + } + if line := gateLine(&inventory.PlanGate{Machines: sent, Grants: grants}); !strings.Contains(line, + "grants step to novox FAILED, sent without it: a bus upgrade waits for a person") { + t.Errorf("plans says the gate as %q", line) + } +} + +// A step whose send would change more than the user list is refused unsent by the send (sendToEach): the +// gate says it FAILED with what differed, and the walk goes on. +func TestAGrantsStepThatWouldCarryMoreIsNotSent(t *testing.T) { + refusal := fmt.Errorf("%w: its send would change more than the bus's user list: -postgres.login-n8n", errNotGrantsOnly) + sends := sendsRecorded(t, "novox", true, refusal) + sent, grants, err := sendJudged(t.Context(), nil, "ace") + if err != nil { + t.Fatal(err) + } + if strings.Join(sent, ",") != "ace" || len(*sends) != 2 { + t.Errorf("the walk made %+v, want the refused step and then ace's judged send", *sends) + } + if grants == nil || grants.At != nil || !strings.Contains(grants.Failed, "-postgres.login-n8n") { + t.Fatalf("the gate keeps %+v", grants) + } +} + +// The step's exemption is the list's content alone: its path or mode moving is a change like any other. +func TestOnlyTheUserListsContentIsExempt(t *testing.T) { + list := "nats.bus-users" + file := func(fields map[string]any) sentResource { + r := map[string]any{"id": list, "type": "file", "path": "/conf/accounts.conf", "mode": "0600", "content": "a"} + for k, v := range fields { + r[k] = v + } + body, _ := json.Marshal(map[string]any{"resources": []any{r}}) + s, err := summarize(body) + if err != nil { + t.Fatal(err) + } + return s[0] + } + was := file(nil) + if other := otherThanTheList(diffSent([]sentResource{was}, []sentResource{file(map[string]any{"content": "b"})}), list); len(other) != 0 { + t.Errorf("the list's new content reads as more: %v", other) + } + for _, f := range []map[string]any{{"mode": "0644"}, {"path": "/elsewhere"}, {"content": "b", "owner": "nats"}} { + other := otherThanTheList(diffSent([]sentResource{was}, []sentResource{file(f)}), list) + if len(other) != 1 || !strings.HasPrefix(other[0], "~"+list) { + t.Errorf("the list's resource changed by %v reads %v; want it refused", f, other) + } + } +} + +// The step keeps the builds of the machine it sends alone: the others its composition resolves are composed +// as any send composes them. +func TestTheGrantsStepKeepsOneMachinesBuilds(t *testing.T) { + ctx := keepingEveryBuild(t.Context(), "novox") + if !everyBuildKept(ctx, "novox") || everyBuildKept(ctx, "ace") || everyBuildKept(t.Context(), "novox") { + t.Error("the grants step keeps the builds of a machine it does not send") + } +} + +// The store test's manifests are manifests the controller takes in: checked here, where no store is needed, +// so a fixture the module rules refuse fails before the store test is run (repo-check of #230 at 325575b2). +func TestTheGrantsStepFixturesAreManifests(t *testing.T) { + for name, m := range map[string]map[string]any{"nats": natsFixture(), "restic-c1": resticFixture(false), + "restic-c2": resticFixture(true), "wallpaper": wallpaperFixture()} { + m["module"], m["version"] = strings.SplitN(name, "-", 2)[0], "1" + raw, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + if _, err := catalogue.ParseManifest(raw); err != nil { + t.Errorf("%s: %v", name, err) + } + } +} + +// The delivery plan says the step before the merge. +func TestAChangePlanSaysTheGrantsStep(t *testing.T) { + const catalogue_ = "http://forge.internal:20000/novox/mesh-catalog.git" + entry := func(name string, on ...string) inventory.Entry { + e := fromRepo(name, catalogue_, "modules/"+name) + e.Source.BuiltFrom = "old" + e.On = on + return e + } + nats := entry("nats", "novox") + nats.Manifest.BusUsers = "/etc/nats/users.conf" + nats.Manifest.Claims = []catalogue.Claim{{Name: catalogue.BrokerSeat, Scope: catalogue.ScopeMesh}} + restic := entry("restic", "ace", "novox", "shanks") + only := entry("bus-tools", "novox") + entries := []inventory.Entry{nats, restic, only} + rollsOut := func(string) (inventory.Upgrade, bool) { return inventory.Upgrade{RollOut: true}, true } + plan := func(paths ...string) link.ChangePlan { + m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: paths} + return changePlanOf("novox/mesh-catalog", "main", "head", reachOfMerge(m, entries, nil, nil), entries, rollsOut) + } + + text := planText(plan("modules/restic/module.json")) + if !strings.Contains(text, "grants step: if this changes what the bus's user list says (a new tool, subject or "+ + "user), novox is sent that list alone, every build there kept, before restic is judged on its first machine") { + t.Errorf("the change plan does not say the grants step:\n%s", text) + } + // A module only on the bus's machine goes there with the list in its own send: no step of its own. + if text := planText(plan("modules/bus-tools/module.json")); strings.Contains(text, "grants step") { + t.Errorf("a module on the bus's machine alone reads:\n%s", text) + } +} diff --git a/cmd/mesh-controller/handacts.go b/cmd/mesh-controller/handacts.go index fdfa987e..147a6dca 100644 --- a/cmd/mesh-controller/handacts.go +++ b/cmd/mesh-controller/handacts.go @@ -84,6 +84,9 @@ var handActVerbs = []handActVerb{ "deliberate test, never a repair", DecidedFor: []string{causeDrill}}, // A drill: something broken on purpose to see the mesh raise and clear it. A person's test, never a // repair, however often it is run. + // A credential a provider's software issued, taken from the provider and kept for its consumers (novox/hq ADR + // 0305): the mesh working as decided, however often the software changes it. + {Verb: handActIssued, Decision: "a credential the provider's software issued, kept for its consumers (ADR 0305)"}, {Verb: "hand-act drill", Decision: "a drill is a person's deliberate test of the mesh, never a repair"}, // A person's decisions by design. {Verb: "retire approve", Decision: "nothing is retired past its bound without a person (ADR 0230)"}, diff --git a/cmd/mesh-controller/issue352_test.go b/cmd/mesh-controller/issue352_test.go index 951f8514..0e48b13b 100644 --- a/cmd/mesh-controller/issue352_test.go +++ b/cmd/mesh-controller/issue352_test.go @@ -25,6 +25,7 @@ import ( // another send reaches anchor, unreported; the gate still passes. And a send that moves a judged module // to another build supersedes the judging: no verdict, nothing put back. func TestReplay352(t *testing.T) { + judgeLeavesAsTheCore(t) t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn) t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging) } @@ -66,10 +67,11 @@ func testANewerSendDoesNotUnreportTheGatesOwn(t *testing.T) { // A plan's own first send waits while a release judges the same module on that machine with another build. func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) { + judgeLeavesAsTheCore(t) b := aBacklog(t) ctx := t.Context() advancePlans(ctx, b.open) // the release judges app c2 on anchor - _, _, err := gatedSend(ctx, b.open, "anchor", []inventory.CarriedMove{{Module: "app", Node: "anchor", From: "c2", To: "c3", Build: "build-app-c3"}}) + _, _, _, err := gatedSend(ctx, b.open, "anchor", []inventory.CarriedMove{{Module: "app", Node: "anchor", From: "c2", To: "c3", Build: "build-app-c3"}}) if !errors.Is(err, errWalkedElsewhere) || !strings.Contains(err.Error(), "release-") { t.Fatalf("a newer build of a judged module was sent under the release's gate: %v", err) } @@ -80,6 +82,7 @@ func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) { // A merge plan's judging is superseded the same way: another send moved its module on the first machine. func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) { + judgeLeavesAsTheCore(t) g := aGateMesh(t) ctx := t.Context() inv := g.open.inventory diff --git a/cmd/mesh-controller/issue459_test.go b/cmd/mesh-controller/issue459_test.go new file mode 100644 index 00000000..37f1782e --- /dev/null +++ b/cmd/mesh-controller/issue459_test.go @@ -0,0 +1,427 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "reflect" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// novox/hq issue 459, ADR 0299: since the merge window one walk carries every merge of its batch, so a stop of one +// group member through `delivery-stop` ended the walk of every other delivery in it. The stop now names the merges +// to leave out: the walk is ended, and every other merge it answered is walked again in a fresh batch without them +// — the stopped merge, and each later merge of its repository's branch, which contains it and ends with it. + +// endsItsLine ends, as the test ends, a line the controller's output left open (a walk's ask prints " tier N: " and +// the asked build the rest, which the tests' fake ask never prints): `go test -v` would put the test's "--- PASS" +// after it, and the store-tests step of merge-check.sh, which reads that marker at a line's start, would say the +// test never ran. +func endsItsLine(t *testing.T) { + t.Cleanup(func() { fmt.Println() }) +} + +// leaved is a kept merge of a repository, made at a moment. +func leaved(repository, commit string, made time.Time) inventory.BatchedMerge { + return inventory.BatchedMerge{Repository: repository, Branch: "main", Commit: commit, Merged: made, Heard: made} +} + +// The stopped merge is left out, and so is each later merge of its repository's branch, naming the merge it +// contains; an earlier merge of that repository and every other repository's merge are walked again. +func TestLeftOutAreTheStoppedMergeAndTheMergesThatContainIt(t *testing.T) { + c1 := leaved("novox/mesh-catalog", "c1aaaaaaaaaa", t0) + c2 := leaved("novox/mesh-catalog", "c2bbbbbbbbbb", t0.Add(10*time.Second)) + c3 := leaved("novox/mesh-catalog", "c3cccccccccc", t0.Add(20*time.Second)) + o1 := leaved("novox/one", "o1dddddddddd", t0.Add(5*time.Second)) + again, out, err := leaveOut([]inventory.BatchedMerge{c1, o1, c2, c3}, []string{"novox/Mesh-Catalog@c2bbbbbbbbbb"}) + if err != nil { + t.Fatal(err) + } + var walked []string + for _, m := range again { + walked = append(walked, m.Commit) + } + if !reflect.DeepEqual(walked, []string{c1.Commit, o1.Commit}) { + t.Fatalf("walked again: %v; want the earlier catalogue merge and the other repository's", walked) + } + want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: c2.Commit}, + {Repository: "novox/mesh-catalog", Commit: c3.Commit, Contains: c2.Commit}} + if !reflect.DeepEqual(out, want) { + t.Fatalf("left out: %+v; want %+v", out, want) + } + // A merge the walk does not answer is refused, naming it: nothing ends. + if _, _, err := leaveOut([]inventory.BatchedMerge{c1, o1}, []string{"novox/mesh-catalog@ffffffffffff"}); err == nil || + !strings.Contains(err.Error(), "ffffffffffff") { + t.Fatalf("a merge the walk does not answer was left out: %v", err) + } + if _, _, err := leaveOut([]inventory.BatchedMerge{c1}, []string{"no-commit"}); err == nil { + t.Fatal("a merge without its commit was taken") + } +} + +// moving is a kept merge that moved modules when it was heard. +func moving(m inventory.BatchedMerge, modules ...string) inventory.BatchedMerge { + m.Event, _ = json.Marshal(keptMerge{Moves: modules}) + return m +} + +// A merge of another repository that moves a module the stopped merge moves is left out too, naming the modules +// it shares: the module is built at its branch, which holds the stopped merge (review of #232). And so on, until +// none more is: a merge sharing a module with that one is left out as well. +func TestAMergeSharingAStoppedMergesModuleIsLeftOut(t *testing.T) { + stopped := moving(leaved("novox/mesh-catalog", "c1aaaaaaaaaa", t0), "app") + reader := moving(leaved("novox/one", "o1bbbbbbbbbb", t0.Add(time.Second)), "app", "one") + next := moving(leaved("novox/two", "t1cccccccccc", t0.Add(2*time.Second)), "one") + free := moving(leaved("novox/three", "h1dddddddddd", t0.Add(3*time.Second)), "three") + again, out, err := leaveOut([]inventory.BatchedMerge{stopped, reader, next, free}, []string{"novox/mesh-catalog@c1aaaaaaa"}) + if err != nil { + t.Fatal(err) + } + if len(again) != 1 || again[0].Commit != free.Commit { + t.Fatalf("walked again: %+v; want the merge that shares nothing alone", again) + } + want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: stopped.Commit}, + {Repository: "novox/one", Commit: reader.Commit, Contains: stopped.Commit, Shares: []string{"app"}}, + {Repository: "novox/two", Commit: next.Commit, Contains: stopped.Commit, Shares: []string{"one"}}} + if !reflect.DeepEqual(out, want) { + t.Fatalf("left out %+v; want %+v", out, want) + } +} + +// A walk withholds a module a stopped merge moved where it would build it only as a dependent; one a merge of its +// own batch moves is built, and empty tiers go. +func TestAWalkWithholdsAStoppedMergesDependents(t *testing.T) { + plan := inventory.Plan{Tiers: [][]string{{"one"}, {"app"}, {"notes", "app2"}}, + Modules: map[string]*inventory.PlanModule{"one": {}, "app": {}, "notes": {}, "app2": {}}} + got := withhold(&plan, []string{"app", "notes", "absent"}, []string{"notes"}) + if !reflect.DeepEqual(got, []string{"app"}) || !reflect.DeepEqual(plan.Tiers, [][]string{{"one"}, {"notes", "app2"}}) { + t.Fatalf("withheld %v, tiers %v", got, plan.Tiers) + } + if _, in := plan.Modules["app"]; in { + t.Fatal("a withheld module is still in the plan") + } +} + +// The seat verb passes the merges to leave out to the command. +func TestDeliveryStopTakesTheMergesToLeaveOut(t *testing.T) { + got, err := argvFor("delivery-stop", map[string]any{"plan": "plan-1", "why": "w", "by": "jochen", + "without": "novox/mesh-lab@b1,novox/mesh-lab@b2"}) + want := []string{"delivery", "stop", "plan-1", "--why", "w", "--by", "mesh-delivery for jochen", + "--without", "novox/mesh-lab@b1,novox/mesh-lab@b2"} + if err != nil || !reflect.DeepEqual(got, want) { + t.Fatalf("delivery-stop with without → %v %v, wanted %v", got, err, want) + } +} + +// A walk of the catalogue's merge and another repository's is stopped without the catalogue's: it ends stopped, +// names the batch that walks the other merge again and the merge it left out, and that batch is cut into a walk +// that builds the other repository's module alone. +func TestAStopWithoutAMergeWalksTheOthersAgain(t *testing.T) { + open := windowed(t) + endsItsLine(t) + asksRecorded(t) + ctx := t.Context() + cat := catalogueMerge("cat1aaaaaaaa", "app", t0) + one := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)) + hear(t, open, cat, t0.Add(time.Second)) + hear(t, open, one, t0.Add(6*time.Second)) + cutAt(t, open, t0.Add(2*time.Minute)) + ws, _ := walks(t, open) + if len(ws) != 1 || !ws[0].Open() { + t.Fatalf("no open walk to stop: %+v", ws) + } + stopped := ws[0] + p, said, err := stopWalkWithout(ctx, open.inventory, stopped.ID, "mesh-delivery for jochen", "its group stopped", + []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute)) + if err != nil { + t.Fatal(err) + } + if p.State != inventory.PlanFailed || p.Delivery.Stopped == "" || p.Delivery.WalkedAgainBy == "" || + !reflect.DeepEqual(p.Delivery.LeftOut, []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: cat.Commit}}) { + t.Fatalf("the stopped walk reads %s %+v", p.State, p.Delivery) + } + if said.WalkedAgainBy != p.Delivery.WalkedAgainBy || !reflect.DeepEqual(said.WalkedAgain, []string{"novox/one@" + one.Commit}) { + t.Fatalf("the stop answered %+v", said) + } + again, err := open.inventory.MergesOf(ctx, p.Delivery.WalkedAgainBy) + if err != nil || len(again) != 1 || again[0].Commit != one.Commit { + t.Fatalf("the fresh batch answers %+v (%v); want the other repository's merge alone", again, err) + } + if left, _ := open.inventory.MergesOf(ctx, stopped.ID); len(left) != 1 || left[0].Commit != cat.Commit { + t.Fatalf("the stopped walk still answers %+v; want the left-out merge alone", left) + } + cutAt(t, open, t0.Add(4*time.Minute)) + fresh, err := open.inventory.PlanByID(ctx, p.Delivery.WalkedAgainBy) + if err != nil || fresh.Batch() || !fresh.Open() { + t.Fatalf("the fresh batch was not cut into a walk: %s %v", fresh.State, err) + } + if _, in := fresh.Modules["one"]; !in { + t.Fatalf("the fresh walk does not build one: %v", fresh.Modules) + } + if _, in := fresh.Modules["app"]; in { + t.Fatalf("the fresh walk builds app, whose merge was left out: %v", fresh.Modules) + } + if len(fresh.Delivery.Merges) != 1 || fresh.Delivery.Merges[0].Commit != one.Commit { + t.Fatalf("the fresh walk answers %+v", fresh.Delivery.Merges) + } + // A stopped walk starts no search of its own (ADR 0276 decision 3 is for a failed one). + cutAt(t, open, t0.Add(5*time.Minute)) + if alone, _ := open.inventory.AloneMerges(ctx); len(alone) != 0 { + t.Fatalf("a merge of the stopped walk waits to be walked alone: %+v", alone) + } +} + +// A later merge of the stopped merge's repository contains it: it is left out too, named with the merge it +// contains, and only the other repository's merge is walked again. With nothing left to walk, the walk is only +// ended. A merge the walk does not answer is refused, and nothing ends. +func TestAStopWithoutAMergeEndsTheMergesThatContainIt(t *testing.T) { + open := windowed(t) + endsItsLine(t) + asksRecorded(t) + ctx := t.Context() + first := catalogueMerge("cat1aaaaaaaa", "app", t0) + later := catalogueMerge("cat2cccccccc", "notes", t0.Add(10*time.Second)) + one := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)) + hear(t, open, first, t0.Add(time.Second)) + hear(t, open, one, t0.Add(6*time.Second)) + hear(t, open, later, t0.Add(11*time.Second)) + cutAt(t, open, t0.Add(2*time.Minute)) + ws, _ := walks(t, open) + walk := ws[0] + + if _, _, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "x", + []string{"novox/mesh-catalog@ffffffffffff"}, t0.Add(3*time.Minute)); err == nil { + t.Fatal("a stop leaving out a merge the walk does not answer was taken") + } + if p, _ := open.inventory.PlanByID(ctx, walk.ID); !p.Open() { + t.Fatalf("a refused stop ended the walk: %s", p.State) + } + + p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped", + []string{"novox/mesh-catalog@" + first.Commit}, t0.Add(3*time.Minute)) + if err != nil { + t.Fatal(err) + } + want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: first.Commit}, + {Repository: "novox/mesh-catalog", Commit: later.Commit, Contains: first.Commit}} + if !reflect.DeepEqual(p.Delivery.LeftOut, want) || !reflect.DeepEqual(said.LeftOut, want) { + t.Fatalf("left out %+v, said %+v; want %+v", p.Delivery.LeftOut, said.LeftOut, want) + } + if !strings.Contains(p.Note, later.Commit[:8]) || !strings.Contains(p.Note, p.Delivery.WalkedAgainBy) { + t.Fatalf("the stopped walk's note does not name the merge ended with it and the fresh batch: %q", p.Note) + } + again, _ := open.inventory.MergesOf(ctx, p.Delivery.WalkedAgainBy) + if len(again) != 1 || again[0].Commit != one.Commit { + t.Fatalf("walked again: %+v; want the other repository's merge alone", again) + } + + // Nothing left once the other repository's merge is left out too: the fresh walk is stopped, and nothing more. + cutAt(t, open, t0.Add(4*time.Minute)) + fresh, _ := open.inventory.PlanByID(ctx, p.Delivery.WalkedAgainBy) + q, said, err := stopWalkWithout(ctx, open.inventory, fresh.ID, "mesh-delivery for jochen", "that one too", + []string{"novox/one@" + one.Commit}, t0.Add(5*time.Minute)) + if err != nil || q.State != inventory.PlanFailed || q.Delivery.WalkedAgainBy != "" || said.WalkedAgainBy != "" { + t.Fatalf("a stop leaving nothing to walk: %s %+v %+v %v", q.State, q.Delivery, said, err) + } + if _, bs := walks(t, open); len(bs) != 0 { + t.Fatalf("a batch of nothing was made: %+v", bs) + } +} + +// A merge heard while the walk ran waits in the open batch, at a later commit of the same repository. The merge +// walked again joins that batch, never a batch of its own that would walk the older commit after the newer one +// (review of #232): its window closes, and its walk is at the newer commit, answering both. +func TestAStopWalksTheOthersAgainInTheOpenBatch(t *testing.T) { + open := windowed(t) + endsItsLine(t) + asksRecorded(t) + ctx := t.Context() + cat := catalogueMerge("cat1aaaaaaaa", "app", t0) + one1 := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)) + hear(t, open, cat, t0.Add(time.Second)) + hear(t, open, one1, t0.Add(6*time.Second)) + cutAt(t, open, t0.Add(2*time.Minute)) + ws, _ := walks(t, open) + walk := ws[0] + one2 := repoMerge("one", "one2cccccccc", t0.Add(3*time.Minute)) + hear(t, open, one2, t0.Add(3*time.Minute)) + _, bs := walks(t, open) + if len(bs) != 1 { + t.Fatalf("the merge heard while the walk ran is in %d batches", len(bs)) + } + waiting := bs[0] + p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped", + []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second)) + if err != nil { + t.Fatal(err) + } + if p.Delivery.WalkedAgainBy != waiting.ID || said.WalkedAgainBy != waiting.ID { + t.Fatalf("walked again by %q (said %q); want the open batch %s", p.Delivery.WalkedAgainBy, said.WalkedAgainBy, waiting.ID) + } + if _, bs := walks(t, open); len(bs) != 1 { + t.Fatalf("%d batches; want the open one alone", len(bs)) + } + b, _ := open.inventory.PlanByID(ctx, waiting.ID) + if b.Delivery.Batch == nil || !b.Delivery.Batch.Closed || !slices.Contains(b.Delivery.Batch.Withheld, "app") { + t.Fatalf("the open batch reads %+v; want its window closed and app withheld", b.Delivery.Batch) + } + // Its window closed by the stop, not by the clock: cut at once. + cutAt(t, open, t0.Add(3*time.Minute+11*time.Second)) + fresh, _ := open.inventory.PlanByID(ctx, waiting.ID) + if fresh.Batch() || fresh.CommitOf("novox/one") != one2.Commit { + t.Fatalf("the batch is %s at %q; want it cut, walking the newer commit", fresh.State, fresh.CommitOf("novox/one")) + } + var answered []string + for _, m := range fresh.Delivery.Merges { + answered = append(answered, m.Commit+">"+m.Carried) + } + if !slices.Contains(answered, one1.Commit+">"+one2.Commit) || !slices.Contains(answered, one2.Commit+">") { + t.Fatalf("the walk answers %v; want the earlier merge carried by the later", answered) + } +} + +// A stop names what the stopped walk already sent and nothing walks again, on which machines; and a later merge of +// the stopped merge's repository that waits in a batch, which will deliver the stopped change. A stopped walk +// that walked its merges again is refused a retry, saying why. +func TestAStopNamesWhatStaysOnTheMachinesAndWhatStillCarriesIt(t *testing.T) { + open := windowed(t) + endsItsLine(t) + asksRecorded(t) + ctx := t.Context() + cat := catalogueMerge("cat1aaaaaaaa", "app", t0) + one1 := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)) + hear(t, open, cat, t0.Add(time.Second)) + hear(t, open, one1, t0.Add(6*time.Second)) + cutAt(t, open, t0.Add(2*time.Minute)) + ws, _ := walks(t, open) + walk := ws[0] + sent := t0.Add(150 * time.Second) + for name, m := range walk.Modules { + if name == "app" || name == "one" { + m.First, m.FirstAt = []string{"anchor"}, &sent + } + } + if err := open.inventory.SavePlan(ctx, &walk); err != nil { + t.Fatal(err) + } + cat2 := catalogueMerge("cat2dddddddd", "notes", t0.Add(3*time.Minute)) + hear(t, open, cat2, t0.Add(3*time.Minute)) + _, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped", + []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second)) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(said.Kept, []inventory.PlanKept{{Module: "app", Machines: []string{"anchor"}}}) { + t.Fatalf("kept %+v; want app on anchor alone (one is walked again)", said.Kept) + } + p, _ := open.inventory.PlanByID(ctx, walk.ID) + if !reflect.DeepEqual(p.Delivery.Kept, said.Kept) || !strings.Contains(p.Note, "app on anchor") { + t.Fatalf("the stopped walk's record does not name what stays: %+v %q", p.Delivery.Kept, p.Note) + } + if len(said.StillCarriedBy) != 1 || !strings.Contains(said.StillCarriedBy[0], cat2.Commit) { + t.Fatalf("still carried by %v; want the later catalogue merge waiting in the batch", said.StillCarriedBy) + } + if _, err := retryPlan(ctx, open, walk.ID); err == nil || !strings.Contains(err.Error(), "was stopped without") { + t.Fatalf("a walk stopped without some merges was retried, or refused for another reason: %v", err) + } +} + +// A batch not yet cut is stopped without a merge: the merge goes to a stopped record of its own, the batch goes on +// with the others and withholds the stopped merge's modules; with every merge left out, the batch is stopped. +func TestAStopWithoutAMergeLeavesItOutOfABatch(t *testing.T) { + open := windowed(t) + endsItsLine(t) + asksRecorded(t) + ctx := t.Context() + cat := catalogueMerge("cat1aaaaaaaa", "app", t0) + one := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)) + hear(t, open, cat, t0.Add(time.Second)) + hear(t, open, one, t0.Add(6*time.Second)) + _, bs := walks(t, open) + b := bs[0] + r, said, err := stopWalkWithout(ctx, open.inventory, b.ID, "mesh-delivery for jochen", "its group stopped", + []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(10*time.Second)) + if err != nil { + t.Fatal(err) + } + if said.WalkedAgainBy != b.ID || said.Stopped == b.ID { + t.Fatalf("said %+v; want a stopped record of its own, and the batch walking the others", said) + } + if left, _ := open.inventory.MergesOf(ctx, said.Stopped); len(left) != 1 || left[0].Commit != cat.Commit { + t.Fatalf("the stopped record holds %+v", left) + } + if stopped, _ := open.inventory.PlanByID(ctx, said.Stopped); stopped.State != inventory.PlanFailed || + stopped.Delivery.Stopped == "" || stopped.Delivery.WalkedAgainBy != b.ID { + t.Fatalf("the stopped record reads %s %+v", stopped.State, stopped.Delivery) + } + _ = r + cutAt(t, open, t0.Add(3*time.Minute)) + w, _ := open.inventory.PlanByID(ctx, b.ID) + if _, in := w.Modules["app"]; in || w.Batch() { + t.Fatalf("the batch's walk is %s and builds %v; want it cut without app", w.State, w.Modules) + } + // A batch whose every merge is left out is stopped itself. + hear(t, open, catalogueMerge("cat3eeeeeeee", "notes", t0.Add(4*time.Minute)), t0.Add(4*time.Minute)) + _, bs = walks(t, open) + last := bs[0] + if _, said, err := stopWalkWithout(ctx, open.inventory, last.ID, "mesh-delivery for jochen", "x", + []string{"novox/mesh-catalog@cat3eeeeeeee"}, t0.Add(4*time.Minute+time.Second)); err != nil || said.Stopped != last.ID { + t.Fatalf("a batch left with nothing: %+v %v", said, err) + } + if p, _ := open.inventory.PlanByID(ctx, last.ID); p.State != inventory.PlanFailed || p.Batch() { + t.Fatalf("the emptied batch is %s", p.State) + } +} + +// The stopped walk, the batch and the merges moved are one act (ADR 0299 decision 7): a plan written by somebody +// else since it was read refuses all of it, and no merge moves, no plan is written. +func TestSavingPlansAndMovingMergesIsOneAct(t *testing.T) { + open := windowed(t) + endsItsLine(t) + ctx := t.Context() + hear(t, open, catalogueMerge("cat1aaaaaaaa", "app", t0), t0.Add(time.Second)) + _, bs := walks(t, open) + stale := bs[0] + moved := stale + moved.Note = "written by somebody else" + if err := open.inventory.SavePlan(ctx, &moved); err != nil { + t.Fatal(err) + } + fresh := inventory.Plan{ID: "plan-fresh", Repository: "novox/mesh-catalog", Branch: "main", Commit: "x", Created: t0, + State: inventory.PlanAssembling, Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{}} + err := open.inventory.SavePlansMoving(ctx, []*inventory.Plan{&fresh, &stale}, + []inventory.MergeMove{{Repository: "novox/mesh-catalog", Commit: "cat1aaaaaaaa", Plan: "plan-fresh"}}) + if !errors.Is(err, inventory.ErrPlanMoved) { + t.Fatalf("a stale plan in the act: %v", err) + } + if _, err := open.inventory.PlanByID(ctx, "plan-fresh"); err == nil { + t.Fatal("the new plan was written though the act was refused") + } + if m, _, _ := open.inventory.MergeOf(ctx, "novox/mesh-catalog", "cat1aaaaaaaa"); m.Plan != stale.ID { + t.Fatalf("the merge moved to %q though the act was refused", m.Plan) + } +} + +// What a stopped walk leaves on the machines names every machine its sends reached (review of #232): a module sent +// with no first machine reached every machine running it; one sent first reached those, and the rest it was sent +// to; a module a merge walked again moves is sent again, and is not named. +func TestKeptNamesEveryMachineASendReached(t *testing.T) { + at := t0 + p := inventory.Plan{Modules: map[string]*inventory.PlanModule{ + "together": {SentAt: &at}, + "first": {First: []string{"anchor"}, FirstAt: &at, Rest: map[string]inventory.SentDeclaration{"laptop": {}}}, + "again": {SentAt: &at}, + "unsent": {}, + }} + got := keptOf(p, []string{"again"}, map[string][]string{"together": {"laptop", "anchor"}, "again": {"anchor"}}) + want := []inventory.PlanKept{{Module: "first", Machines: []string{"anchor", "laptop"}}, + {Module: "together", Machines: []string{"anchor", "laptop"}}} + if !reflect.DeepEqual(got, want) { + t.Fatalf("kept %+v; want %+v", got, want) + } +} diff --git a/cmd/mesh-controller/issued.go b/cmd/mesh-controller/issued.go new file mode 100644 index 00000000..1f1b348c --- /dev/null +++ b/cmd/mesh-controller/issued.go @@ -0,0 +1,272 @@ +package main + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/secrets" +) + +// `secret issued `: the credential a provider's software issued, taken from the provider and kept as +// the provision's one credential for every consumer (novox/hq ADR 0305, issue 526). +// +// **The value comes from the provider, never from whoever asks.** A module served by its machine's runtime calls +// under the runtime's one bus account, which carries that machine's agents' calls too, so a value handed to a +// verb could be anybody's. This line carries none: it asks the provider's own tool, on the provider's machine, in +// the controller's own name — the subject that names its caller, which the bus lets the controller alone publish +// (issue 365) — with a sealing key made for this one call. The tool answers only the controller, the value sealed +// to that key; the controller opens it, seals it to every holder and forgets it, as the desk's value is (ADR 0277 +// §23). Asking again fetches the same value again, and a value every holder already holds is not sealed anew. +// +// **What this trusts: the provider's machine.** The answer is taken from whoever answers the provider's tool +// subject on the provider's machine first. On that machine its runtime serves the subject for every module it +// carries, so anything that runtime runs, or anything else granted that subject there, could answer before the +// provider does and hand the controller a value of its own. ADR 0305 accepts that by trusting the provider's +// machine, as the provider itself is trusted; the bus keeps every other machine from answering it. + +// handActIssued is the verb a change of an issued value is recorded under: the mesh working as decided. +const handActIssued = "secret issued" + +// issuedAsk is how long the provider's tool is given to answer. +const issuedAsk = 20 * time.Second + +// issuedProvider is one assignment of a module whose offer says its software issues the provision's credential. +type issuedProvider struct { + Node, Module, Tool string +} + +// issuedTaker is the line's work, with what it reaches handed in: the store, the bus and the sends. +type issuedTaker struct { + // providers are the assignments to ask, and whether any module offers the provision so at all. + providers func(provision string) ([]issuedProvider, bool, error) + ask func(p issuedProvider, args map[string]any) (link.Answer, error) + keep func(provision, provider, value string) (inventory.IssuedKept, error) + record func(act link.HandAct) error + send func(machines []string) error +} + +// take asks every provider of the provision for its software's value and keeps each; it answers what it did, in +// words that never carry the value or its fingerprint. +func (t issuedTaker) take(provision string) (string, error) { + providers, offered, err := t.providers(provision) + if err != nil { + return "", err + } + if !offered { + return "", fmt.Errorf("no module offers %q with a credential its software issues (\"credential\": "+ + "{\"issued-by\": \"software\", \"tool\": …}, ADR 0305), so there is nothing to take from a provider; "+ + "nothing was asked", provision) + } + if len(providers) == 0 { + return fmt.Sprintf("no consumer holds %s from a provider whose software issues it yet, so there is nothing "+ + "to seal; a consumer that binds holds a value the mesh made until the provider next asks", provision), nil + } + var said []string + var failed []string + for _, p := range providers { + words, err := t.takeFrom(provision, p) + if err != nil { + failed = append(failed, fmt.Sprintf("%s on %s: %v", p.Module, p.Node, err)) + continue + } + said = append(said, words) + } + out := strings.Join(said, "\n") + if len(failed) > 0 { + return out, fmt.Errorf("%s was not taken from every provider:\n %s", provision, strings.Join(failed, "\n ")) + } + return out, nil +} + +// takeFrom is one provider: asked, opened, kept, recorded and sent when it changed. +func (t issuedTaker) takeFrom(provision string, p issuedProvider) (string, error) { + public, private, err := secrets.Keypair() + if err != nil { + return "", fmt.Errorf("no key could be made to take the value: %w", err) + } + answer, err := t.ask(p, map[string]any{"provision": provision, "seal_to": public}) + if err != nil { + return "", fmt.Errorf("its tool %s could not be asked: %w", p.Tool, err) + } + if answer.Error != "" { + return "", fmt.Errorf("its tool %s refused: %s", p.Tool, answer.Error) + } + var given struct { + Sealed string `json:"sealed"` + } + if err := unmarshalAnswer(answer, &given); err != nil || given.Sealed == "" { + return "", fmt.Errorf("its tool %s answered no sealed value", p.Tool) + } + opened, err := secrets.Open(private, given.Sealed) + if err != nil { + // Never the value, never what failed to open: only that it was not sealed to this call. + return "", fmt.Errorf("its tool %s answered a value not sealed to this call; nothing was taken", p.Tool) + } + value := strings.TrimSpace(string(opened)) + for i := range opened { + opened[i] = 0 + } + if value == "" { + return "", fmt.Errorf("its tool %s answered an empty value; nothing was taken", p.Tool) + } + kept, err := t.keep(provision, p.Node, value) + value = "" + if err != nil { + return "", err + } + holders := make([]string, 0, len(kept.Holders)) + for _, h := range kept.Holders { + holders = append(holders, fmt.Sprintf("%s on %s%s", h.ConsumerModule, h.Consumer, asLocal(h.Local))) + } + words := "" + if len(kept.Unsealable) > 0 { + var names []string + for _, h := range kept.Unsealable { + names = append(names, fmt.Sprintf("%s on %s", h.ConsumerModule, h.Consumer)) + } + words = fmt.Sprintf("\n not sealed to %s: no sealing key on that machine", strings.Join(names, ", ")) + } + if !kept.Changed { + return fmt.Sprintf("%s from %s on %s: every holder already holds the value its software issued (%s); "+ + "nothing was sent%s", provision, p.Module, p.Node, strings.Join(holders, ", "), words), nil + } + sealedTo := make([]string, 0, len(kept.Sealed)) + for _, h := range kept.Sealed { + sealedTo = append(sealedTo, fmt.Sprintf("%s on %s%s", h.ConsumerModule, h.Consumer, asLocal(h.Local))) + } + recorded := "" + if err := t.record(link.HandAct{Verb: handActIssued, Args: []string{provision}, + Why: fmt.Sprintf("%s on %s gave the credential its software issued for %s; sealed to %s", p.Module, p.Node, + provision, strings.Join(sealedTo, ", ")), + Cause: "issued-by-software"}); err != nil { + recorded = fmt.Sprintf("\n this was NOT recorded in the hand-act log, and is done anyway: %v", err) + } + if err := t.send(kept.Machines); err != nil { + return "", fmt.Errorf("%s from %s on %s is sealed to %s and could not be sent: %w; `push --behind` sends it", + provision, p.Module, p.Node, strings.Join(sealedTo, ", "), err) + } + return fmt.Sprintf("%s from %s on %s: the value its software issued is sealed to %s; sent %s, and each of "+ + "them starts again on it%s%s", provision, p.Module, p.Node, strings.Join(sealedTo, ", "), + strings.Join(kept.Machines, ", "), words, recorded), nil +} + +// issuedProvidersOf are the assignments to ask for a provision: every machine providing it to a pair the mesh +// holds, and on it the assigned module whose offer says its software issues the credential (ADR 0305 §1). +func issuedProvidersOf(ctx context.Context, inv *inventory.Inventory, provision string) ([]issuedProvider, bool, error) { + shelf, err := inv.Catalogue(ctx) + if err != nil { + return nil, false, err + } + if !offersIssued(shelf, provision) { + return nil, false, nil + } + holders, err := inv.HoldersOf(ctx, provision, "") + if err != nil { + return nil, true, err + } + nodes := map[string]bool{} + for _, h := range holders { + nodes[h.Provider] = true + } + names := make([]string, 0, len(nodes)) + for n := range nodes { + names = append(names, n) + } + sort.Strings(names) + var out []issuedProvider + for _, node := range names { + assigned, err := inv.Assigned(ctx, node) + if err != nil { + return nil, true, err + } + out = append(out, issuedOn(shelf, node, assigned, provision)...) + } + return out, true, nil +} + +// offersIssued is whether any module offers the provision with a credential its software issues. +func offersIssued(shelf map[string]catalogue.Manifest, provision string) bool { + for _, m := range shelf { + if _, issued := m.IssuedCredentialOf(provision); issued { + return true + } + } + return false +} + +// issuedOn are the modules assigned to one machine whose offer says their software issues the provision's +// credential, each with the tool that answers it. +func issuedOn(shelf map[string]catalogue.Manifest, node string, assigned []string, provision string) []issuedProvider { + var out []issuedProvider + for _, module := range assigned { + if tool, issued := shelf[module].IssuedCredentialOf(provision); issued { + out = append(out, issuedProvider{Node: node, Module: module, Tool: tool}) + } + } + return out +} + +// issuedRefusesRotation refuses `rotate` of a provision its provider's software issues (ADR 0305 §9): a value the +// mesh made would replace one the software holds, and every consumer would hold a dead credential until the +// provider next asks. +func issuedRefusesRotation(shelf map[string]catalogue.Manifest, provision string) error { + if !offersIssued(shelf, provision) { + return nil + } + return fmt.Errorf("%s's credential is issued by its provider's software, which the mesh cannot make or "+ + "rotate (ADR 0305): change it in that software, and `secret issued %s` takes the value it holds now. "+ + "Nothing was rotated", provision, provision) +} + +// secretIssued is `secret issued `, on this controller's stores and bus. +func secretIssued(ctx context.Context, args []string) error { + if len(args) != 1 || args[0] == "" || strings.HasPrefix(args[0], "-") { + return errors.New("secret issued ") + } + provision := args[0] + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + t := issuedTaker{ + providers: func(provision string) ([]issuedProvider, bool, error) { + return issuedProvidersOf(ctx, open.inventory, provision) + }, + ask: func(p issuedProvider, args map[string]any) (link.Answer, error) { + var answer link.Answer + err := onTheBus(func(conn *nats.Conn) error { + var err error + answer, err = link.AskModuleToolOnAs(ctx, conn, broker.ControllerName, p.Module, p.Tool, p.Node, + args, issuedAsk) + return err + }) + return answer, err + }, + keep: func(provision, provider, value string) (inventory.IssuedKept, error) { + return open.inventory.KeepIssuedSecret(ctx, provision, provider, value) + }, + record: func(act link.HandAct) error { + return onTheBus(func(conn *nats.Conn) error { + _, err := link.RecordHandAct(ctx, conn, act) + return err + }) + }, + send: func(machines []string) error { return sendTo(ctx, open, machines) }, + } + words, err := t.take(provision) + if words != "" { + fmt.Println(words) + } + return err +} diff --git a/cmd/mesh-controller/issued_test.go b/cmd/mesh-controller/issued_test.go new file mode 100644 index 00000000..c9b1b6c8 --- /dev/null +++ b/cmd/mesh-controller/issued_test.go @@ -0,0 +1,218 @@ +package main + +import ( + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/secrets" +) + +// The `issued` verb runs `secret issued ` and nothing else, and that line is the one `secret` line +// beside rotate and the desk's that a verb may run (novox/hq ADR 0305 §3, §10, issue 526). +func TestTheIssuedVerbRunsTheOneLineThatCarriesNoValue(t *testing.T) { + argv, err := argvFor("issued", map[string]any{"provision": "plex-api"}) + if err != nil { + t.Fatal(err) + } + if strings.Join(argv, " ") != "secret issued plex-api" { + t.Fatalf("the verb composed %q", argv) + } + if _, err := argvFor("issued", map[string]any{}); err == nil { + t.Fatal("the verb ran with no provision") + } + if _, err := argvFor("issued", map[string]any{"provision": "plex-api", "secret": "a value"}); err == nil { + t.Fatal("the verb took a value beside the provision") + } + for _, line := range [][]string{ + {"secret", "issued", "plex-api", "--from", "/tmp/x"}, + {"secret", "issued", "--from"}, + {"secret", "issued"}, + {"secret", "accept", "ace", "kometa", "plex-api", "--provider", "ace"}, + {"secret", "recover", "ace", "kometa", "plex-api"}, + } { + if err := terminalOnly(line); err == nil { + t.Errorf("%q ran through a verb", line) + } + } + if err := terminalOnly([]string{"secret", "issued", "plex-api"}); err != nil { + t.Fatalf("the issued line was refused through its verb: %v", err) + } + found := false + for _, v := range catalogue.ControllerVerbs { + if v.Name == "issued" { + found = true + } + } + if !found { + t.Fatal("the controller's seat does not declare the issued verb") + } +} + +// A provider's tool answering the value sealed to the call's key: what a provider does (ADR 0305 §2). +func issuedSealed(t *testing.T, value string) func(issuedProvider, map[string]any) (link.Answer, error) { + return func(_ issuedProvider, args map[string]any) (link.Answer, error) { + key, _ := args["seal_to"].(string) + if key == "" { + t.Fatal("the provider was asked without a key to seal to") + } + if _, carried := args["value"]; carried { + t.Fatal("the ask carried a value") + } + sealed, err := secrets.Seal(key, []byte(value)) + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(map[string]string{"sealed": sealed}) + return link.Answer{Result: raw}, nil + } +} + +type issuedRecorder struct { + kept []string + acts []link.HandAct + sent [][]string + again bool // the value is held by every holder already +} + +func (r *issuedRecorder) taker(t *testing.T, ask func(issuedProvider, map[string]any) (link.Answer, error)) issuedTaker { + return issuedTaker{ + providers: func(provision string) ([]issuedProvider, bool, error) { + return []issuedProvider{{Node: "ace", Module: "plex", Tool: "plex_issued_credential"}}, true, nil + }, + ask: ask, + keep: func(provision, provider, value string) (inventory.IssuedKept, error) { + r.kept = append(r.kept, value) + holders := []inventory.Holder{{Provision: provision, Consumer: "ace", ConsumerModule: "kometa", Provider: provider}, + {Provision: provision, Consumer: "ace", ConsumerModule: "tautulli", Provider: provider}} + if r.again { + return inventory.IssuedKept{Holders: holders}, nil + } + return inventory.IssuedKept{Changed: true, Holders: holders, Sealed: holders, Machines: []string{"ace"}}, nil + }, + record: func(act link.HandAct) error { r.acts = append(r.acts, act); return nil }, + send: func(machines []string) error { r.sent = append(r.sent, machines); return nil }, + } +} + +// The value the provider's tool sealed to the call is kept, recorded and sent, and the answer never says it. +func TestAnIssuedValueIsTakenSealedFromTheProviderAndSent(t *testing.T) { + var r issuedRecorder + words, err := r.taker(t, issuedSealed(t, "the-servers-own-token")).take("plex-api") + if err != nil { + t.Fatal(err) + } + if len(r.kept) != 1 || r.kept[0] != "the-servers-own-token" { + t.Fatalf("the provider's value was not kept: %d kept", len(r.kept)) + } + if len(r.sent) != 1 || strings.Join(r.sent[0], ",") != "ace" { + t.Fatalf("the holders' machines were not sent: %v", r.sent) + } + if len(r.acts) != 1 || r.acts[0].Verb != handActIssued { + t.Fatalf("the change was not recorded in the hand-act log: %+v", r.acts) + } + for _, said := range []string{words, r.acts[0].Why, strings.Join(r.acts[0].Args, " ")} { + if strings.Contains(said, "the-servers-own-token") || strings.Contains(said, inventory.IssuedStamp("the-servers-own-token")) { + t.Fatalf("the value or its fingerprint was said: %q", said) + } + } + if !strings.Contains(words, "kometa on ace") || !strings.Contains(words, "tautulli on ace") { + t.Fatalf("the answer does not name the holders: %q", words) + } +} + +// The same value again writes nothing, records nothing and sends nobody (ADR 0305 §5). +func TestAnIssuedValueEveryHolderHoldsSendsNobody(t *testing.T) { + r := issuedRecorder{again: true} + words, err := r.taker(t, issuedSealed(t, "the-servers-own-token")).take("plex-api") + if err != nil { + t.Fatal(err) + } + if len(r.sent) != 0 || len(r.acts) != 0 { + t.Fatalf("an unchanged value was sent or recorded: %v %v", r.sent, r.acts) + } + if !strings.Contains(words, "nothing was sent") { + t.Fatalf("the answer does not say nothing was sent: %q", words) + } +} + +// A value not sealed to the call's key is not taken, nor is a refusal, nor an empty answer (ADR 0305 §2). +func TestOnlyAValueSealedToTheCallIsTaken(t *testing.T) { + other, _, _ := secrets.Keypair() + elsewhere, _ := secrets.Seal(other, []byte("somebody's value")) + raw, _ := json.Marshal(map[string]string{"sealed": elsewhere}) + plain, _ := json.Marshal(map[string]string{"value": "the-servers-own-token"}) + for name, ask := range map[string]func(issuedProvider, map[string]any) (link.Answer, error){ + "sealed to another key": func(issuedProvider, map[string]any) (link.Answer, error) { return link.Answer{Result: raw}, nil }, + "answered in the clear": func(issuedProvider, map[string]any) (link.Answer, error) { return link.Answer{Result: plain}, nil }, + "refused": func(issuedProvider, map[string]any) (link.Answer, error) { + return link.Answer{Error: "the controller alone"}, nil + }, + "not reached": func(issuedProvider, map[string]any) (link.Answer, error) { + return link.Answer{}, errors.New("503 no responders") + }, + "sealed empty": issuedSealed(t, " "), + } { + var r issuedRecorder + if _, err := r.taker(t, ask).take("plex-api"); err == nil { + t.Errorf("%s: taken", name) + } + if len(r.kept) != 0 || len(r.sent) != 0 { + t.Errorf("%s: something was kept or sent", name) + } + } +} + +// A provision no offer says its software issues is refused before anything is asked. +func TestAProvisionNoOfferSaysIsIssuedIsRefused(t *testing.T) { + asked := false + taker := issuedTaker{ + providers: func(string) ([]issuedProvider, bool, error) { return nil, false, nil }, + ask: func(issuedProvider, map[string]any) (link.Answer, error) { + asked = true + return link.Answer{}, nil + }, + } + if _, err := taker.take("postgres-database"); err == nil || !strings.Contains(err.Error(), "ADR 0305") { + t.Fatalf("expected a refusal naming the record, got %v", err) + } + if asked { + t.Fatal("a provider was asked for a provision no offer says its software issues") + } +} + +// Which assignments are asked: the modules on the providing machine whose offer says so, with their tool. +func TestTheProvidersAskedAreTheOffersThatSaySo(t *testing.T) { + shelf := map[string]catalogue.Manifest{ + "plex": {Module: "plex", Provides: []catalogue.Offer{{Name: "plex-api", Credential: &catalogue.OfferCredential{ + IssuedBy: catalogue.IssuedBySoftware, Tool: "plex_issued_credential"}}}}, + "jellyfin": {Module: "jellyfin", Provides: []catalogue.Offer{{Name: "plex-api"}}}, + } + got := issuedOn(shelf, "ace", []string{"jellyfin", "plex", "kometa"}, "plex-api") + if len(got) != 1 || got[0] != (issuedProvider{Node: "ace", Module: "plex", Tool: "plex_issued_credential"}) { + t.Fatalf("the providers asked: %+v", got) + } + if err := issuedRefusesRotation(shelf, "plex-api"); err == nil || !strings.Contains(err.Error(), "secret issued plex-api") { + t.Fatalf("rotate of an issued provision was not refused with the way to take it: %v", err) + } + if err := issuedRefusesRotation(shelf, "postgres-database"); err != nil { + t.Fatalf("rotate of an ordinary provision was refused: %v", err) + } +} + +// A change of an issued value is the mesh working as decided, never a repair (ADR 0305 §8). +func TestAnIssuedValueIsADecisionInTheHandActLog(t *testing.T) { + for _, v := range handActVerbs { + if v.Verb == handActIssued { + if v.Decision == "" { + t.Fatal("secret issued counts as a repair") + } + return + } + } + t.Fatal("secret issued is not in the hand-act table") +} diff --git a/cmd/mesh-controller/last_send_test.go b/cmd/mesh-controller/last_send_test.go new file mode 100644 index 00000000..0fecd717 --- /dev/null +++ b/cmd/mesh-controller/last_send_test.go @@ -0,0 +1,160 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// novox/hq issue 485: the mesh records every send — sender, lease epoch, generation, modules — and only `plan` +// said it. `status` says per machine when it was last sent a declaration and by whom, in one line; `node show` +// says the last send in full and what the machine answered; and `status --json` carries it as fields, since +// the store's reader may not read the controller's tables and these verbs are the only window onto them. + +func aLastSend(answer inventory.SendAnswer) inventory.Send { + sent := time.Date(2026, 10, 11, 18, 40, 5, 0, time.Local) + return inventory.Send{NodeName: "laptop", Sequence: 12, Epoch: 57, + Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12", + Modules: []string{"docker", "sudo"}, SentAt: sent, Recorded: true, Answer: answer} +} + +func TestStatusSaysEachMachinesLastSendInOneLine(t *testing.T) { + at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.Local) + asked := answers{ + nodes: []inventory.Node{{Name: "anchor", LastSeen: time.Now()}, {Name: "laptop", LastSeen: time.Now()}}, + lastSent: map[string]inventory.Send{"laptop": aLastSend(inventory.SendAnswer{ + Outcome: inventory.OutcomeApplied, At: &at})}, + } + shown := printed(t, func() error { return printStatus(asked) }) + var laptop, anchor []string + for _, line := range strings.Split(shown, "\n") { + fields := strings.Fields(line) + if len(fields) > 0 && fields[0] == "laptop" { + laptop = append(laptop, line) + } + if len(fields) > 0 && fields[0] == "anchor" { + anchor = append(anchor, line) + } + } + if len(laptop) != 1 || !strings.Contains(laptop[0], "2026-10-11 18:40") || + !strings.Contains(laptop[0], "by a one-shot push by jochen at a shell on anchor") || + !strings.Contains(laptop[0], "generation 40") || !strings.Contains(laptop[0], "applied") { + t.Fatalf("status does not say laptop's last send in one line:\n%s", shown) + } + if len(anchor) != 1 || !strings.Contains(anchor[0], "no send recorded") { + t.Fatalf("status does not say anchor has no send recorded, in one line:\n%s", shown) + } +} + +func TestStatusSaysTheLastSendsCouldNotBeRead(t *testing.T) { + asked := answers{nodes: []inventory.Node{{Name: "laptop", LastSeen: time.Now()}}, + lastSentUnread: "the store went away"} + if shown := printed(t, func() error { return printStatus(asked) }); !strings.Contains(shown, "the store went away") { + t.Fatalf("a record of sends that could not be read is not said:\n%s", shown) + } + if asked.well() { + t.Error("a mesh whose sends could not be read is called well") + } +} + +func TestNodeShowSaysTheLastSendInFull(t *testing.T) { + at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.Local) + refused := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeRefused, + Refused: "unknown field \"generation\"\nsecond line", At: &at})), "\n") + for _, want := range []string{"sequence 12", "2026-10-11 18:40:05", "a one-shot push by jochen at a shell on anchor", + "lease epoch 57", "generation 40", "docker, sudo", "refused", "unknown field \"generation\""} { + if !strings.Contains(refused, want) { + t.Errorf("node show's last send does not say %q:\n%s", want, refused) + } + } + if applied := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeApplied, + At: &at})), "\n"); !strings.Contains(applied, "applied it at 2026-10-11 18:41:00") { + t.Errorf("an applied send is not said applied:\n%s", applied) + } + if waiting := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{})), "\n"); !strings.Contains(waiting, + "not reported on it yet") { + t.Errorf("a send not reported on is not said so:\n%s", waiting) + } + stale := aLastSend(inventory.SendAnswer{}) + when := at + stale.RefusedAt, stale.RefusedApplied = &when, 44 + if s := strings.Join(lastSendLines(stale), "\n"); !strings.Contains(s, "refused it at 2026-10-11 18:41:00") || + !strings.Contains(s, "generation 44") { + t.Errorf("a send refused for its generation is not said so:\n%s", s) + } +} + +func TestStatusJSONCarriesEachMachinesLastSend(t *testing.T) { + at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.UTC) + asked := answers{nodes: []inventory.Node{{Name: "anchor"}, {Name: "laptop"}}, + lastSent: map[string]inventory.Send{"laptop": aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeRefused, + Refused: "unknown field", At: &at})}} + body, err := statusAsJSON(asked) + if err != nil { + t.Fatal(err) + } + var got struct { + LastSent []map[string]any `json:"lastSent"` + } + if err := json.Unmarshal(body, &got); err != nil { + t.Fatal(err) + } + if len(got.LastSent) != 1 { + t.Fatalf("status --json carries %d last sends, want laptop's only:\n%s", len(got.LastSent), body) + } + s := got.LastSent[0] + if s["node"] != "laptop" || s["sequence"] != float64(12) || s["epoch"] != float64(57) || + s["generation"] != float64(40) || s["sender"] != "a one-shot push by jochen at a shell on anchor" || + s["digest"] != "d12" || s["sentAt"] == nil || s["answer"] != "refused" || s["refused"] != "unknown field" || + s["answeredAt"] == nil { + t.Fatalf("laptop's last send reads %v", s) + } + if modules, _ := s["modules"].([]any); len(modules) != 2 || modules[0] != "docker" { + t.Fatalf("the modules it named read %v", s["modules"]) + } +} + +// And through the store: a send recorded and refused by the machine is what `node show` and `status --json` +// read back. +func TestNodeShowAndStatusReadTheLastSendFromTheStore(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + record, err := open.inventory.NodeByName(ctx, "laptop") + if err != nil { + t.Fatal(err) + } + if err := open.inventory.RecordSentWith(ctx, record.ID, "d12", nil, 57, 40, inventory.Send{Sequence: 12, + Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12", + Modules: []string{"docker", "sudo"}}); err != nil { + t.Fatal(err) + } + if _, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeRefused, + Declared: "d12", Refused: "unknown field \"generation\""}); err != nil { + t.Fatal(err) + } + shown := printed(t, func() error { return showNode(ctx, open, "laptop") }) + for _, want := range []string{"a one-shot push by jochen at a shell on anchor", "lease epoch 57", "generation 40", + "docker, sudo", "unknown field \"generation\""} { + if !strings.Contains(shown, want) { + t.Errorf("node show laptop does not say %q:\n%s", want, shown) + } + } + body := printed(t, func() error { return statusFor(ctx, open, true) }) + var got struct { + LastSent []struct { + Node string `json:"node"` + Sender string `json:"sender"` + Answer string `json:"answer"` + } `json:"lastSent"` + } + if err := json.Unmarshal([]byte(body), &got); err != nil { + t.Fatalf("%v:\n%s", err, body) + } + if len(got.LastSent) != 1 || got.LastSent[0].Node != "laptop" || got.LastSent[0].Answer != "refused" || + got.LastSent[0].Sender != "a one-shot push by jochen at a shell on anchor" { + t.Fatalf("status --json's last sends read %+v", got.LastSent) + } +} diff --git a/cmd/mesh-controller/left_out.go b/cmd/mesh-controller/left_out.go new file mode 100644 index 00000000..c66a21d3 --- /dev/null +++ b/cmd/mesh-controller/left_out.go @@ -0,0 +1,222 @@ +package main + +import ( + "errors" + "fmt" + "sort" + "strings" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" +) + +// A module assigned to a machine and left out of its declaration is said (novox/hq issue 380). +// +// **An omission is a finding, never a refusal of the push** (ADR 0163, rule 6): the machine is sent everything +// else, and the module's held things are kept. Until issue 380 the only place it showed was `plan`: nfs-server was +// assigned to the home server for days, every send left it out for a setting nobody gave, and the operator believed +// it ran. So the self-check (D1) judges every machine as the next send would (Resolution.LeftOutBecause, the send's +// own judgement) and raises a condition on that machine for each module it leaves out: +// +// - a setting nobody gave (catalogue.UnsetSettingError) is the operator's to give: kind needs-operator, naming +// the module, the setting and the command that sets it; +// - any other cause is said as a module not working is: kind left-out, a warning with the reason as evidence. +// +// Never urgent and never escalated by age (as ADR 0283 decision 5): nothing the machine ran was undone. It clears +// on the first run that no longer finds it — the module composed again, or no longer assigned. `status` and +// `node show` list the same modules under "assigned, not applied" with the same reason. +// +// A module left out because its stored manifest has a key this controller does not know is not raised here: the +// catalogue's own unknown-field condition says it once for the whole mesh (ADR 0262); it is still listed. + +const ( + // probeLeftOutID is the self-check's probe that raises and clears these conditions: D1, which already + // resolves every machine (probeDeclarations), so the judgement costs no resolution of its own. + probeLeftOutID = "D1" + // kindLeftOut is a module left out for any cause but a setting nobody gave; it is also every such condition's + // token, whichever its kind, so a cause that changes is the same condition said anew. + kindLeftOut = "left-out" +) + +// leftOutModule is one module of a machine's set that its declaration leaves out, and why. +type leftOutModule struct { + Module string + // Setting is the setting nobody gave, when that is the cause. + Setting string + // Why is the declaration's own reason, whole. + Why string + // Unread is a stored manifest this controller cannot read whole (said by the catalogue's condition). + Unread bool +} + +// leftOutOf is every module of a machine's resolution that a push would leave out, sorted. Pure: the judgement +// the push makes (catalogue.Resolution.LeftOutBecause), nothing allocated. +func leftOutOf(plan catalogue.Resolution, settings catalogue.SettingsBy, adopted bool) []leftOutModule { + because := plan.LeftOutBecause(settings, adopted) + out := make([]leftOutModule, 0, len(because)) + for module, why := range because { + l := leftOutModule{Module: module, Why: oneLine(why.Error())} + var unset *catalogue.UnsetSettingError + var unread *catalogue.UnreadManifestError + switch { + case errors.As(why, &unset): + l.Setting = unset.Setting + case errors.As(why, &unread): + l.Unread = true + } + out = append(out, l) + } + sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module }) + return out +} + +// settingCommand is the command that gives a module's setting on one machine. +func settingCommand(module, setting, node string) string { + return fmt.Sprintf("`settings set %s '{%q: …}' --node %s`", module, setting, node) +} + +// reason is why a module is left out, as `status`, `node show` and the condition's summary say it: for a setting +// nobody gave, the setting and the command that gives it; otherwise the declaration's own words. +func (l leftOutModule) reason(node string) string { + if l.Setting != "" { + return fmt.Sprintf("nothing sets its setting %q, so every send leaves it out of its declaration — %s sets it", l.Setting, + settingCommand(l.Module, l.Setting, node)) + } + return "every send leaves it out of its declaration: " + l.Why +} + +// leftOutObservations are the conditions a machine's left-out modules raise, one each. +func leftOutObservations(node string, left []leftOutModule) []conditions.Observation { + var out []conditions.Observation + for _, l := range left { + if l.Unread { + continue + } + out = append(out, leftOutObservation(node, l)) + } + return out +} + +// leftOutObservation is one module left out of one machine's declaration: needs-operator for a setting nobody +// gave, left-out otherwise; a warning either way, the operator's to resolve. +func leftOutObservation(node string, l leftOutModule) conditions.Observation { + o := conditions.Observation{Scope: conditions.ScopeModule, ID: l.Module + "." + node, Token: kindLeftOut, + Kind: kindLeftOut, Machine: node, Severity: conditions.Warning, Resolver: conditions.ResolverOperator, + Summary: fmt.Sprintf("%s is assigned to %s and not applied: %s", l.Module, node, l.reason(node)), + Said: l.Why} + w := leftOutWords(l.Module, node, l.Setting) + if l.Setting != "" { + o.Kind = kindNeedsOperator + } else { + // The words of why may name a path or an address, which the operator's channel withholds: the + // summary sends them to the evidence, and `plan` says them in full. + o.Summary = fmt.Sprintf("%s is assigned to %s and not applied: every send leaves it out of its declaration, "+ + "because what is set for it does not fit its manifest — the evidence and `plan %s` say why", l.Module, node, node) + } + o.Headline, o.Explanation, o.Needs, o.Resolved = w.Headline, w.Explanation, w.Needs, w.Resolved + return o +} + +// leftOutWords is what the operator reads of a module left out (ADR 0253): plain, the act named. +func leftOutWords(module, node, setting string) words { + w := words{ + Headline: fmt.Sprintf("%s is not applied on %s", module, node), + Explanation: fmt.Sprintf("%s is assigned to %s, and every send to %s leaves it out of the declaration because what "+ + "is set for it does not fit its manifest. Nothing of it changes there; the rest of %s is sent as usual.", + module, node, node, node), + Needs: fmt.Sprintf("read why in the details, then change what is set for %s or unassign it.", module), + Resolved: fmt.Sprintf("%s on %s is no longer left out", module, node), + } + if setting != "" { + w.Explanation = fmt.Sprintf("%s is assigned to %s, and every send to %s leaves it out of the declaration because "+ + "nothing sets its setting %s. Nothing of it runs there until it is set; the rest of %s is sent as usual.", + module, node, node, setting, node) + w.Needs = fmt.Sprintf("set %s for %s on %s, or approve it when it is proposed to you.", setting, module, node) + // A setting's name that is not plain (a dotted key) is in the summary instead. + if _, ok := conditions.PlainWords(w, node); !ok { + w.Explanation = fmt.Sprintf("%s is assigned to %s, and every send to %s leaves it out of the declaration because a "+ + "setting it needs is not set. Nothing of it runs there until it is set; the details name it.", + module, node, node) + w.Needs = fmt.Sprintf("set what %s needs on %s; the details name the setting.", module, node) + } + } + return w +} + +// notAppliedLines is a machine's "assigned, not applied" as `node show` prints it: one line a module, with the +// reason its condition says. +func notAppliedLines(node string, left []leftOutModule) []string { + if len(left) == 0 { + return nil + } + lines := []string{"", " assigned, not applied:"} + for _, l := range left { + lines = append(lines, fmt.Sprintf(" %-22s %s", l.Module, l.reason(node))) + } + return lines +} + +// machineNotApplied is one module assigned to a machine and left out of its declaration, in `status --json`. +type machineNotApplied struct { + Node string `json:"node"` + Module string `json:"module"` + Setting string `json:"setting,omitempty"` + Reason string `json:"reason"` + // UnreadManifest is a module left out because this controller cannot read its manifest whole: it raises no + // condition of its own on the machine, since the catalogue's unknown-field condition says it once (ADR 0262). + UnreadManifest bool `json:"unread-manifest,omitempty"` +} + +// notApplied is every machine's left-out modules, in a stated order, as `status --json` carries them. +func notApplied(left map[string][]leftOutModule) []machineNotApplied { + names := make([]string, 0, len(left)) + for name := range left { + names = append(names, name) + } + sort.Strings(names) + var out []machineNotApplied + for _, name := range names { + for _, l := range left[name] { + out = append(out, machineNotApplied{Node: name, Module: l.Module, Setting: l.Setting, Reason: l.reason(name), + UnreadManifest: l.Unread}) + } + } + return out +} + +// printNotApplied is status's "assigned, not applied", every machine's. +func printNotApplied(left map[string][]leftOutModule) { + rows := notApplied(left) + if len(rows) == 0 { + return + } + unread := 0 + for _, r := range rows { + if r.UnreadManifest { + unread++ + } + } + // Which raise a condition is said, not implied (review of #223): one whose manifest this controller cannot + // read is listed here and raises none of its own on the machine — the catalogue's condition says it. + raises := "each raises a condition on its machine" + switch { + case unread == len(rows): + raises = "none raises a condition on its machine: this controller cannot read their manifests, which the " + + "catalogue's own condition says" + case unread > 0: + raises += fmt.Sprintf(", except the %d whose manifest this controller cannot read, which the catalogue's own "+ + "condition says", unread) + } + fmt.Printf("%d module(s) assigned, not applied — every send leaves them out of their declaration; %s:\n", + len(rows), raises) + for _, r := range rows { + fmt.Printf(" %-12s %-22s %s\n", r.Node, r.Module, r.Reason) + } + fmt.Println() +} + +// isLeftOutCondition is whether a condition is a module left out of its declaration, which the machine's health +// statements neither raise nor clear: by its token, which every one carries whatever its kind. +func isLeftOutCondition(c conditions.Condition) bool { + return c.Subject.Scope == conditions.ScopeModule && strings.HasSuffix(c.Key, "."+kindLeftOut) +} diff --git a/cmd/mesh-controller/left_out_test.go b/cmd/mesh-controller/left_out_test.go new file mode 100644 index 00000000..7c75d307 --- /dev/null +++ b/cmd/mesh-controller/left_out_test.go @@ -0,0 +1,259 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// novox/hq issue 380: nfs-server was assigned to the home server for days and every send left it out — "nfs-server +// has a file that says ${setting:shares}, and nothing sets shares for it" — and nothing but `plan` said so. The +// manifest is the catalogue's own at the commit that added it (mesh-catalog 48fba44), which still says +// ${setting:shares}; the reason below is the one the live push printed. + +// leftOutNFS is the resolution of a machine assigned that nfs-server, with the settings given. +func leftOutNFS(t *testing.T) catalogue.Resolution { + t.Helper() + raw, err := os.ReadFile("testdata/left-out/nfs-server.json") + if err != nil { + t.Fatal(err) + } + m, err := catalogue.ParseManifest(raw) + if err != nil { + t.Fatal(err) + } + return catalogue.Resolution{Modules: []catalogue.Manifest{m}} +} + +// sharesGiven is the operator's setting for it on the machine. +func sharesGiven(value string) catalogue.SettingsBy { + return catalogue.SettingsBy{"nfs-server": {{From: "anchor", Values: map[string]any{"shares": value}}}} +} + +func TestAModuleLeftOutForASettingNobodyGaveNeedsTheOperatorNamingTheSettingAndTheCommand(t *testing.T) { + left := leftOutOf(leftOutNFS(t), nil, false) + if len(left) != 1 || left[0].Module != "nfs-server" || left[0].Setting != "shares" { + t.Fatalf("left out: %+v", left) + } + if !strings.Contains(left[0].Why, `nothing sets "shares" for it`) { + t.Fatalf("the reason is not the push's own: %s", left[0].Why) + } + obs := leftOutObservations("anchor", left) + if len(obs) != 1 { + t.Fatalf("raised %+v", obs) + } + o := obs[0] + if o.Key() != "module.nfs-server.anchor.left-out" || o.Kind != kindNeedsOperator || o.Machine != "anchor" || + o.Severity != conditions.Warning || o.Resolver != conditions.ResolverOperator { + t.Fatalf("raised %s as %s, %s, by %s, on %q", o.Key(), o.Kind, o.Severity, o.Resolver, o.Machine) + } + for _, want := range []string{"nfs-server", "anchor", `"shares"`, "`settings set nfs-server '{\"shares\": …}' --node anchor`"} { + if !strings.Contains(o.Summary, want) { + t.Errorf("the summary does not name %s: %s", want, o.Summary) + } + } + if o.Said != left[0].Why { + t.Errorf("the evidence is not the reason the send gives: %s", o.Said) + } + plainExample(t, o, "nfs-server is not applied on anchor", + "Needs you: set shares for nfs-server on anchor, or approve it when it is proposed to you. nfs-server is assigned to "+ + "anchor, and every send to anchor leaves it out of the declaration because nothing sets its setting shares. "+ + "Nothing of it runs there until it is set; the rest of anchor is sent as usual.") +} + +func TestAModuleLeftOutForAnotherCauseIsAWarningWithTheReasonAsEvidence(t *testing.T) { + // A setting stored that its manifest can no longer take: one with a line break (issue 339). + left := leftOutOf(leftOutNFS(t), sharesGiven("library=/srv/library\nmedia=/srv/media"), false) + if len(left) != 1 || left[0].Setting != "" { + t.Fatalf("left out: %+v", left) + } + obs := leftOutObservations("anchor", left) + if len(obs) != 1 { + t.Fatalf("raised %+v", obs) + } + o := obs[0] + if o.Key() != "module.nfs-server.anchor.left-out" || o.Kind != kindLeftOut || o.Severity != conditions.Warning { + t.Fatalf("raised %s as %s, %s", o.Key(), o.Kind, o.Severity) + } + if !strings.Contains(o.Said, "holds a line break") || strings.Contains(o.Summary, "/srv/") { + t.Fatalf("the reason is not the evidence, or the summary carries it to the channel:\n%s\n%s", o.Summary, o.Said) + } + plainExample(t, o, "nfs-server is not applied on anchor", + "Needs you: read why in the details, then change what is set for nfs-server or unassign it. nfs-server is assigned to "+ + "anchor, and every send to anchor leaves it out of the declaration because what is set for it does not fit "+ + "its manifest. Nothing of it changes there; the rest of anchor is sent as usual.") +} + +// The self-check raises it, keeps it a warning however long it stands, and clears it when the module composes +// again or is no longer assigned; a machine's health statement neither clears nor raises it. +func TestALeftOutModulesConditionClearsWhenItComposesAgainOrIsUnassigned(t *testing.T) { + plan, settings := leftOutNFS(t), catalogue.SettingsBy(nil) + withProbes(t, probe{ID: probeLeftOutID, Asserts: "the test's", Kind: kindLeftOut, Phase: 1, + run: func(context.Context, *doctor) ([]conditions.Observation, error) { + return leftOutObservations("anchor", leftOutOf(plan, settings, false)), nil + }}) + store := conditions.NewInMemory() + k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store}) + defer k.Close(context.Background()) + d := &doctor{keeper: k, teller: &conditions.Told{}, host: "anchor"} + key := "module.nfs-server.anchor.left-out" + openOnes := func() map[string]conditions.Condition { + t.Helper() + open, err := k.Open(t.Context()) + if err != nil { + t.Fatal(err) + } + out := map[string]conditions.Condition{} + for _, c := range open { + out[c.Key] = c + } + return out + } + + d.runOnce(t.Context(), "a test") + c, raised := openOnes()[key] + if !raised || c.Kind != kindNeedsOperator || c.Severity != conditions.Warning { + t.Fatalf("a module left out raised %+v", openOnes()) + } + // A statement from the machine that says nothing of it — the module runs nothing there — leaves it open. + if err := judgeModuleHealth(t.Context(), nil, k, "anchor", map[string][]inventory.ResourceHealth{}, nil, + time.Now().Add(72*time.Hour)); err != nil { + t.Fatal(err) + } + if _, still := openOnes()[key]; !still { + t.Fatal("a health statement that says nothing of the module cleared its left-out condition") + } + d.runOnce(t.Context(), "a test") + if c := openOnes()[key]; c.Severity != conditions.Warning { + t.Fatalf("after a health statement and days, it is %+v", openOnes()) + } + + // The setting given: it composes, and the condition clears. + settings = sharesGiven("library=/srv/library") + d.runOnce(t.Context(), "a test") + if _, still := openOnes()[key]; still { + t.Fatalf("composed again, still open: %+v", openOnes()) + } + + // Left out again, then unassigned: no longer in the machine's set, and it clears. + settings = nil + d.runOnce(t.Context(), "a test") + if _, raised := openOnes()[key]; !raised { + t.Fatal("left out again and not raised") + } + plan = catalogue.Resolution{} + d.runOnce(t.Context(), "a test") + if _, still := openOnes()[key]; still { + t.Fatalf("unassigned, still open: %+v", openOnes()) + } +} + +func TestTheLeftOutProbeIsInTheRegistry(t *testing.T) { + for _, p := range probeRegistry { + if p.ID == probeLeftOutID { + if p.run == nil || !slices.Contains(p.Raises, kindLeftOut) || !slices.Contains(p.Raises, kindNeedsOperator) { + t.Fatalf("%+v", p) + } + return + } + } + t.Fatalf("no probe %s: a module left out is said nowhere", probeLeftOutID) +} + +// status and node show list it under "assigned, not applied" with the reason the condition says, and status is +// not well while there is one. +func TestStatusAndNodeListAModuleAssignedAndNotApplied(t *testing.T) { + left := map[string][]leftOutModule{"anchor": leftOutOf(leftOutNFS(t), nil, false)} + reason := leftOutObservations("anchor", left["anchor"])[0].Summary + asked := answers{leftOut: left} + if asked.well() { + t.Fatal("a mesh with a module assigned and not applied is called well") + } + shown := printed(t, func() error { return printStatus(asked) }) + if !strings.Contains(shown, "1 module(s) assigned, not applied") || !strings.Contains(shown, "nfs-server") || + !strings.Contains(shown, "`settings set nfs-server '{\"shares\": …}' --node anchor` sets it") { + t.Fatalf("status says:\n%s", shown) + } + if !strings.Contains(reason, left["anchor"][0].reason("anchor")) { + t.Fatalf("status and the condition say different reasons:\n%s\n%s", shown, reason) + } + body, err := statusAsJSON(asked) + if err != nil { + t.Fatal(err) + } + var doc struct { + NotApplied []machineNotApplied `json:"not-applied"` + } + if err := json.Unmarshal(body, &doc); err != nil || len(doc.NotApplied) != 1 || + doc.NotApplied[0].Setting != "shares" || doc.NotApplied[0].Node != "anchor" { + t.Fatalf("status --json: %v %s", err, body) + } + lines := strings.Join(notAppliedLines("anchor", left["anchor"]), "\n") + if !strings.Contains(lines, "assigned, not applied:") || !strings.Contains(lines, "nfs-server") || + !strings.Contains(lines, `nothing sets its setting "shares"`) { + t.Fatalf("node show says:\n%s", lines) + } +} + +// The skip is this probe's alone (review of #223): a part waiting for the operator (ADR 0283) keeps its own +// needs-operator condition, `module...needs-operator`, which still clears on the first statement that no +// longer names it — beside a left-out condition on the same machine, which stays. +func TestAWaitsNeedsOperatorStillClearsWhenItsStatementStopsNamingItBesideALeftOutOne(t *testing.T) { + store := conditions.NewInMemory() + k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store}) + defer k.Close(context.Background()) + left := leftOutObservations("anchor", leftOutOf(leftOutNFS(t), nil, false)) + if err := k.Reconcile(t.Context(), probeLeftOutID, left); err != nil { + t.Fatal(err) + } + waiting := map[string][]inventory.ResourceHealth{"notes": {{Module: "notes", Resource: "server", State: link.StateWaiting, + Waits: []inventory.Wait{{Setting: "domain", What: "the domain it serves"}}}}} + if err := judgeModuleHealth(t.Context(), nil, k, "anchor", waiting, map[string]int{"notes": 2}, time.Now()); err != nil { + t.Fatal(err) + } + waitKey, leftKey := needsOperatorKey("notes", "anchor"), "module.nfs-server.anchor.left-out" + open := func() map[string]conditions.Condition { + t.Helper() + list, err := k.Open(t.Context()) + if err != nil { + t.Fatal(err) + } + out := map[string]conditions.Condition{} + for _, c := range list { + out[c.Key] = c + } + return out + } + if c, raised := open()[waitKey]; !raised || c.Kind != kindNeedsOperator { + t.Fatalf("the wait raised %+v", open()) + } + if c := open()[leftKey]; c.Kind != kindNeedsOperator { + t.Fatalf("the left-out condition is not open as needs-operator: %+v", open()) + } + // D1 runs again and still finds nfs-server left out: its reconcile clears only what D1 raised, never the wait, + // which the machine's statement raised. + if err := k.Reconcile(t.Context(), probeLeftOutID, left); err != nil { + t.Fatal(err) + } + if _, still := open()[waitKey]; !still { + t.Fatalf("D1's reconcile cleared the wait's needs-operator: %+v", open()) + } + if err := judgeModuleHealth(t.Context(), nil, k, "anchor", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil { + t.Fatal(err) + } + if _, still := open()[waitKey]; still { + t.Fatalf("the statement no longer names the wait, and its needs-operator is still open: %+v", open()) + } + if _, still := open()[leftKey]; !still { + t.Fatalf("the left-out condition was cleared by a health statement: %+v", open()) + } +} diff --git a/cmd/mesh-controller/module_health.go b/cmd/mesh-controller/module_health.go index fc7cabbe..58f0d0ac 100644 --- a/cmd/mesh-controller/module_health.go +++ b/cmd/mesh-controller/module_health.go @@ -147,9 +147,11 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi } standing := map[string]conditions.Condition{} for _, c := range open { + // A module left out of the composition is the self-check's to raise and clear, never a statement's + // (novox/hq issue 380): its needs-operator is not cleared for not being in what the machine runs. if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound || c.Kind == kindNeedsOperator) && - c.Subject.Machine == node { + c.Subject.Machine == node && !isLeftOutCondition(c) { standing[c.Key] = c } } diff --git a/cmd/mesh-controller/nodes.go b/cmd/mesh-controller/nodes.go index a89adcf2..def9e9a3 100644 --- a/cmd/mesh-controller/nodes.go +++ b/cmd/mesh-controller/nodes.go @@ -44,7 +44,7 @@ func nodeCommand(ctx context.Context, args []string) error { if len(args) != 2 { return errors.New("node show ") } - return showNode(ctx, inv, args[1]) + return showNode(ctx, open, args[1]) case "add": return addNode(ctx, inv, args[1:]) @@ -787,7 +787,8 @@ func roughly(d time.Duration) string { // // It is also where "what should it be configured as" is read. The same line that gates an // assignment carries `card1-DP-1`, and a person composing settings for that machine needs it. -func showNode(ctx context.Context, inv *inventory.Inventory, name string) error { +func showNode(ctx context.Context, stored *stores, name string) error { + inv := stored.inventory node, err := inv.NodeByName(ctx, name) if err != nil { return err @@ -803,6 +804,19 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error if err := showMode(ctx, inv, node); err != nil { return err } + // What it was last sent, by whom, under which epoch, from which generation, naming which modules, and what + // it answered (novox/hq issue 485): the record of sends is read through this verb, never the store's reader. + // Unreadable is said, not taken for none. + switch last, found, err := inv.LastSend(ctx, name); { + case err != nil: + fmt.Printf(" what it was last sent could NOT be read: %v\n", err) + case !found: + fmt.Printf(" last sent no send recorded since the mesh began keeping them\n") + default: + for _, line := range lastSendLines(last) { + fmt.Println(line) + } + } // Whom agents run as here, and whether that account can become root without a person (ADR 0266). for _, line := range agentAccountLines(ctx, inv, node) { fmt.Println(line) @@ -889,6 +903,17 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error if len(assigned) > 0 { fmt.Printf("\n assigned: %s\n", strings.Join(assigned, ", ")) } + // And which of them a push leaves out, and why (novox/hq issue 380): judged as the push judges it, the same + // reason its condition says. Not computable is said, never read as "all applied". + plan, settings, err := planFor(ctx, stored, name) + switch { + case err != nil: + fmt.Printf("\n whether a send leaves any assigned module out is NOT known: %s\n", oneLine(err.Error())) + default: + for _, line := range notAppliedLines(name, leftOutOf(plan, settings, node.Adopted)) { + fmt.Println(line) + } + } return nil } diff --git a/cmd/mesh-controller/over_budget.go b/cmd/mesh-controller/over_budget.go index 8fbc895c..a3b8101e 100644 --- a/cmd/mesh-controller/over_budget.go +++ b/cmd/mesh-controller/over_budget.go @@ -17,8 +17,8 @@ import ( // A delivery over its budget is loud (novox/hq ADR 0282 decision 7, issue 382): the self-check reads // mesh-delivery's `times` and raises the warning `delivery..over-budget` when the newest delivery of a -// class whose delivery time is known took longer than its class's budget — five minutes for a leaf module, ten -// for a core module — naming the delivery and its longest phase. It clears when the next delivery of that class +// class whose delivery time is known took longer than its class's budget — three minutes for a leaf module (ADR +// 0304 decision 4), ten for a core module — naming the delivery and its longest phase. It clears when the next delivery of that class // lands within its budget. Measurement only: the condition says, it never holds or acts on a delivery. // probeBudgetsID is the probe that reads the delivery times. @@ -27,8 +27,9 @@ const probeBudgetsID = "D16" // kindOverBudget is what a delivery over its class's budget raises. const kindOverBudget = "over-budget" -// deliveryBudgets are the budgets by class (ADR 0282 decision 1); the probe raises nothing for another class. -var deliveryBudgets = map[string]time.Duration{inventory.ClassLeaf: 5 * time.Minute, inventory.ClassCore: 10 * time.Minute} +// deliveryBudgets are the budgets by class (ADR 0282 decision 1, the leaf's as ADR 0304 decision 4 changed it); +// the probe raises nothing for another class. +var deliveryBudgets = map[string]time.Duration{inventory.ClassLeaf: 3 * time.Minute, inventory.ClassCore: 10 * time.Minute} // timesAnswer is what mesh-delivery's `times` answers, as far as the probe reads it. type timesAnswer struct { diff --git a/cmd/mesh-controller/over_budget_test.go b/cmd/mesh-controller/over_budget_test.go index 483e068a..3131e9b8 100644 --- a/cmd/mesh-controller/over_budget_test.go +++ b/cmd/mesh-controller/over_budget_test.go @@ -10,11 +10,12 @@ import ( "github.com/novox/mesh-controller/internal/inventory" ) -// A leaf delivery of 5 minutes 10 seconds raises delivery.leaf.over-budget naming its longest phase; one of -// exactly five minutes, a core one of nine and a class without a budget raise nothing (ADR 0282 decision 7). +// A leaf delivery of 3 minutes 10 seconds raises delivery.leaf.over-budget naming its longest phase; one of +// exactly three minutes, a core one of nine and a class without a budget raise nothing (ADR 0282 decision 7, +// the leaf's budget as ADR 0304 decision 4 changed it). func TestADeliveryOverItsBudgetIsLoud(t *testing.T) { a := ×Answer{Classes: []timesClass{ - {Class: inventory.ClassLeaf, Latest: ×Latest{ID: "novox/mesh-catalog@abc", TookMS: (5*time.Minute + 10*time.Second).Milliseconds(), + {Class: inventory.ClassLeaf, Latest: ×Latest{ID: "novox/mesh-catalog@abc", TookMS: (3*time.Minute + 10*time.Second).Milliseconds(), Longest: "judgement 2m40s"}}, {Class: inventory.ClassCore, Latest: ×Latest{ID: "novox/mesh-controller@def", TookMS: (9 * time.Minute).Milliseconds(), Longest: "build 1m"}}, @@ -30,7 +31,7 @@ func TestADeliveryOverItsBudgetIsLoud(t *testing.T) { t.Fatalf("the condition names its delivery and longest phase: %s — %s", o.Key(), o.Summary) } // The next leaf delivery within its budget clears it: the probe raises nothing for the class. - a.Classes[0].Latest = ×Latest{ID: "novox/mesh-catalog@ghi", TookMS: (5 * time.Minute).Milliseconds()} + a.Classes[0].Latest = ×Latest{ID: "novox/mesh-catalog@ghi", TookMS: (3 * time.Minute).Milliseconds()} if obs := overBudgetObservations(a); len(obs) != 0 { t.Fatalf("a delivery of exactly its budget is within it: %+v", obs) } diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go index 3a493a15..4ce02db1 100644 --- a/cmd/mesh-controller/plain_words.go +++ b/cmd/mesh-controller/plain_words.go @@ -231,6 +231,14 @@ var plainWordings = map[string]func(conditions.Observation) words{ w := needsOperatorWords(orModule(module), node, nil) return w }), + kindLeftOut: worded(func(o conditions.Observation) words { + // The observation carries its own words (novox/hq issue 380); these are its kind's alone. + module := "" + if o.Scope == conditions.ScopeModule && o.Machine != "" { + module = strings.TrimSuffix(o.ID, "."+o.Machine) + } + return leftOutWords(orModule(module), machineOr(o, "a machine"), "") + }), kindProviderFailing: worded(func(o conditions.Observation) words { thing, consumer := conditions.ThingWords(o), idPart(o, 2) if consumer == "" { diff --git a/cmd/mesh-controller/plan_retry.go b/cmd/mesh-controller/plan_retry.go index df28926e..a41acb69 100644 --- a/cmd/mesh-controller/plan_retry.go +++ b/cmd/mesh-controller/plan_retry.go @@ -278,6 +278,13 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) { plans = append(plans, recent...) // **A failed walk whose earlier merges are walked alone is not retried** (novox/hq ADR 0276): the search for // the merge that brought the failure answers them now, and a retried walk would name them twice. + // **A walk stopped without some merges is not retried** (novox/hq ADR 0299): the merges it left out were stopped, + // and the others are another walk's now. + if p.Delivery != nil && (p.Delivery.WalkedAgainBy != "" || len(p.Delivery.LeftOut) > 0) { + return "", fmt.Errorf("%s was stopped without %s: those merges were stopped, and %s walks the others; a "+ + "newer merge, or `rebuild `, builds again", p.ID, mergesLeftOutWords(p.Delivery.LeftOut), + orWords(p.Delivery.WalkedAgainBy, "nothing")) + } if p.Delivery != nil && len(p.Delivery.Merges) > 0 { kept, err := inv.MergesOf(ctx, p.ID) if err != nil { diff --git a/cmd/mesh-controller/probes.go b/cmd/mesh-controller/probes.go index 22ef45a3..481571f7 100644 --- a/cmd/mesh-controller/probes.go +++ b/cmd/mesh-controller/probes.go @@ -78,6 +78,11 @@ func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err) } var problems, foreseen []string + // Each module the next send leaves out of this machine's declaration (novox/hq issue 380), judged from this + // resolution as the send judges it: a finding, never a refusal. + if err == nil { + out = append(out, leftOutObservations(n.Name, leftOutOf(plan, settings, n.Adopted))...) + } if err == nil && gensErr == nil { var declared sendable if declared, err = declarationWith(ctx, open, n.Name, plan, settings, gens, Foreseeing); err == nil { diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 89e1caa1..812f6d79 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -1236,6 +1236,17 @@ func sendToEach(ctx context.Context, open *stores, names []string) ([]string, er continue } numbered.stamp(&declared) + // **The grants step sends the user list and nothing else** (novox/hq issue 490): judged on this very + // declaration, the one sent, so nothing composed between a check and the send can slip through. + if everyBuildKept(ctx, name) { + other, err := grantsOnlyIn(ctx, open, name, plan, declared) + if err != nil { + return nil, err + } + if other != "" { + return nil, fmt.Errorf("%w: %s", errNotGrantsOnly, other) + } + } reportLeftOut(name, declared) sending = append(sending, readyNode{name, declared}) } diff --git a/cmd/mesh-controller/readable.go b/cmd/mesh-controller/readable.go index d286b693..32155fbc 100644 --- a/cmd/mesh-controller/readable.go +++ b/cmd/mesh-controller/readable.go @@ -69,6 +69,9 @@ type meshStatus struct { // **A document without this said an outage was a well mesh.** Read from what each machine // reported, so it is the machine's account and not the mesh's take-time listing. Untaken []machineUntaken `json:"untaken,omitempty"` + // NotApplied is every module assigned to a machine and left out of its composition, with why (novox/hq + // issue 380). Absent when every module composes. + NotApplied []machineNotApplied `json:"not-applied,omitempty"` // Filtered is every converged machine that is not filtered by the mesh alone (novox/hq ADR // 0168), one entry per rule set the mesh did not write — the found firewall in force again, // or a chain nobody speaks for. Absent when every converged machine is filtered by the mesh @@ -106,6 +109,33 @@ type meshStatus struct { // why they could not be read. Pending []pendingStatus `json:"pending,omitempty"` PendingUnread string `json:"pendingUnread,omitempty"` + // LastSent is every machine's last recorded send — who sent it, under which lease epoch, from which + // assignment generation, naming which modules — and what the machine answered of it (novox/hq issue 485). + // A machine the mesh has recorded no send to is absent. LastSentUnread says why it could not be read. + LastSent []machineLastSent `json:"lastSent"` + LastSentUnread string `json:"lastSentUnread,omitempty"` +} + +// machineLastSent is one machine's last send as status says it. +type machineLastSent struct { + Node string `json:"node"` + Sequence int64 `json:"sequence"` + SentAt time.Time `json:"sentAt"` + Sender string `json:"sender"` + Epoch int64 `json:"epoch"` + Generation int64 `json:"generation"` + Digest string `json:"digest"` + Modules []string `json:"modules"` + // Answer is what the machine said of it: applied, failed or refused, as its report has it; empty when it + // has not reported on it. Refused is its words when it refused it whole, Failed how many resources failed. + Answer string `json:"answer"` + Refused string `json:"refused,omitempty"` + Failed int `json:"failed,omitempty"` + AnsweredAt *time.Time `json:"answeredAt,omitempty"` + // RefusedForGeneration is when the machine refused it for coming from an older assignment generation than + // it applied, and AppliedGeneration that generation; absent when it did not. + RefusedForGeneration *time.Time `json:"refusedForGeneration,omitempty"` + AppliedGeneration int64 `json:"appliedGeneration,omitempty"` } // pendingStatus is one pending assignment as status says it. @@ -251,6 +281,7 @@ func statusAsJSON(asked answers) ([]byte, error) { } } out.Unheld = asked.unheld + out.NotApplied = notApplied(asked.leftOut) out.HandActsThisWeek, out.HandActsUnread = asked.handActs, asked.handActsUnread out.HealsThisWeek, out.HealsUnread = asked.heals, asked.healsUnread // In brief, as `conditions` lists them: status leads with every open condition, and their whole @@ -263,6 +294,7 @@ func statusAsJSON(asked answers) ([]byte, error) { out.Pending = append(out.Pending, pendingStatus{Node: p.Node, Module: p.Module, State: p.State, Build: p.Build, Repository: p.Repository, Path: p.Path, Since: p.Since, Settled: p.Settled, Note: p.Note}) } + out.LastSent, out.LastSentUnread = lastSentStatus(asked.lastSent), asked.lastSentUnread for name := range asked.refused { out.Unresolved = append(out.Unresolved, machineUnresolved{ Node: name, Problem: asked.refused[name]}) @@ -319,3 +351,20 @@ func say(value any) error { fmt.Println(string(body)) return nil } + +// lastSentStatus is every machine's last send as status --json says it, by machine name (novox/hq issue 485). +func lastSentStatus(sends map[string]inventory.Send) []machineLastSent { + out := []machineLastSent{} + for _, s := range sends { + modules := s.Modules + if modules == nil { + modules = []string{} + } + out = append(out, machineLastSent{Node: s.NodeName, Sequence: s.Sequence, SentAt: s.SentAt, Sender: s.Sender, + Epoch: s.Epoch, Generation: s.Generation, Digest: s.Digest, Modules: modules, Answer: s.Answer.Outcome, + Refused: s.Answer.Refused, Failed: s.Answer.Failed, AnsweredAt: s.Answer.At, + RefusedForGeneration: s.RefusedAt, AppliedGeneration: s.RefusedApplied}) + } + sort.Slice(out, func(i, j int) bool { return out[i].Node < out[j].Node }) + return out +} diff --git a/cmd/mesh-controller/recorded_kept.go b/cmd/mesh-controller/recorded_kept.go index bbe6f18f..a8e8cf5b 100644 --- a/cmd/mesh-controller/recorded_kept.go +++ b/cmd/mesh-controller/recorded_kept.go @@ -61,20 +61,57 @@ func keptExcept(ctx context.Context) (map[string]bool, bool) { return skip, on } +type keepEveryBuildKey struct{} + +// keepingEveryBuild is a context whose sends compose every module of one machine — recorded or rolling out — +// at the build that machine was last sent (novox/hq issue 490): the grants step, which sends the machine +// holding the bus its new user list and nothing of any module's new code. That code waits there for a gate +// like any other move; the list must not, or the first machine's gate is judged against a bus that refuses +// what the change newly grants. +// +// **That machine alone.** Composing one machine resolves the others too — who is on the private network, +// who answers a requirement — on the same context, and those are no part of the step's send: they are +// composed as any send composes them. Kept for every machine, a machine whose last send is not known +// refused the bus's machine's composition. +func keepingEveryBuild(ctx context.Context, node string) context.Context { + return context.WithValue(ctx, keepEveryBuildKey{}, node) +} + +// everyBuildKept is whether this context keeps every module of this machine at the build it runs. +func everyBuildKept(ctx context.Context, node string) bool { + on, _ := ctx.Value(keepEveryBuildKey{}).(string) + return on != "" && on == node +} + // recordedKept is, for a send under keepingRecorded, every recorded module the machine was last sent a // build of that the mesh's build is not identical to: module → the commit it keeps. Nil when the context // keeps nothing, or when what the machine was last sent is not known (it is then held whole elsewhere — // ADR 0221). +// +// Under keepingEveryBuild, every module the machine was sent is kept, whatever its policy (novox/hq issue +// 490), and a machine whose last send is not known refuses the send: there is nothing to keep it at, and +// composing the mesh's builds would be the very move the grants step must not make. func recordedKept(ctx context.Context, open *stores, node string) (map[string]string, error) { + every := everyBuildKept(ctx, node) skip, on := keptExcept(ctx) - if !on { + if !on && !every { return nil, nil } + if every { + skip = nil + } inv := open.inventory sent, known, err := inv.SentBuilds(ctx, node) - if err != nil || !known { + if err != nil { return nil, err } + if !known { + if every { + return nil, fmt.Errorf("what %s was last sent is not known, so the bus's user list cannot be sent "+ + "there alone with every build kept (novox/hq issue 490)", node) + } + return nil, nil + } current, err := inv.CurrentBuilds(ctx) if err != nil { return nil, err @@ -83,7 +120,7 @@ func recordedKept(ctx context.Context, open *stores, node string) (map[string]st out := map[string]string{} for m, was := range sent { now, held := current[m] - if !held || now.RollOut || skip[m] || was == "" || sameCommit(was, now.Commit) { + if !held || (now.RollOut && !every) || skip[m] || was == "" || sameCommit(was, now.Commit) { continue } if f == nil { @@ -118,6 +155,10 @@ func keepRecorded(ctx context.Context, open *stores, node string, shelf map[stri if err != nil { return nil, err } + if !found && everyBuildKept(ctx, node) { + return nil, fmt.Errorf("%s runs %s's build %s, which the build records no longer hold: the bus's user "+ + "list cannot be sent there alone with it kept (novox/hq issue 490)", node, m, short(kept[m])) + } if !found { return nil, fmt.Errorf("%s records rather than rolls out, and %s runs its build %s, which the build "+ "records no longer hold: this send cannot keep it and does not move it — `push %s` sends the new "+ diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go index 56b702c8..555ebeb2 100644 --- a/cmd/mesh-controller/release.go +++ b/cmd/mesh-controller/release.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "errors" "flag" "fmt" @@ -253,8 +254,19 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st if err != nil { return nil, err } + // The grants step composes every module at the build its machine was last sent (novox/hq issue 490): + // a move it keeps is not made. Read, not assumed, so a move it could not keep is still refused here. + var keeps map[string]string + if everyBuildKept(ctx, n) { + if keeps, err = recordedKept(ctx, open, n); err != nil { + return nil, err + } + } var waiting []string for _, mv := range moves { + if was, kept := keeps[mv.Module]; kept && sameCommit(was, mv.From) { + continue + } if !scope.judged[n] && !scope.modules[mv.Module] { waiting = append(waiting, fmt.Sprintf("%s %s → %s", mv.Module, short(mv.From), short(mv.To))) } @@ -283,15 +295,18 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st // owns are the moves the send exists for — every module of a plan's tier whose first machine this is, in // one send (novox/hq issue 281): a send carries the machine's whole declaration (ADR 0221), so a send per // module was the same declaration sent again and again, each one setting aside the one before. -func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, []string, error) { +// +// And the grants step first, when the send changes what the bus's user list must say (novox/hq issue 490): +// answered for the gate to keep, nil when there was none. +func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, []string, *inventory.GrantsStep, error) { inv := open.inventory f, err := readMoveFacts(ctx, inv) if err != nil { - return nil, nil, err + return nil, nil, nil, err } moves, err := machineMoves(ctx, open, f, node, true) if err != nil { - return nil, nil, err + return nil, nil, nil, err } own := func(module string) bool { return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module }) @@ -301,7 +316,7 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory. continue } if id := f.walkedBy(mv.Module, node, mv.To); id != "" { - return nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere, + return nil, nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere, mv.Module, short(mv.To), node, id) } } @@ -309,7 +324,7 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory. // plan, or a release, on this machine is not crossed with a newer build; this send waits for its gate. for _, o := range owns { if id := f.walkedBy(o.Module, node, o.To); id != "" { - return nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere, + return nil, nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere, o.Module, short(o.To), node, id) } } @@ -323,22 +338,22 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory. } } if len(moves) == 0 && len(owns) == 0 { - return nil, nil, nil + return nil, nil, nil, nil } for i := range moves { if moves[i].Build == "" { if moves[i].Build, err = inv.BuildOf(ctx, moves[i].Module, moves[i].To); err != nil { - return nil, nil, err + return nil, nil, nil, err } } } sort.Slice(moves, func(i, j int) bool { return moves[i].Module < moves[j].Module }) sayRecreations(ctx, open, moves) - sent, err := sendRollout(withScope(ctx, sendScope{judged: map[string]bool{node: true}}), open, []string{node}) + sent, grants, err := sendJudged(ctx, open, node) if err != nil { - return nil, nil, err + return nil, nil, nil, err } - return moves, sent, nil + return moves, sent, grants, nil } // passCarried keeps a pass as the verdict of every build the gate judged beside its own module. @@ -674,7 +689,7 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, r.Next++ continue } - moves, sent, err := gatedSend(ctx, open, node, nil) + moves, sent, grants, err := gatedSend(ctx, open, node, nil) if errors.Is(err, errWalkedElsewhere) { note := fmt.Sprintf("waiting before %s: %v", node, err) changed := p.Note != note @@ -689,8 +704,12 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, r.Next++ continue } - r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves, Sent: sentNow(ctx, open.inventory, sent)} + r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves, Sent: sentNow(ctx, open.inventory, sent), + Grants: grants} p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves)) + if said := grantsSaid(grants); said != "" { + p.Note += "; " + said + } if said := recreationsSaid(moves); said != "" { p.Note += "; " + said } @@ -916,3 +935,162 @@ func recreationsSaid(moves []inventory.CarriedMove) string { } return strings.Join(said, "; ") } + +// The grants step (novox/hq issue 490). +// +// The bus's user list — every module's grants, composed mesh-wide from the catalogue — travels only in the +// declaration of the machine holding the bus. A send to any other machine that changed it put that machine +// first (issue 249), unless a build waited there for a gate: then ungatedIn left it out, and said so. On +// 2026-10-11 a merge gave restic a new health tool; its walk sent restic to its first machine and judged it +// there, the machine's node-engine asked the tool and the bus refused it ("this host's grant does not name +// …"), because the machine holding the bus also runs restic, whose new build waited for that very gate. The +// gate could not pass; a person pushed the bus's machine by hand, which carried restic's new build there +// unjudged. +// +// So before a gated send, when the user list composed now is not the one the machine holding the bus was +// last sent, that machine is sent its declaration with **every build kept at the one it runs**: the new +// list, and nothing of any module's new code. Said on the gate (`plans`), and, when it cannot be sent, said +// with why and passed over: the send goes on as it did before, and its gate says what it finds. + +// brokerBehindOf is brokerBehind: a variable so a test of the walk needs no store. +var brokerBehindOf = brokerBehind + +// grantsStep sends the machine holding the bus its user list alone, ahead of a gated send to node, when the +// list changed; answers what it did, or nil when there was nothing to send. +func grantsStep(ctx context.Context, open *stores, node string) *inventory.GrantsStep { + holder, behind, err := brokerBehindOf(ctx, open, []string{node}) + if err != nil { + fmt.Printf("grants step before %s: whether the bus's user list changed could not be read: %v\n", node, err) + return &inventory.GrantsStep{Node: "the machine holding the bus", Failed: err.Error()} + } + if holder == "" || holder == node || !behind { + // No bus with a user list, the gate's own machine holds it (its send carries the list first), or the + // list is the one already sent. + return nil + } + if _, err := sendRollout(keepingEveryBuild(withScope(ctx, sendScope{}), holder), open, []string{holder}); err != nil { + fmt.Printf("grants step: the bus's user list could not be sent to %s ahead of %s, which is sent without "+ + "it — the bus may refuse what the send newly grants: %v\n", holder, node, err) + return &inventory.GrantsStep{Node: holder, Failed: err.Error()} + } + now := time.Now().UTC() + fmt.Printf("grants step: %s sent the bus's user list alone, every build there kept, before %s is judged\n", + holder, node) + return &inventory.GrantsStep{Node: holder, At: &now} +} + +// errNotGrantsOnly is a grants step refused unsent: its declaration would change more than the user list. +var errNotGrantsOnly = errors.New("the grants step would change more than the bus's user list, and is not sent") + +// grantsOnlyIn is what a grants step's declaration, composed for sending, would change on the machine holding +// the bus besides its user list, against what it was last sent (novox/hq issue 490); empty when nothing else. +// Judged by sendToEach on the very declaration it then sends — one composition, judged, then sent or refused +// unsent. Kept builds are not the whole of a declaration: a new assignment, a settings change, an assignment +// taken away, a provision's logins are composed as the mesh holds them now, and a step that carried any of +// it would be the unjudged change this step exists to avoid. Compared resource by resource with the summary +// the last send kept (ADR 0217): exact, rather than a list of the cases that can differ. A module there that +// was never sent is a new assignment, refused whatever its resources. +func grantsOnlyIn(ctx context.Context, open *stores, holder string, plan catalogue.Resolution, declared sendable) (string, error) { + inv := open.inventory + sent, known, err := inv.SentBuilds(ctx, holder) + if err != nil { + return "", err + } + if !known { + return fmt.Sprintf("what %s was last sent is not known", holder), nil + } + var other []string + listID := "" + for _, m := range plan.Modules { + if _, was := sent[m.Module]; !was { + other = append(other, m.Module+" newly assigned") + } + if m.BusUsers != "" && m.ClaimsSeat(catalogue.BrokerSeat) { + listID = m.Module + "." + catalogue.BusUsersID() + } + } + var facts *moveFacts + for m, was := range sent { + now, carried := declared.Builds[m] + if !carried || sameCommit(now, was) { + continue + } + if facts == nil { + f, err := readMoveFacts(ctx, inv) + if err != nil { + return "", err + } + facts = &f + } + if !facts.identical(m, was, now) { + other = append(other, fmt.Sprintf("%s %s → %s", m, short(was), short(now))) + } + } + for _, f := range declared.foreseen { + other = append(other, f+" would be minted") + } + body, err := declared.Body() + if err != nil { + return "", err + } + after, err := summarize(body) + if err != nil { + return "", err + } + prev, err := inv.SentSummary(ctx, holder) + if err != nil { + return "", err + } + if len(prev) == 0 { + return fmt.Sprintf("what %s was last sent is not kept for comparison", holder), nil + } + var before []sentResource + if err := json.Unmarshal(prev, &before); err != nil { + return "", fmt.Errorf("what %s was last sent is kept in a form this version does not read: %w", holder, err) + } + other = append(other, otherThanTheList(diffSent(before, after), listID)...) + sort.Strings(other) + if len(other) == 0 { + return "", nil + } + return "its send would change more than the bus's user list: " + strings.Join(other, "; "), nil +} + +// otherThanTheList is what a diff against the last send changes besides the user list's content: every +// resource added, removed or changed, but the list's own resource when its content is all that changed — +// its path, mode and every other field must be as last sent (novox/hq issue 490). +func otherThanTheList(d sentDiff, listID string) []string { + var other []string + for _, id := range d.Added { + other = append(other, "+"+id) + } + for _, id := range d.Removed { + other = append(other, "-"+id) + } + for _, c := range d.Changed { + if c.ID == listID && len(c.Fields) == 1 && c.Fields[0] == "content" { + continue + } + other = append(other, fmt.Sprintf("~%s (%s)", c.ID, strings.Join(c.Fields, ", "))) + } + return other +} + +// sendJudged is a gated send's sends: the grants step when the user list changed, then the machine judged. +func sendJudged(ctx context.Context, open *stores, node string) ([]string, *inventory.GrantsStep, error) { + grants := grantsStep(ctx, open, node) + sent, err := sendRollout(withScope(ctx, sendScope{judged: map[string]bool{node: true}}), open, []string{node}) + return sent, grants, err +} + +// grantsSaid is a grants step as `plans` and a walk's note say it. +func grantsSaid(g *inventory.GrantsStep) string { + switch { + case g == nil: + return "" + case g.Failed != "": + return fmt.Sprintf(grantsStepWord+" to %s FAILED, sent without it: %s", g.Node, g.Failed) + default: + return fmt.Sprintf(grantsStepWord+": %s sent the bus's user list first, its builds kept", g.Node) + } +} diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 7adc089c..d3a9b260 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -971,7 +971,7 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string } // A gated send (ADR 0236): everything waiting on the machine goes with the tier, and the gate judges // all of it there. - carried, sent, err := gatedSend(ctx, open, node, owns) + carried, sent, grants, err := gatedSend(ctx, open, node, owns) if err != nil { return err } @@ -992,6 +992,7 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string s.GatedBy = "" if m == lead { s.Gate.Carried = carried + s.Gate.Grants = grants } else { s.GatedBy = lead } @@ -1001,6 +1002,10 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string strings.Join(sent, ", ")) fmt.Printf("%s: tier %d built; sent %d module(s) to %s first in one send (%s), the rest once its gate passes\n", p.ID, p.Tier, len(modules), strings.Join(sent, ", "), strings.Join(modules, ", ")) + // The grants step taken before it (novox/hq issue 490), said with it. + if said := grantsSaid(grants); said != "" { + p.Note += "; " + said + } // What the send recreates, said with it (novox/hq ADR 0245). if said := recreationsSaid(carried); said != "" { p.Note += "; " + said diff --git a/cmd/mesh-controller/release_test.go b/cmd/mesh-controller/release_test.go index 11c4c828..0560cdd8 100644 --- a/cmd/mesh-controller/release_test.go +++ b/cmd/mesh-controller/release_test.go @@ -160,6 +160,7 @@ func (b *backlogMesh) release(t *testing.T) inventory.Plan { // The backlog goes out one machine at a time: the first judged before the second is sent, each build's // pass kept; a rebuild with the same bytes is no move at all; and a send outside a gate is refused. func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) { + judgeLeavesAsTheCore(t) b := aBacklog(t) ctx := t.Context() inv := b.open.inventory diff --git a/cmd/mesh-controller/replay335_test.go b/cmd/mesh-controller/replay335_test.go index 1b0c4d33..2ec3e0f3 100644 --- a/cmd/mesh-controller/replay335_test.go +++ b/cmd/mesh-controller/replay335_test.go @@ -77,6 +77,7 @@ func testAWalkStoppedAfterItsGatePassedKeepsThePass(t *testing.T) { // the first send is older than the wait for a first machine's report. Read before the pass is taken over, the // carried module's step said the first machine never applied it, and the passed build was put back. func TestACarriedModuleTakesItsLeadsPassBeforeItsStepIsRead(t *testing.T) { + judgeLeavesAsTheCore(t) tm := aTierMesh(t, "m01", "m02") ctx := t.Context() inv := tm.open.inventory diff --git a/cmd/mesh-controller/rotate.go b/cmd/mesh-controller/rotate.go index 06d7a88a..f9168ddf 100644 --- a/cmd/mesh-controller/rotate.go +++ b/cmd/mesh-controller/rotate.go @@ -55,6 +55,15 @@ func rotateCommand(ctx context.Context, args []string) error { defer open.Close() inv := open.inventory + // A credential the provider's software issued is not the mesh's to remake (novox/hq ADR 0305 §9). + shelf, err := inv.Catalogue(ctx) + if err != nil { + return err + } + if err := issuedRefusesRotation(shelf, provision); err != nil { + return err + } + holders, err := inv.HoldersOf(ctx, provision, *only) if err != nil { return err diff --git a/cmd/mesh-controller/same_source_test.go b/cmd/mesh-controller/same_source_test.go index be2dc8c2..01abd454 100644 --- a/cmd/mesh-controller/same_source_test.go +++ b/cmd/mesh-controller/same_source_test.go @@ -162,6 +162,7 @@ func TestABusRebuiltFromAnUnchangedSourceDemandsNoBusStep(t *testing.T) { // A plan's build made from the source every machine running the module runs is not sent and not // judged: nothing moves. A build of a changed source is sent to its first machine, gated, as before. func TestAPlanSendsNothingForAnUnchangedSource(t *testing.T) { + judgeLeavesAsTheCore(t) for _, tc := range []struct { name string source string diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index 81838973..2b013154 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -410,6 +410,10 @@ func (a *verbArguments) commandLine() ([]string, error) { if b := str("by"); b != "" { argv = append(argv, "--by", catalogue.DeliverySeat+" for "+b) } + // The merges to leave out: the walk's other merges are walked again without them (novox/hq ADR 0299). + if w := str("without"); w != "" { + argv = append(argv, "--without", w) + } return argv, nil case "delivery-walks": argv := []string{"delivery", "walks"} @@ -773,6 +777,13 @@ func (a *verbArguments) commandLine() ([]string, error) { return nil, err } return []string{"secret", "ask", str("node"), str("module"), str("secret"), "--at", str("at")}, nil + case "issued": + // The credential a provider's software issued, taken from the provider (novox/hq ADR 0305): no value in + // the arguments, the provision alone. + if err := need("provision"); err != nil { + return nil, err + } + return []string{"secret", "issued", str("provision")}, nil case "secret-ask": // A module's own secret asked for, typed by the operator at the desk (novox/hq ADR 0277): never a value // in the arguments. The desk is the module's machine unless at names another. @@ -1579,6 +1590,14 @@ func givenAtTheDesk(argv []string) bool { return argv[5] == "--at" && argv[6] != "" && !strings.HasPrefix(argv[6], "-") } +// takenFromTheProvider is exactly the line the `issued` verb composes, and nothing beside it: `secret issued +// ` (novox/hq ADR 0305 §10). It carries no value — the value comes from the provider's own tool, sealed to +// the call — and answers none. +func takenFromTheProvider(argv []string) bool { + return len(argv) == 3 && argv[0] == "secret" && argv[1] == "issued" && argv[2] != "" && + !strings.HasPrefix(argv[2], "-") +} + // terminalOnly refuses, through any verb, a command that is the operator's at the controller's terminal // alone (novox/hq ADR 0266). **Every `node` subcommand that is not a read**: `node account` and // `node agent-account` above all. Whoever may call a verb includes agents, and an agent that named itself @@ -1595,7 +1614,7 @@ func terminalOnly(argv []string) error { // Of a secret's commands only rotation, which seals the new value to the machine that uses it, and the // `give` verb's own line: an own secret typed by the operator into the desk's hidden prompt, sealed to this // call and then to the module's machine, so no value travels in the verb or its answer (hq ADR 0259 §10). - if argv[0] == "secret" && (len(argv) < 2 || argv[1] != "rotate") && !givenAtTheDesk(argv) { + if argv[0] == "secret" && (len(argv) < 2 || argv[1] != "rotate") && !givenAtTheDesk(argv) && !takenFromTheProvider(argv) { return terminalRefusal("secret %s is run at the controller's terminal only, never through a verb: accepting, "+ "recovering or exporting a secret hands it to whoever asks, and that includes agents (novox/hq ADR "+ "0266). Nothing was done", strings.Join(argv[1:], " ")) diff --git a/cmd/mesh-controller/secret.go b/cmd/mesh-controller/secret.go index 37c3cc3c..3b15167c 100644 --- a/cmd/mesh-controller/secret.go +++ b/cmd/mesh-controller/secret.go @@ -41,6 +41,9 @@ func secretCommand(ctx context.Context, args []string) error { case "accept": case "ask": return secretAsk(ctx, args[1:]) + case "issued": + // The credential a provider's software issued, taken from the provider (novox/hq ADR 0305). + return secretIssued(ctx, args[1:]) case "rotate": return secretRotate(ctx, args[1:]) case "recover": @@ -168,6 +171,7 @@ func secretAsk(ctx context.Context, args []string) error { const secretUsage = "secret rotate [--why [--cause ]]\n" + "secret ask [--at ]\n" + + "secret issued \n" + "secret accept [--from | --at-desk ] [--provider [--local ]]\n" + "secret recover --key [--out ] [--from-export ] [--provider ]\n" + "secret export [--out ]" diff --git a/cmd/mesh-controller/standing_test.go b/cmd/mesh-controller/standing_test.go index e7fd3121..d174dd0c 100644 --- a/cmd/mesh-controller/standing_test.go +++ b/cmd/mesh-controller/standing_test.go @@ -66,7 +66,7 @@ func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) { } // Both machines' `node show` name it: where the provider runs, and where the consumer is. for _, node := range []string{"anchor", "laptop"} { - shown := printed(t, func() error { return showNode(ctx, open.inventory, node) }) + shown := printed(t, func() error { return showNode(ctx, open, node) }) if !strings.Contains(shown, "open condition(s) about this machine") || !strings.Contains(shown, "mesh_laptop_dashboard") { t.Fatalf("node show %s does not name it:\n%s", node, shown) } diff --git a/cmd/mesh-controller/status.go b/cmd/mesh-controller/status.go index a9699cce..af68b7f5 100644 --- a/cmd/mesh-controller/status.go +++ b/cmd/mesh-controller/status.go @@ -289,6 +289,10 @@ func printStatus(asked answers) error { fmt.Printf("\n `take ` compares what runs against what it declares, and runs it\n\n") } + // Assigned and not applied (novox/hq issue 380): before what is merely reported, because it reads like work + // finished and is none. + printNotApplied(asked.leftOut) + if len(asked.unheld) > 0 { // **Reported, and not refused yet** (novox/hq ADR 0207 §4). Each machine still resolves and // is sent what it would be; this says which of its modules depend on a seat nothing there @@ -337,6 +341,10 @@ func printStatus(asked answers) error { "lists them, and each is in its condition's tried\n\n", asked.heals.Acts, asked.heals.Escalated) } + // What each machine was last sent and by whom, one line each (novox/hq issue 485). Not a fault, so it does not + // break "all well"; a record that could not be read does. + printLastSends(nodes, asked.lastSent, asked.lastSentUnread) + if adopted := adoptedNodes(nodes); len(adopted) > 0 { // Said, because nothing forces the flip: a node left adopted is visible here rather than // read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well". @@ -456,6 +464,13 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) { return answers{}, err } plans[n.Name] = planned{plan, settings} + // And which of its modules a push leaves out (novox/hq issue 380), judged as the push judges it. + if left := leftOutOf(plan, settings, n.Adopted); len(left) > 0 { + if out.leftOut == nil { + out.leftOut = map[string][]leftOutModule{} + } + out.leftOut[n.Name] = left + } out.unheld = append(out.unheld, plan.Unheld...) // And which of its modules a provider leaves out of its grants, for an identity too long // for what the provision keeps (novox/hq ADR 0225) — judged from the consumer's own @@ -474,6 +489,12 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) { out.pendingUnread = err.Error() err = nil } + // And what each machine was last sent, by whom (novox/hq issue 485): the record of sends is read through + // the controller's verbs alone, the store's reader being shut out of its tables. Unreadable is said. + if out.lastSent, err = inv.LastSends(ctx); err != nil { + out.lastSentUnread = err.Error() + err = nil + } out.plans, err = inv.RecentPlans(ctx, 5) if err != nil { return answers{}, err @@ -604,9 +625,10 @@ func untakenModules(ctx context.Context, inv *inventory.Inventory, nodes []inven // read as success for the whole of the edge cut-over outage (novox/hq 04-ISSUES/125). func (a answers) well() bool { return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 && - len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 && + len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 && len(a.leftOut) == 0 && len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.overflowing) == 0 && - len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending) + len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending) && + a.lastSentUnread == "" } // pendingOpen is whether any pending assignment is still to be made. One that ended without being made is diff --git a/cmd/mesh-controller/testdata/left-out/nfs-server.json b/cmd/mesh-controller/testdata/left-out/nfs-server.json new file mode 100644 index 00000000..780e09ba --- /dev/null +++ b/cmd/mesh-controller/testdata/left-out/nfs-server.json @@ -0,0 +1,142 @@ +{ + "module": "nfs-server", + "version": "1", + "upgrade": { + "policy": "record", + "why": "the folders other machines mount: a build that breaks the exports leaves every client's mount hanging or refused, and the gate on this one machine does not see the clients (hq ADR 0236, ADR 0263)" + }, + "capabilities": [ + "package-manager", + "service-manager" + ], + "provides": [ + { + "name": "nfs-share", + "scope": "mesh", + "identity": false + } + ], + "data": { + "consumers": { + "nfs-share": { + "class": "none", + "why": "the shared folders are the operator's data (hq ADR 0051): what a client writes lands in them, and they are protected where the operator declares them, never by this module, which keeps nothing of a consumer's" + } + } + }, + "claims": [ + { + "name": "node-nfs-server", + "scope": "node", + "serves": [ + "exports", + "clients", + "test", + "reload", + "adopt" + ] + } + ], + "state": [ + { + "name": "exports", + "ttl-seconds": 120, + "per-machine": true + } + ], + "tools": [ + "nfs_health" + ], + "listens": [ + { + "name": "nfs", + "port": 2049, + "protocol": "tcp", + "from": "mesh", + "fixed": true, + "why": "the shares, to the mesh's machines only (hq ADR 0263): NFS version 4 alone, which needs no other port, and never the home network, where a device that is not a node could claim any user id" + } + ], + "resources": [ + { + "id": "package", + "type": "package", + "package": "nfs-utils" + }, + { + "id": "nfs-conf", + "type": "file", + "path": "/etc/nfs.conf.d/50-mesh.conf", + "mode": "0644", + "content": "# Written by the mesh (module nfs-server, novox/hq ADR 0263). Replaced on every push; a drop-in of\n# the operator's that sorts after this one overrides it, and is theirs.\n#\n# NFS version 4 only: a client needs port 2049 and nothing else, so the module opens nothing more\n# than that, to the private network. Version 3 needs rpcbind and mountd, on ports the mesh does not open.\n[nfsd]\nvers2=n\nvers3=n\nvers4=y\nvers4.0=n\nvers4.1=y\nvers4.2=y\n" + }, + { + "id": "config-dir", + "type": "directory", + "path": "/etc/nfs-server", + "mode": "0755" + }, + { + "id": "config", + "type": "file", + "path": "/etc/nfs-server/shares.conf", + "mode": "0644", + "content": "# Written by the mesh (module nfs-server, novox/hq ADR 0263) from this machine's assignment.\n# Replaced on every push; change the `shares` setting, never this file.\n#\n# The shares: name=folder, or name=folder:ro, one share per folder. The module's process exports each\n# to the private network's range below, every client mapped to the folder's owner.\nshares=${setting:shares}\nrange=${machine:mesh-range}\n" + }, + { + "id": "run-dir", + "type": "directory", + "path": "/run/nfs-server", + "mode": "0755" + }, + { + "id": "server", + "type": "service", + "unit": "nfs-server.service", + "state": "running", + "boot": "enabled", + "restart-on": [ + "nfs-conf" + ], + "health": { + "kind": "unit" + } + }, + { + "id": "exports", + "type": "process", + "name": "nfs-server-exports", + "artifact": "tools", + "run": [ + "./nfs-server", + "exports" + ], + "restart-on": [ + "config" + ], + "health": { + "kind": "tool", + "tool": "nfs_health", + "interval": "60s", + "timeout": "10s", + "looks": 2, + "grace": "90s" + } + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/nfs-server", + "binary": "nfs-server", + "loads": [ + "nfs-server" + ] + } + ] + } +} diff --git a/cmd/mesh-controller/tier_send_test.go b/cmd/mesh-controller/tier_send_test.go index 34be200d..c91a8fc0 100644 --- a/cmd/mesh-controller/tier_send_test.go +++ b/cmd/mesh-controller/tier_send_test.go @@ -167,6 +167,7 @@ func (tm *tierMesh) raise(machine, kind, summary string) { // The incident's tier, small: a dozen modules whose first machine is the same are sent there in one // send, judged by one gate, and the rest of their machines are sent once — two sends, not two dozen. func TestATierOfManyModulesIsSentToEachMachineOnce(t *testing.T) { + judgeLeavesAsTheCore(t) var modules []string for i := 1; i <= 12; i++ { modules = append(modules, fmt.Sprintf("m%02d", i)) @@ -268,6 +269,7 @@ func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) { // A machine-level condition about anything else holds the machine back as a whole: everything the send // moved there fails together, once, with that reason — not one module for it. func TestAMachineUnhealthyAsAWholeFailsTheWholeSendTogether(t *testing.T) { + judgeLeavesAsTheCore(t) tm := aTierMesh(t, "app1", "app2", "app3") ctx := t.Context() inv := tm.open.inventory @@ -330,6 +332,7 @@ func TestTheConditionsAboutAMachineItself(t *testing.T) { // rollback that failed; and `plans retry` asks it again rather than refusing a build that was never // judged (the incident's "NOT put back: no build kept"). func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) { + judgeLeavesAsTheCore(t) g := aGateMesh(t) ctx := t.Context() inv := g.open.inventory @@ -384,6 +387,7 @@ func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) { // them, they are registered back and marked in the judging that finds them broken, the plan goes on judging // the other module to its own pass, and only then fails. func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) { + judgeLeavesAsTheCore(t) for _, order := range [][]string{{broker.RuntimeModule, "app1"}, {"app1", broker.RuntimeModule}} { t.Run("gate kept on "+order[0], func(t *testing.T) { tm := aTierMesh(t, order...) @@ -435,3 +439,38 @@ func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) { }) } } + +// **A send carrying a core module waits for the core's readings** (issue 524 review): the leaf beside the node +// tools has its one reading at once, but the send is judged as the core: not before the settle time, and three +// readings. +func TestASendCarryingTheCoreIsJudgedAsTheCore(t *testing.T) { + tm := aTierMesh(t, broker.RuntimeModule, "app1") + ctx := t.Context() + // One reading counted per advance, as every test with a gate left open does: a gate judged with no time + // between readings would count a pass at every look and never let an advance rest. + gateEvery, gateSettle = time.Hour, time.Hour + for i := 0; i < 6; i++ { + advancePlans(ctx, tm.open) + } + p := tm.plan(t) + lead := p.Modules[broker.RuntimeModule].Gate + if lead == nil { + lead = p.Modules["app1"].Gate + } + if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) || !p.Open() || lead == nil || lead.Verdict != "" || lead.Wants != gatePasses { + t.Fatalf("passed before the core's settle time: sent %v, plan %s (%s), gate %+v", tm.sent, p.State, p.Note, lead) + } + gateEvery, gateSettle = 0, 0 + for i := 0; i < 6; i++ { + advancePlans(ctx, tm.open) + } + p = tm.plan(t) + if p.State != inventory.PlanDone || !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"laptop"}}) { + t.Fatalf("the plan is %s (%s), sent %v", p.State, p.Note, tm.sent) + } + for _, m := range []string{broker.RuntimeModule, "app1"} { + if g := p.Modules[m].Gate; g != nil && (g.Verdict != inventory.GatePassed || g.Passes < gatePasses) { + t.Fatalf("%s's gate: %+v; want passed on %d readings", m, g, gatePasses) + } + } +} diff --git a/internal/broker/issued_test.go b/internal/broker/issued_test.go new file mode 100644 index 00000000..3e0a36d7 --- /dev/null +++ b/internal/broker/issued_test.go @@ -0,0 +1,53 @@ +package broker + +import "testing" + +// The controller asks a tool that answers a software-issued credential in its own name, and nothing else under +// `call` (novox/hq ADR 0305 §2, issue 526): the grant is composed from the records, so the installer's first user +// list, a controller with no records, carries none. +func TestTheControllerAsksAnIssuedCredentialsToolInItsOwnNameAlone(t *testing.T) { + r := Records{Nodes: []string{"n", "ace"}, Assigned: map[string][]Declared{ + "n": {{Module: "x", Issued: []string{"t"}}}, + "ace": {{Module: "plex", Issued: []string{"plex_issued_credential"}}, {Module: "kometa"}}, + }} + users, err := Users(r) + if err != nil { + t.Fatal(err) + } + controller := users[0] + if controller.Kind != KindController { + t.Fatalf("the first user is %s", controller.Username()) + } + perms, err := PermissionsFor(controller) + if err != nil { + t.Fatal(err) + } + for _, s := range []string{ + "mesh.mod.x.call.t.n.controller", + "mesh.mod.plex.call.plex_issued_credential.ace.controller", + } { + if !MayPublish(perms, s) { + t.Errorf("the controller may not publish %s", s) + } + } + for _, s := range []string{ + "mesh.mod.x.call.t.n.ace.node-tools", + "mesh.mod.x.call.t.n." + CallerToken("ace.node-tools"), + "mesh.mod.plex.call.plex_issued_credential.ace." + CallerToken("ace.node-tools"), + "mesh.mod.plex.call.plex_status.ace.controller", + "mesh.mod.kometa.call.t.ace.controller", + "mesh.seat.mesh-controller.call.status.controller", + } { + if MayPublish(perms, s) { + t.Errorf("the controller may publish %s", s) + } + } + + bare, err := PermissionsFor(Principal{Kind: KindController}) + if err != nil { + t.Fatal(err) + } + if MayPublish(bare, "mesh.mod.x.call.t.n.controller") { + t.Fatal("a controller composed with no records is granted a caller-named call") + } +} diff --git a/internal/broker/nats.go b/internal/broker/nats.go index 3cdf473a..2280c9cf 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -123,6 +123,13 @@ type Principal struct { // permission and nothing beside it. Invokes []string + // AsksIssued are the tools the controller asks in its own name for a credential a provider's software + // issued, each `.` (novox/hq ADR 0305): on the subject that names its caller, so the tool + // knows the controller asks. Composed from the records, so the installer's first user list — a controller + // with no records — carries none, and moves with issue 464 like the rest of the controller's caller-named + // grants. + AsksIssued []string + // State is the local names of the state this principal's module keeps, and Reads the state of // others it reads as `.` (novox/hq ADR 0201): a bucket each, kept by the owner's // instances and read by whoever declares it. @@ -900,6 +907,17 @@ func PermissionsFor(p Principal) (Permissions, error) { // and moves with issue 464. if p.Kind != KindController { pub, sub = callerNamed(p.Username(), pub, sub) + } else { + // **But the tools that answer a software-issued credential** (novox/hq ADR 0305 §2): those answer the + // controller alone, and know it only from the subject's last token, so it asks them in its own name — + // those tools, on any machine, and nothing else under `call`. + for _, key := range p.AsksIssued { + module, tool, ok := strings.Cut(key, ".") + if !ok || module == "" || tool == "" || strings.ContainsAny(key, "*> ") { + continue + } + pub = append(pub, "mesh.mod."+module+"."+CallKind+"."+tool+".*."+CallerToken(p.Username())) + } } sort.Strings(pub) diff --git a/internal/broker/users.go b/internal/broker/users.go index 08cac990..1768b028 100644 --- a/internal/broker/users.go +++ b/internal/broker/users.go @@ -29,6 +29,9 @@ type Declared struct { Holds []Seat // Uses are the seats this module sends to. Uses []Seat + // Issued are the tools that answer a credential its software issued, one per offer that says so (novox/hq + // ADR 0305): the controller is granted to ask each in its own name, and no other tool so. + Issued []string // Watches are the seats whose events it consumes. Watches []Seat // Invokes are the tools it calls, `.` or `*` (novox/hq ADR 0152). @@ -84,7 +87,7 @@ type Records struct { // is a mesh that cannot be told anything, and there is no state of the records in which that is // correct. func Users(r Records) ([]Principal, error) { - out := []Principal{{Kind: KindController, Uses: asksSeatOf(r)}} + out := []Principal{{Kind: KindController, Uses: asksSeatOf(r), AsksIssued: issuedOf(r)}} for _, node := range sortedCopy(r.Nodes) { witness := false @@ -206,6 +209,25 @@ const controllerModule = "mesh-controller" // asksSeatOf is the seat an ask is made on, as its holder declares it (novox/hq ADR 0259): the controller // asks the operator through it like any other user, and is granted what its declaration names for a caller. // None while nothing holds it. +// issuedOf is every tool of a module assigned anywhere that answers a credential its software issued, as +// `.` (novox/hq ADR 0305): what the controller asks in its own name, and nothing beside. +func issuedOf(r Records) []string { + seen := map[string]bool{} + var out []string + for _, node := range sortedCopy(r.Nodes) { + for _, d := range r.Assigned[node] { + for _, t := range d.Issued { + if key := d.Module + "." + t; !seen[key] { + seen[key] = true + out = append(out, key) + } + } + } + } + sort.Strings(out) + return out +} + func asksSeatOf(r Records) []Seat { for _, node := range sortedCopy(r.Nodes) { for _, d := range r.Assigned[node] { diff --git a/internal/catalogue/contribution_dependencies_test.go b/internal/catalogue/contribution_dependencies_test.go index a1dc1b16..e56c9579 100644 --- a/internal/catalogue/contribution_dependencies_test.go +++ b/internal/catalogue/contribution_dependencies_test.go @@ -79,23 +79,38 @@ func TestAHolderMeetsItsOwnContribution(t *testing.T) { } } -func TestTwoModulesDeclaringOnePackageAreRefusedBeforeAnythingIsRecorded(t *testing.T) { +func TestTwoModulesMayDeclareOnePackage(t *testing.T) { pacman := withResources(mod("pacman", nil, nil, nil, Claim{Name: PackageManagerSeat}), res("package", "pacman-contrib")) bar := withResources(mod("bar", nil, nil, nil), res("package", "bar"), res("package", "pacman-contrib")) - other := withResources(mod("other", nil, nil, nil), res("package", "other")) - catalogue := map[string]Manifest{"pacman": pacman, "bar": bar, "other": other} + catalogue := map[string]Manifest{"pacman": pacman, "bar": bar} + if err := CollisionRefusal(catalogue, "laptop", []string{"pacman"}, []string{"bar"}); err != nil { + t.Fatalf("a second declaration of a package was refused (novox/hq ADR 0303): %v", err) + } +} - err := CollisionRefusal(catalogue, "laptop", []string{"pacman"}, []string{"bar"}) - if err == nil || !strings.Contains(err.Error(), "pacman-contrib") || !strings.Contains(err.Error(), "bar") { - t.Fatalf("the second owner of a package was not refused by name: %v", err) +func TestAPackageDeclaredPresentAndAbsentIsRefusedNamingBoth(t *testing.T) { + gone := res("package", "ufw") + gone["absent"] = true + filter := withResources(mod("nftables", nil, nil, nil), gone) + wants := withResources(mod("legacy", nil, nil, nil), res("package", "ufw")) + other := withResources(mod("other", nil, nil, nil), res("package", "other")) + catalogue := map[string]Manifest{"nftables": filter, "legacy": wants, "other": other} + + err := CollisionRefusal(catalogue, "laptop", []string{"nftables"}, []string{"legacy"}) + if err == nil || !strings.Contains(err.Error(), "ufw") || !strings.Contains(err.Error(), "nftables") || + !strings.Contains(err.Error(), "legacy") || !strings.Contains(err.Error(), "ADR 0303") { + t.Fatalf("a package declared present and absent was not refused naming both: %v", err) } - if err := CollisionRefusal(catalogue, "laptop", []string{"pacman"}, []string{"other"}); err != nil { - t.Errorf("a module declaring nothing shared is refused: %v", err) + if strings.Contains(err.Error(), "one owner per node") { + t.Errorf("the refusal gives the one-owner rule, which no longer holds for packages: %v", err) } - // A collision already on the node is status's, not a reason to refuse an unrelated assignment. - if err := CollisionRefusal(catalogue, "laptop", []string{"pacman", "bar"}, []string{"other"}); err != nil { - t.Errorf("an unrelated assignment is refused for a collision already there: %v", err) + if err := CollisionRefusal(catalogue, "laptop", []string{"nftables"}, []string{"other"}); err != nil { + t.Errorf("a module declaring nothing contradictory is refused: %v", err) + } + // A contradiction already on the node is status's, not a reason to refuse an unrelated assignment. + if err := CollisionRefusal(catalogue, "laptop", []string{"nftables", "legacy"}, []string{"other"}); err != nil { + t.Errorf("an unrelated assignment is refused for a contradiction already there: %v", err) } } diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index fb9010bc..a74ca6dc 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -342,18 +342,38 @@ func (e *NotMadeError) Error() string { // compose. Empty when every module composes. The same judgement SetSettings makes before storing. func (r Resolution) LeftOut(settings SettingsBy, adopted bool) map[string]string { out := map[string]string{} + for module, why := range r.LeftOutBecause(settings, adopted) { + out[module] = why.Error() + } + return out +} + +// LeftOutBecause is LeftOut with each reason as the error it was, so a reader can tell a setting nobody +// gave (an *UnsetSettingError) from any other cause and say it as the operator's to give (novox/hq issue +// 380). An UnreadManifestError for a stored manifest this controller cannot read whole. +func (r Resolution) LeftOutBecause(settings SettingsBy, adopted bool) map[string]error { + out := map[string]error{} for _, m := range r.Modules { if why := UnknownFieldReason(m); why != "" { - out[m.Module] = why + out[m.Module] = &UnreadManifestError{Module: m.Module, said: why} continue } if err := JudgeSettings(m, settings[m.Module], adopted); err != nil { - out[m.Module] = err.Error() + out[m.Module] = err } } return out } +// UnreadManifestError is a module left out because its stored manifest has a key this controller does not know +// (novox/hq ADR 0262): said once for the whole mesh, by the catalogue's own condition, not per machine. +type UnreadManifestError struct { + Module string + said string +} + +func (e *UnreadManifestError) Error() string { return e.said } + // Compose is Declaration with the owner of every resource said. func (r Resolution) Compose(with Rendering) (Composed, error) { owner := map[string]string{} diff --git a/internal/catalogue/delivery_seat.go b/internal/catalogue/delivery_seat.go index 7871f694..682723e1 100644 --- a/internal/catalogue/delivery_seat.go +++ b/internal/catalogue/delivery_seat.go @@ -21,7 +21,7 @@ func deliveryVerbs() []Verb { // Delivery time against the delivery budgets (novox/hq ADR 0282): what probe D16 reads, optional until its // holder serves it. {Name: "times", Description: "Delivery time: from a merge to every machine of its walk running the build. " + - "Each class's delivery budget (a leaf module five minutes, a core module ten), the last days' median and " + + "Each class's delivery budget (a leaf module three minutes, a core module ten), the last days' median and " + "worst, how many within and over, the newest delivery of each class, every delivery over its budget with its " + "longest phase, and the delivered ones whose time is not known. Given a delivery's id, its time phase by phase.", Input: schema(map[string]string{"days": "how many days back (default 7)", "id": "one delivery's id: its phases"}, nil), diff --git a/internal/catalogue/engine_guard_test.go b/internal/catalogue/engine_guard_test.go new file mode 100644 index 00000000..85b75970 --- /dev/null +++ b/internal/catalogue/engine_guard_test.go @@ -0,0 +1,71 @@ +package catalogue + +import ( + "go/ast" + "go/parser" + "go/token" + "path/filepath" + "slices" + "strconv" + "testing" + + "github.com/novox/mesh-controller/internal/beside" +) + +// The controller refuses a definition's resolved path by the node-engine's own rules, no more (novox/hq issue +// 496): the same lists, read here from mesh-host's internal/apply/placement_guard.go — in a merge check the clone +// beside it at the commit the mesh runs, elsewhere the copy captured in testdata/beside. When the engine's lists +// move, this fails until the controller's move with them, so the gate never refuses what the engine applies nor +// passes what it refuses by path alone. +func TestTheResolvedPathRulesAreTheNodeEnginesOwn(t *testing.T) { + file := filepath.Join(beside.Dir(t, "mesh-host"), "internal", "apply", "placement_guard.go") + parsed, err := parser.ParseFile(token.NewFileSet(), file, nil, 0) + if err != nil { + t.Fatalf("the node-engine's guard does not parse: %v", err) + } + lists := map[string][]string{} + consts := map[string]string{} + ast.Inspect(parsed, func(n ast.Node) bool { + spec, ok := n.(*ast.ValueSpec) + if !ok { + return true + } + for i, name := range spec.Names { + if i >= len(spec.Values) { + continue + } + switch v := spec.Values[i].(type) { + case *ast.CompositeLit: + for _, elt := range v.Elts { + if lit, ok := elt.(*ast.BasicLit); ok && lit.Kind == token.STRING { + s, _ := strconv.Unquote(lit.Value) + lists[name.Name] = append(lists[name.Name], s) + } + } + case *ast.BasicLit: + if v.Kind == token.STRING { + consts[name.Name], _ = strconv.Unquote(v.Value) + } + } + } + return true + }) + for name, ours := range map[string][]string{ + "protectedRoots": protectedRoots, "forbiddenBelow": forbiddenBelow, "engineTrees": engineTrees, + } { + theirs := lists[name] + if len(theirs) == 0 { + t.Errorf("the node-engine's guard names no %s any more; read it and say where its rule went", name) + continue + } + a, b := slices.Clone(ours), slices.Clone(theirs) + slices.Sort(a) + slices.Sort(b) + if !slices.Equal(a, b) { + t.Errorf("%s differs from the node-engine's:\n controller %v\n node-engine %v", name, a, b) + } + } + if consts["engineModule"] != engineModule { + t.Errorf("the node-engine's own module is %q there and %q here", consts["engineModule"], engineModule) + } +} diff --git a/internal/catalogue/issued_credential_test.go b/internal/catalogue/issued_credential_test.go new file mode 100644 index 00000000..221e67ae --- /dev/null +++ b/internal/catalogue/issued_credential_test.go @@ -0,0 +1,42 @@ +package catalogue + +import ( + "strings" + "testing" +) + +// A provision's credential may be issued by its provider's software (novox/hq ADR 0305, issue 526): the offer +// names the tool that answers it and no own secret, and every other shape is refused at parse. +func TestAnOfferMaySayItsSoftwareIssuesTheCredential(t *testing.T) { + m, err := ParseManifest([]byte(`{"module":"media","version":"1", + "provides":[{"name":"media-api","scope":"mesh","credential":{"issued-by":"software","tool":"media_issued_credential"}}], + "serves":{"media-api":{"port":32400}}}`)) + if err != nil { + t.Fatal(err) + } + if tool, issued := m.IssuedCredentialOf("media-api"); !issued || tool != "media_issued_credential" { + t.Fatalf("the offer's issued credential was not read: %q %v", tool, issued) + } + if _, shared := m.SharedCredentialOf("media-api"); shared { + t.Fatal("an issued credential was read as an own secret shared") + } + if _, issued := m.IssuedCredentialOf("other"); issued { + t.Fatal("a provision the module does not offer reads as issued") + } + + for want, raw := range map[string]string{ + "the one issuer an offer may name": `{"module":"d","version":"1", + "provides":[{"name":"d-api","credential":{"issued-by":"vendor","tool":"d_token"}}]}`, + "names no tool that answers it": `{"module":"d","version":"1", + "provides":[{"name":"d-api","credential":{"issued-by":"software"}}]}`, + "names its own secret": `{"module":"d","version":"1","own-secrets":{"token":{"path":"/p","taken":"at-start"}}, + "provides":[{"name":"d-api","credential":{"issued-by":"software","tool":"d_token","own":"token"}}]}`, + "the one issuer an offer may name ": `{"module":"d","version":"1", + "provides":[{"name":"d-api","credential":{"tool":"d_token"}}]}`, + } { + want = strings.TrimSpace(want) + if _, err := ParseManifest([]byte(raw)); err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("expected a refusal saying %q, got %v", want, err) + } + } +} diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index 611e46f5..efb32aea 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -268,9 +268,29 @@ func KeepsConsumerData(catalogue map[string]Manifest, provision string) bool { // MachineReach is whether a provision is usable only on its provider's own machine. func (o Offer) MachineReach() bool { return o.Reach == ReachMachine } -// OfferCredential names which of the provider's own secrets a provision's consumers receive. +// OfferCredential names which of the provider's own secrets a provision's consumers receive — or, with +// IssuedBy, that the provider's software issues the provision's one credential itself (novox/hq ADR 0305, +// issue 526): a media server's sign-in token from its vendor, which the mesh can neither make nor rotate. +// Tool is then the provider's own tool that answers that value, sealed to a key made for the controller's +// one call, and to the controller alone. type OfferCredential struct { - Own string `json:"own"` + Own string `json:"own,omitempty"` + IssuedBy string `json:"issued-by,omitempty"` + Tool string `json:"tool,omitempty"` +} + +// IssuedBySoftware says a provision's credential is issued by its provider's software (novox/hq ADR 0305). +const IssuedBySoftware = "software" + +// IssuedCredentialOf is the tool through which this module answers the credential its software issued for a +// provision, and whether its offer says the software issues it (novox/hq ADR 0305). +func (m Manifest) IssuedCredentialOf(provision string) (string, bool) { + for _, o := range m.Provides { + if o.Name == provision && o.Credential != nil && o.Credential.IssuedBy == IssuedBySoftware && o.Credential.Tool != "" { + return o.Credential.Tool, true + } + } + return "", false } // SharedCredentialOf is the own secret an offer of this module names as the provision's credential, @@ -1555,7 +1575,24 @@ func ParseManifest(raw []byte) (Manifest, error) { "makes is %d", m.Module, p, id.Max, shortest)) } } - if offer.Credential != nil { + if c := offer.Credential; c != nil && (c.IssuedBy != "" || c.Tool != "") { + // The software issues it (novox/hq ADR 0305): named by the tool that answers it, never by an own + // secret, which would be a second copy for the mesh to keep in step with the software's. + switch { + case c.IssuedBy != IssuedBySoftware: + problems = append(problems, fmt.Sprintf( + "%s provides %q with a credential issued by %q; the one issuer an offer may name is %q (ADR 0305)", + m.Module, p, c.IssuedBy, IssuedBySoftware)) + case c.Own != "": + problems = append(problems, fmt.Sprintf( + "%s provides %q with a credential its software issues and names its own secret %q too; the "+ + "software's value is the one, so the offer names no own secret (ADR 0305)", m.Module, p, c.Own)) + case strings.TrimSpace(c.Tool) == "": + problems = append(problems, fmt.Sprintf( + "%s provides %q with a credential its software issues and names no tool that answers it: "+ + "\"tool\": \"\" (ADR 0305)", m.Module, p)) + } + } else if offer.Credential != nil { own, declared := m.OwnSecrets[offer.Credential.Own] switch { case offer.Credential.Own == "": @@ -2145,6 +2182,7 @@ func ParseManifest(raw []byte) (Manifest, error) { problems = append(problems, m.undeclaredMounts()...) problems = append(problems, m.unknownDirRefs()...) problems = append(problems, m.unknownAccessRefs()...) + problems = append(problems, m.resolvedPathProblems()...) problems = append(problems, m.jailProblems()...) // What a module adds to the account's environment and to the login shell, and the holder's // placeholders for them (novox/hq ADR 0203, ADR 0204) — here, so the catalogue check refuses diff --git a/internal/catalogue/placement.go b/internal/catalogue/placement.go index b552ba30..400c5b3f 100644 --- a/internal/catalogue/placement.go +++ b/internal/catalogue/placement.go @@ -92,6 +92,105 @@ func systemPath(path string) string { return "" } +// Where no directory or file a module's definition resolves to may be (novox/hq issue 496). +// +// mesh-catalog #205 gave docker's `state` directory `"place": "."`, which resolves to /docker: with the +// default root, /var/lib/docker, every container's filesystem. systemPath judged only the `places` and `accesses` +// settings, so the default layout and a definition's own paths reached the merge gate unjudged; it composed every +// machine and passed, and only the node-engine refused the directory, at apply, failing the walk. +// +// **Judged where a definition is judged, never where a machine is composed.** resolvedPathProblems runs in +// ParseManifest, which every route a definition takes into the mesh passes: `module check`, registration of a +// build (the builder's and the registry verbs'), and the merge gate's reading of the changed repository. A +// refusal there stops one definition before it reaches any machine. Composition reads registered manifests +// without ParseManifest, and judges nothing of this: refusing there would fail the whole machine's declaration and +// freeze every module on it for one module's path, where the node-engine fails only that resource. +// +// **The node-engine's rules, no more** (mesh-host's internal/apply/placement_guard.go, with files judged as +// directories are after novox/hq issue 495, rule 6). A path is refused when it is one of protectedRoots or holds +// one, when it is at or below a tree in forbiddenBelow, or at or below one of engineTrees and its module is not +// the node-engine's. What the engine judges with what only the machine knows stays the engine's: where the +// runtimes really keep their data, links, the accounts' homes, a directory's owner below /etc. The lists are the +// engine's own words, and a test (engine_guard_test.go) holds them equal to mesh-host's beside this repository. +// systemPath stays the stricter rule for a setting: a setting is an operator's word about one machine, and the +// trees it lists (/etc, /usr, /run, the mesh's own) are where the mesh's own modules write by design. +var ( + protectedRoots = []string{"/", "/bin", "/boot", "/dev", "/etc", "/home", "/lib", "/lib32", "/lib64", + "/media", "/mnt", "/opt", "/proc", "/root", "/run", "/sbin", "/srv", "/sys", "/tmp", "/usr", "/usr/bin", + "/usr/lib", "/usr/lib64", "/usr/local", "/usr/local/bin", "/usr/local/lib", "/usr/local/sbin", "/usr/sbin", + "/usr/share", "/var", "/var/cache", "/var/lib", "/var/lib/mesh", "/var/log", "/var/run", "/var/tmp", + "/var/spool"} + forbiddenBelow = []string{"/proc", "/sys", "/dev", "/boot", "/root", "/var/spool", "/opt", "/var/lib/docker", + "/var/lib/containers", "/var/lib/containerd"} + engineTrees = []string{"/var/lib/mesh-host", "/usr/lib/nox-mesh-host"} +) + +// engineModule is the node-engine's own module, the one that places in engineTrees. +const engineModule = "mesh-host" + +// enginePath says why the node-engine refuses a directory or file at path for module, or "". +func enginePath(path, module string) string { + path = filepath.Clean(path) + if !filepath.IsAbs(path) { + return "" + } + atOrBelow := func(tree string) bool { return path == tree || strings.HasPrefix(path, tree+"/") } + for _, root := range protectedRoots { + if path == root || path == "/" || strings.HasPrefix(root, path+"/") { + return root + " is one of the machine's own directories, and owning it is owning everything in it" + } + } + for _, tree := range forbiddenBelow { + if atOrBelow(tree) { + return "nothing is placed in " + tree + } + } + if module != engineModule { + for _, tree := range engineTrees { + if atOrBelow(tree) { + return tree + " is the node-engine's own, placed in by its own module alone" + } + } + } + return "" +} + +// resolvedPathProblems is every directory and file of the definition whose path, resolved as a node with the +// default root resolves it, the node-engine would refuse (novox/hq issue 496). A path still holding a placeholder +// only a machine fills (a setting, an access the definition gives no default) is the engine's to judge. +func (m Manifest) resolvedPathProblems() []string { + dirs := dirsFor(m, Rendering{}) + accesses := map[string]string{} + for _, a := range m.Accesses { + if a.ID != "" && a.Path != "" { + accesses[a.ID] = a.Path + } + } + var problems []string + for _, r := range m.Resources { + kind := fmt.Sprint(r["type"]) + if kind != "directory" && kind != "file" { + continue + } + id := fmt.Sprint(r["id"]) + path, _ := r["path"].(string) + if kind == "directory" && path == "" { + path = dirs[id] + } + path, _ = dirFill(path, dirs, m.Module) + path, _ = accessFill(path, accesses, m.Module) + if path == "" || strings.Contains(path, "${") { + continue + } + if why := enginePath(path, m.Module); why != "" { + problems = append(problems, fmt.Sprintf("%s's %s %q resolves to %s, which the node-engine refuses: %s. "+ + "A module's directories and files are judged when its definition is, so the merge gate refuses it "+ + "before any machine does (novox/hq issue 496)", m.Module, kind, id, filepath.Clean(path), why)) + } + } + return problems +} + // accessRef is how a module names one of its accesses: ${access:}. var accessRef = regexp.MustCompile(`\$\{access:([a-z0-9][a-z0-9-]*)\}`) diff --git a/internal/catalogue/resolve.go b/internal/catalogue/resolve.go index 6ed02150..5cd3aecf 100644 --- a/internal/catalogue/resolve.go +++ b/internal/catalogue/resolve.go @@ -960,6 +960,7 @@ func checkResources(modules []Manifest) []string { var problems []string owner := map[string]string{} ownedPath := map[string]string{} // path → owning module, for the access check below + packagePresent, packageAbsent := map[string][]string{}, map[string][]string{} for _, m := range modules { // Compared placed, not as written (novox/hq ADR 0112): ${dir:state} is the same six @@ -992,7 +993,22 @@ func checkResources(modules []Manifest) []string { } continue } - for _, field := range []string{"path", "unit", "name", "package"} { + if fmt.Sprint(r["type"]) == "package" { + // **A package may be declared by several modules** (novox/hq ADR 0303): declaring it says + // it is needed, and the host installs a missing one, never removes one a declaration + // stopped naming, so a second declaration changes nothing on the machine. What two + // modules can contradict is presence: one declaring it present and another absent + // (ADR 0180) would have the host install and remove it in turn, so that pair is refused. + if name, _ := r["package"].(string); name != "" { + if r["absent"] == true { + packageAbsent[name] = append(packageAbsent[name], m.Module) + } else { + packagePresent[name] = append(packagePresent[name], m.Module) + } + } + continue + } + for _, field := range []string{"path", "unit", "name"} { value, ok := r[field].(string) if !ok || value == "" { if field == "path" && fmt.Sprint(r["type"]) == "directory" { @@ -1038,6 +1054,12 @@ func checkResources(modules []Manifest) []string { ownedPath[fact.Path] = m.Module } } + for _, name := range packageNames(packageAbsent) { + if wanted := packagePresent[name]; len(wanted) > 0 { + problems = append(problems, fmt.Sprintf("%s declares the package %q absent and %s declare it present (novox/hq ADR 0303)", + strings.Join(packageAbsent[name], ", "), name, strings.Join(wanted, ", "))) + } + } // An accessed path is the operator's, so no module may declare it as one of its own // (novox/hq ADR 0051). Refused here rather than silently tolerated: an owner would create and @@ -1308,3 +1330,13 @@ func answeredElsewhere(want string, node Node, world World, brokered, keeps map[ holder, held := HolderAmong(want, world.Offered[want], world.Held) return held && holder.Node != node.Name } + +// packageNames are the packages a map names, in order. +func packageNames(m map[string][]string) []string { + names := make([]string, 0, len(m)) + for n := range m { + names = append(names, n) + } + sort.Strings(names) + return names +} diff --git a/internal/catalogue/resolved_path_test.go b/internal/catalogue/resolved_path_test.go new file mode 100644 index 00000000..55b5519e --- /dev/null +++ b/internal/catalogue/resolved_path_test.go @@ -0,0 +1,127 @@ +package catalogue + +// A definition's directories and files are judged at their resolved paths when the definition is (novox/hq issue +// 496). mesh-catalog #205 gave docker's `state` directory `"place": "."`, which resolves to /docker — +// /var/lib/docker, every container's filesystem. The merge gate composed every machine and passed it; only the +// node-engine refused it, at apply, and failed the walk. ParseManifest is what `module check`, registration and the +// merge gate's reading of a changed repository all run, so a refusal here is a refusal at each of them. + +import ( + "strings" + "testing" +) + +func TestADirectoryPlacedInDockersDataIsRefusedWhereTheDefinitionIsJudged(t *testing.T) { + // The #205 shape, as it was merged. + _, err := ParseManifest([]byte(`{"module": "docker", "version": "1", + "resources": [{"id": "state", "type": "directory", "place": "."}]}`)) + if err == nil { + t.Fatal("a directory resolving to /var/lib/docker was accepted; the node-engine refuses it at apply") + } + for _, said := range []string{"docker", `"state"`, "/var/lib/docker", "issue 496"} { + if !strings.Contains(err.Error(), said) { + t.Errorf("the refusal names the module, the resource, the path and why; %q is missing from %q", said, err) + } + } +} + +func TestTheMeshsPlaceForDockerIsAccepted(t *testing.T) { + // The fix #205 needed: the mesh's own directory for the module, /mesh/docker. + m, err := ParseManifest([]byte(`{"module": "docker", "version": "1", "resources": [ + {"id": "state", "type": "directory", "place": "mesh"}, + {"id": "marker", "type": "file", "path": "${dir:state}/applied", "content": "x"}]}`)) + if err != nil { + t.Fatalf("place %q is in the mesh's tree, where the mesh writes for every module: %v", "mesh", err) + } + if got := dirsFor(m, Rendering{}); got["state"] != "/var/lib/mesh/docker" { + t.Fatalf("got %v", got) + } +} + +// The node-engine's rules, for directories and — as the engine judges them since novox/hq issue 495 — files. +func TestADefinitionIsRefusedWhereTheNodeEngineRefusesItsPaths(t *testing.T) { + refused := map[string]string{ + "a stated directory in docker's data": `{"module": "sidecar", "version": "1", "resources": [ + {"id": "volumes", "type": "directory", "path": "/var/lib/docker/volumes/x"}]}`, + "a file in containerd's data": `{"module": "images", "version": "1", "resources": [ + {"id": "f", "type": "file", "path": "/var/lib/containerd/x", "content": "x"}]}`, + "a file beneath a placed directory that climbs out of it": `{"module": "docker", "version": "1", "resources": [ + {"id": "state", "type": "directory", "place": "mesh"}, + {"id": "f", "type": "file", "path": "${dir:state}/../../containers/x", "content": "x"}]}`, + "a file in /boot": `{"module": "grub", "version": "1", "resources": [ + {"id": "cfg", "type": "file", "path": "/boot/grub/custom.cfg", "content": "x"}]}`, + "a directory that is the mesh's whole tree": `{"module": "mesh", "version": "1", "resources": [ + {"id": "all", "type": "directory", "place": "."}]}`, + "a directory that holds /etc": `{"module": "x", "version": "1", "resources": [ + {"id": "d", "type": "directory", "path": "/"}]}`, + "a directory in the node-engine's own tree, by another module": `{"module": "intruder", "version": "1", + "resources": [{"id": "d", "type": "directory", "path": "/var/lib/mesh-host/x"}]}`, + } + for name, raw := range refused { + if _, err := ParseManifest([]byte(raw)); err == nil || !strings.Contains(err.Error(), "issue 496") { + t.Errorf("%s: accepted, or refused for another reason: %v", name, err) + } + } + + // What the engine applies, the mesh's own modules' paths among them (read from every machine's live plan): + // the machine's configuration, run directories, programs below /usr/local, the node-engine's own trees by its + // own module, an account's keys, a module's own root, and — not on the engine's lists, so not refused here — + // below /lib and at /storage, /data, /services and /var/lock. + accepted := map[string]string{ + "a unit in /etc": `{"module": "power", "version": "1", "resources": [ + {"id": "d", "type": "directory", "path": "/etc/systemd/system/x.service.d"}, + {"id": "u", "type": "file", "path": "/etc/systemd/system/x.service.d/a.conf", "content": "x"}]}`, + "a run directory": `{"module": "fail2ban", "version": "1", "resources": [ + {"id": "run-dir", "type": "directory", "path": "/var/run/fail2ban"}]}`, + "a program in /usr/local/bin": `{"module": "claude-code", "version": "1", "resources": [ + {"id": "start", "type": "file", "path": "/usr/local/bin/claude-agent", "content": "x"}]}`, + "the node-engine's launcher and state, by the node-engine": `{"module": "mesh-host", "version": "1", "resources": [ + {"id": "launcher", "type": "file", "path": "/usr/lib/nox-mesh-host/launch", "content": "x"}, + {"id": "state", "type": "directory", "path": "/var/lib/mesh-host"}]}`, + "an account's .ssh": `{"module": "ssh-client", "version": "1", "resources": [ + {"id": "ssh-dir", "type": "directory", "path": "/home/someone/.ssh"}, + {"id": "config", "type": "file", "path": "/home/someone/.ssh/config", "content": "x"}]}`, + "a module's own root": `{"module": "mailu", "version": "1", "resources": [ + {"id": "state", "type": "directory", "place": "."}]}`, + "a file below /lib": `{"module": "udev", "version": "1", "resources": [ + {"id": "rule", "type": "file", "path": "/lib/udev/rules.d/99-x.rules", "content": "x"}]}`, + "directories at /storage, /data, /services and /var/lock": `{"module": "roots", "version": "1", "resources": [ + {"id": "a", "type": "directory", "path": "/storage"}, {"id": "b", "type": "directory", "path": "/data"}, + {"id": "c", "type": "directory", "path": "/services"}, {"id": "d", "type": "directory", "path": "/var/lock"}]}`, + } + for name, raw := range accepted { + if _, err := ParseManifest([]byte(raw)); err != nil { + t.Errorf("%s: refused: %v", name, err) + } + } +} + +func TestAPathThroughAnAccessIsJudgedWithTheAccessFilledIn(t *testing.T) { + // A file's path may name an access; the access's default path is the definition's, so it is judged with it. + _, err := ParseManifest([]byte(`{"module": "backup", "version": "1", + "accesses": [{"id": "images", "path": "/var/lib/docker/volumes"}], + "resources": [{"id": "marker", "type": "file", "path": "${access:images}/marker", "content": "x"}]}`)) + if err == nil || !strings.Contains(err.Error(), "/var/lib/docker/volumes/marker") || + !strings.Contains(err.Error(), "issue 496") { + t.Fatalf("a file reaching Docker's data through an access's default path was accepted: %v", err) + } + // An access the definition gives no path is placed by a setting, which Places and AccessPlaces judge, and on + // the machine by the node-engine: nothing here to resolve it against, so nothing is refused for it. + if _, err := ParseManifest([]byte(`{"module": "backup", "version": "1", + "accesses": [{"id": "images"}], + "resources": [{"id": "marker", "type": "file", "path": "${access:images}/marker", "content": "x"}]}`)); err != nil { + t.Fatalf("an access placed only by a setting cannot be judged at the definition: %v", err) + } +} + +func TestComposingAMachineIsNeverStoppedByOneModulesPath(t *testing.T) { + // A module registered from outside the catalogue never passes the gate. Refusing its path while a machine's + // declaration is composed would fail the whole declaration and freeze every module on that machine; the + // node-engine fails only the one resource. So composition leaves it to the engine. + m := Manifest{Module: "docker", Version: "1", Resources: []map[string]any{ + {"id": "state", "type": "directory", "place": "."}, + }} + if _, err := (Resolution{Node: "anchor", Modules: []Manifest{m}}).Declaration(Rendering{}); err != nil { + t.Fatalf("the machine's declaration failed for one module's path: %v", err) + } +} diff --git a/internal/catalogue/seat_dependencies.go b/internal/catalogue/seat_dependencies.go index 2acb6ec5..088d83fb 100644 --- a/internal/catalogue/seat_dependencies.go +++ b/internal/catalogue/seat_dependencies.go @@ -349,6 +349,10 @@ func CollisionRefusal(catalogue map[string]Manifest, node string, assigned, addi if before[p] { continue // on the node already; not this assignment's doing } + if strings.Contains(p, "ADR 0303") { + problems = append(problems, p) // a package declared present and absent says its own rule + continue + } problems = append(problems, p+" (novox/hq ADR 0210: one owner per node; the other module "+ "depends on the owner's seat instead)") } diff --git a/internal/catalogue/setting_into.go b/internal/catalogue/setting_into.go index 9ee165b7..30967997 100644 --- a/internal/catalogue/setting_into.go +++ b/internal/catalogue/setting_into.go @@ -38,6 +38,17 @@ func settingsUsed(content string) []string { return keys } +// UnsetSettingError is a module whose definition says ${setting:} where nothing sets that key: typed, so +// that whoever reads why a module was left out of a machine can tell a setting nobody gave — the operator's +// to give, named with the command that gives it — from any other reason (novox/hq issue 380). Its words are +// the refusal's, unchanged. +type UnsetSettingError struct { + Module, Setting string + said string +} + +func (e *UnsetSettingError) Error() string { return e.said } + // settingInto fills a file's ${setting:…} placeholders from the layers over a module. // // The last layer setting a key wins, which is the node's over the mesh's over the module's own @@ -58,12 +69,12 @@ func settingInto(resource map[string]any, layers []Layer, module string) error { for _, key := range settingsUsed(content) { value, set := settingValue(layers, key) if !set { - return fmt.Errorf( + return &UnsetSettingError{Module: module, Setting: key, said: fmt.Sprintf( "%s has a file that says ${setting:%s}, and nothing sets %q for it — an operator's "+ "value is the assignment's, never the definition's (novox/hq ADR 0112), and only a "+ "preference has a default in the definition (ADR 0262): "+ "`settings set %s ` with {%q: …}%s", - module, key, key, module, key, orNoSettings(layers)) + module, key, key, module, key, orNoSettings(layers))} } content = strings.ReplaceAll(content, "${setting:"+key+"}", plainly(value)) } @@ -114,11 +125,11 @@ func settingIntoUnit(resource map[string]any, layers []Layer, module string) err for _, key := range settingsUsed(unit) { value, set := settingValue(layers, key) if !set { - return fmt.Errorf( + return &UnsetSettingError{Module: module, Setting: key, said: fmt.Sprintf( "%s has a service whose unit says ${setting:%s}, and nothing sets %q for it — an operator's "+ "value is the assignment's, never the definition's (novox/hq ADR 0112): "+ "`settings set %s ` with {%q: …}%s", - module, key, key, module, key, orNoSettings(layers)) + module, key, key, module, key, orNoSettings(layers))} } v := plainly(value) if !unitPart.MatchString(v) { diff --git a/internal/catalogue/verbs.go b/internal/catalogue/verbs.go index f3aaedaf..013f5c77 100644 --- a/internal/catalogue/verbs.go +++ b/internal/catalogue/verbs.go @@ -160,8 +160,14 @@ var ControllerVerbs = []Verb{ Input: schema(map[string]string{"plan": "the walk's id", "why": "what lets it go: its turn, a person's release"}, []string{"plan"})}, {Name: "delivery-stop", Description: "End a walk on its delivery's word (novox/hq ADR 0239): failed, said as stopped " + - "by whom and why; what it asked still builds and registers, nothing further is asked or sent.", - Input: schema(map[string]string{"plan": "the walk's id", "why": "why", "by": "who stopped the delivery"}, + "by whom and why; what it asked still builds and registers, nothing further is asked or sent. Given merges to " + + "leave out, every other merge the walk answered is walked again — in the open batch of its kind, or a new one — " + + "without them, without each later merge of their repository, which contains one, and without each merge moving " + + "a module one moves, each left out too and named (novox/hq ADR 0299); a batch not yet cut drops the merges. The " + + "answer names the batch, the merges walked again, the merges left out, the modules the stopped walk sent that " + + "stay on their machines, and the merges elsewhere that will still deliver a stopped change.", + Input: schema(map[string]string{"plan": "the walk's id", "why": "why", "by": "who stopped the delivery", + "without": "merges to leave out, @, comma-separated"}, []string{"plan", "why"})}, {Name: "delivery-walks", Description: "The walks the controller keeps (novox/hq ADR 0239): every open one and the " + "last ended ones, each whole — its tiers, each module's state, first machines and first-node gate with its readings, and its " + @@ -243,6 +249,15 @@ var ControllerVerbs = []Verb{ "secret": "the own secret's name in the module's definition", "at": "the machine the operator sits at, where the prompt opens", }, []string{"node", "module", "secret", "at"})}, + {Name: "issued", Description: "Take the credential a provider's software issued for a provision — a media " + + "server's sign-in token — and keep it as that provision's one credential for every consumer (novox/hq ADR " + + "0305): the controller asks the provider's own tool on its machine, in the controller's own name, for the " + + "value sealed to a key made for this call alone, seals it to every holder and forgets it, and sends their " + + "machines when it changed. Carries no value and answers none; a value every holder already holds is not " + + "sealed again, so asking again restarts nothing. Refused for a provision no offer says its software issues.", + Input: schema(map[string]string{ + "provision": "the provision whose provider's software issues its credential, e.g. plex-api", + }, []string{"provision"})}, {Name: "secret-ask", Description: "Ask the operator for a module's own secret (novox/hq ADR 0277): a prompt " + "that shows nothing of what is typed opens at the desk — the module's machine, or the one at names — " + "naming the module, the secret and who asked; the operator types the value there, never on a channel " + diff --git a/internal/inventory/busrecords.go b/internal/inventory/busrecords.go index 30c1cc5b..fa540dff 100644 --- a/internal/inventory/busrecords.go +++ b/internal/inventory/busrecords.go @@ -3,6 +3,7 @@ package inventory import ( "context" "fmt" + "slices" "strings" "github.com/novox/mesh-controller/internal/broker" @@ -172,6 +173,9 @@ func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaratio Checks: catalogue.HealthChecks(m), // And whether it runs as an account of its own (novox/hq ADR 0259 §8). RunsAs: m.RunsAs, + // And the tools that answer a credential its software issued, which the controller asks in its own + // name (novox/hq ADR 0305). + Issued: issuedTools(m), } // Whether it can be given an account at all: delivered as its own secret named broker, so one // that declares none has nowhere to read it (novox/hq issue 195). @@ -199,6 +203,17 @@ func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaratio return d } +// issuedTools are the tools through which a module answers the credentials its software issues (novox/hq ADR 0305). +func issuedTools(m catalogue.Manifest) []string { + var out []string + for _, o := range m.Provides { + if tool, issued := m.IssuedCredentialOf(o.Name); issued && !slices.Contains(out, tool) { + out = append(out, tool) + } + } + return out +} + // bucketsOf is the state a module keeps, as the bus holds it. func bucketsOf(m catalogue.Manifest) []broker.Bucket { var out []broker.Bucket diff --git a/internal/inventory/issued.go b/internal/inventory/issued.go new file mode 100644 index 00000000..a4ec4cd1 --- /dev/null +++ b/internal/inventory/issued.go @@ -0,0 +1,155 @@ +package inventory + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "sort" + "strings" + + "github.com/novox/mesh-controller/internal/secrets" +) + +// A credential a provider's software issued, kept as the provision's one credential (novox/hq ADR 0305, issue +// 526). +// +// **The software made it, so the mesh carries it.** A media server's sign-in token comes from its vendor; the mesh +// can neither make nor rotate it. The controller fetches it from the provider's own tool, sealed to a key made for +// that one call, and hands the value here, which seals it to every consumer that holds the provision from that +// provider — in the pair credential's shape, so a consumer's definition does not know where it came from (ADR 0158 +// §5) — and to the operator, and forgets it. +// +// **Asked often, written seldom.** The provider asks hourly and on every start. The mesh keeps no plaintext, so it +// cannot compare values; it stamps every row with the value's fingerprint instead, and a value whose stamp every +// row already carries is not sealed again — sealing anew would change every consumer's declaration and restart +// each of them for nothing. + +// OriginIssued is a pair credential the provider's software issued and the provider gave (novox/hq ADR 0305). +const OriginIssued = "issued" + +// IssuedStamp is the generation a value issued by a provider's software is kept under: a fingerprint of the value, +// so the same value fetched again is known as the same without being kept. +func IssuedStamp(value string) string { + sum := sha256.Sum256([]byte(value)) + return "issued:sha256:" + hex.EncodeToString(sum[:8]) +} + +// IssuedKept is what keeping an issued value did: whether it was sealed anew, the pairs that hold it, and the +// machines a send must reach — the provider's and every holder's — when it was. +type IssuedKept struct { + Changed bool + // Holders are every pair that holds the provision from the provider; Sealed those sealed anew by this act. + Holders []Holder + Sealed []Holder + Machines []string + // Unsealable are the holders whose machine has no sealing key: left as they are, said by name. + Unsealable []Holder +} + +// KeepIssuedSecret seals a value a provider's software issued to every pair of provision from provider that does +// not hold it yet, under one stamp; a pair that does is left as it is, and when every pair does nothing is written +// (novox/hq ADR 0305 §4, §5). +func (i *Inventory) KeepIssuedSecret(ctx context.Context, provision, provider, value string) (IssuedKept, error) { + if strings.TrimSpace(value) == "" { + return IssuedKept{}, fmt.Errorf("there is nothing to seal") + } + providerNode, err := i.NodeByName(ctx, provider) + if err != nil { + return IssuedKept{}, err + } + providerKey, err := i.SealingKeyOf(ctx, provider) + if err != nil { + return IssuedKept{}, err + } + if providerKey == "" { + return IssuedKept{}, fmt.Errorf("%s has no sealing key, so %s cannot be sealed to it", provider, provision) + } + stamp := IssuedStamp(value) + + type row struct { + holder Holder + consumerID any + generation, heldC, heldP string + consumerKey string + } + rows, err := i.store.Pool().Query(ctx, + `select s.consumer, c.name, s.consumer_module, s.local, coalesce(s.generation, ''), + coalesce(s.consumer_key, ''), coalesce(s.provider_key, ''), coalesce(c.sealing_key, '') + from secret s join node c on c.id = s.consumer + where s.provider = $1 and s.name = $2 + order by c.name, s.consumer_module, s.local`, providerNode.ID, provision) + if err != nil { + return IssuedKept{}, err + } + var held []row + for rows.Next() { + var r row + if err := rows.Scan(&r.consumerID, &r.holder.Consumer, &r.holder.ConsumerModule, &r.holder.Local, + &r.generation, &r.heldC, &r.heldP, &r.consumerKey); err != nil { + rows.Close() + return IssuedKept{}, err + } + r.holder.Provision, r.holder.Provider = provision, provider + held = append(held, r) + } + rows.Close() + if err := rows.Err(); err != nil { + return IssuedKept{}, err + } + + var kept IssuedKept + var stale []row + for _, r := range held { + if r.consumerKey == "" { + kept.Unsealable = append(kept.Unsealable, r.holder) + continue + } + kept.Holders = append(kept.Holders, r.holder) + // Only a pair that does not hold this value, sealed to the keys its ends have now, is sealed anew: a + // holder that already does is left byte for byte as it is, so it is not restarted for another's sake. + if r.generation != stamp || r.heldC != r.consumerKey || r.heldP != providerKey { + stale = append(stale, r) + } + } + if len(stale) == 0 { + return kept, nil + } + + forOperator, operatorKey, err := i.operatorSeal(ctx, value) + if err != nil { + return IssuedKept{}, err + } + tx, err := i.store.Pool().Begin(ctx) + if err != nil { + return IssuedKept{}, err + } + defer func() { _ = tx.Rollback(ctx) }() + machines := map[string]bool{provider: true} + for _, r := range stale { + sealed, err := secrets.Accept(value, r.consumerKey, providerKey) + if err != nil { + return IssuedKept{}, err + } + if _, err := tx.Exec(ctx, + `update secret set for_consumer = $6, for_provider = $7, consumer_key = $8, provider_key = $9, + operator_sealed = $10, operator_key = $11, origin = $12, generation = $13, created_at = now() + where name = $1 and consumer = $2 and consumer_module = $3 and provider = $4 and local = $5`, + provision, r.consumerID, r.holder.ConsumerModule, providerNode.ID, r.holder.Local, + sealed.ForConsumer, sealed.ForProvider, sealed.ConsumerKey, sealed.ProviderKey, + forOperator, operatorKey, OriginIssued, stamp); err != nil { + return IssuedKept{}, err + } + machines[r.holder.Consumer] = true + kept.Sealed = append(kept.Sealed, r.holder) + } + if err := tx.Commit(ctx); err != nil { + return IssuedKept{}, err + } + kept.Changed = true + for m := range machines { + kept.Machines = append(kept.Machines, m) + } + sort.Strings(kept.Machines) + return kept, nil +} diff --git a/internal/inventory/issued_test.go b/internal/inventory/issued_test.go new file mode 100644 index 00000000..f8725f20 --- /dev/null +++ b/internal/inventory/issued_test.go @@ -0,0 +1,178 @@ +package inventory + +import ( + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// A value a provider's software issued is sealed to every pair that holds the provision from that provider, under +// one stamp of the value, and the same value given again writes nothing and sends nobody (novox/hq ADR 0305 §4, +// §5, issue 526). +func TestAnIssuedValueIsSealedToEveryHolderOnceAndNotAgain(t *testing.T) { + inv, ctx := twoNodesWithKeys(t) + consumerKey, consumerOpens := aSealingKey(t) + cn, _ := inv.NodeByName(ctx, "consumer") + if err := inv.RecordSealingKey(ctx, cn.ID, consumerKey); err != nil { + t.Fatal(err) + } + for _, m := range []string{"kometa", "tautulli"} { + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1", Requires: []string{"plex-api"}, + Secrets: map[string]string{"plex-api": "/run/plex-api.secret"}}, Source{}); err != nil { + t.Fatal(err) + } + // What the mesh made before the provider gave its value: a random value per pair (issue 164). + if _, err := inv.SecretFor(ctx, "plex-api", "consumer", m, "provider", ""); err != nil { + t.Fatal(err) + } + } + + kept, err := inv.KeepIssuedSecret(ctx, "plex-api", "provider", "the-servers-own-token") + if err != nil { + t.Fatal(err) + } + if !kept.Changed || len(kept.Holders) != 2 { + t.Fatalf("the value was not sealed to both holders: %+v", kept) + } + if len(kept.Machines) != 2 || kept.Machines[0] != "consumer" || kept.Machines[1] != "provider" { + t.Fatalf("the machines to send are the provider's and every holder's: %v", kept.Machines) + } + for _, m := range []string{"kometa", "tautulli"} { + got, err := inv.SecretFor(ctx, "plex-api", "consumer", m, "provider", "") + if err != nil { + t.Fatal(err) + } + if got.Origin != OriginIssued { + t.Fatalf("%s's pair says it came from %q, not the provider's software", m, got.Origin) + } + opened, ok := consumerOpens(got.ForConsumer) + if !ok || string(opened) != "the-servers-own-token" { + t.Fatalf("%s's copy is not the provider's value", m) + } + var generation string + pn, _ := inv.NodeByName(ctx, "provider") + if err := inv.store.Pool().QueryRow(ctx, `select generation from secret where name = 'plex-api' and consumer = $1 + and consumer_module = $2 and provider = $3`, cn.ID, m, pn.ID).Scan(&generation); err != nil { + t.Fatal(err) + } + if generation != IssuedStamp("the-servers-own-token") { + t.Fatalf("%s's pair is not stamped with the value's fingerprint: %q", m, generation) + } + } + before, _ := inv.SecretFor(ctx, "plex-api", "consumer", "kometa", "provider", "") + + // A consumer that binds later gets a pair of its own, and no holder's value is remade for it: the offer names + // no own secret, so ADR 0158 §6's remaking on a bind does not reach it (ADR 0305 §7). + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "overseerr", Version: "1", Requires: []string{"plex-api"}, + Secrets: map[string]string{"plex-api": "/run/plex-api.secret"}}, Source{}); err != nil { + t.Fatal(err) + } + late, err := inv.SecretFor(ctx, "plex-api", "consumer", "overseerr", "provider", "") + if err != nil { + t.Fatal(err) + } + if late.Origin == OriginIssued { + t.Fatal("a consumer that bound after the ask holds the issued value without an ask") + } + if held, _ := inv.SecretFor(ctx, "plex-api", "consumer", "kometa", "provider", ""); held.ForConsumer != before.ForConsumer { + t.Fatal("a later bind remade a holder's issued value") + } + + // The next ask seals the provider's value to the late consumer alone; the holders that had it are not touched. + again, err := inv.KeepIssuedSecret(ctx, "plex-api", "provider", "the-servers-own-token") + if err != nil { + t.Fatal(err) + } + if !again.Changed || len(again.Sealed) != 1 || again.Sealed[0].ConsumerModule != "overseerr" { + t.Fatalf("the late consumer was not sealed to, or not alone: %+v", again) + } + if held, _ := inv.SecretFor(ctx, "plex-api", "consumer", "kometa", "provider", ""); held.ForConsumer != before.ForConsumer { + t.Fatal("sealing the late consumer changed what a holder already held") + } + if got, _ := inv.SecretFor(ctx, "plex-api", "consumer", "overseerr", "provider", ""); got.Origin != OriginIssued { + t.Fatal("the late consumer does not hold the issued value after the ask") + } + + // And once every holder holds it, the same value writes nothing and sends nobody. + still, err := inv.KeepIssuedSecret(ctx, "plex-api", "provider", "the-servers-own-token") + if err != nil { + t.Fatal(err) + } + if still.Changed || len(still.Machines) != 0 { + t.Fatalf("the same value was sealed again, which restarts every consumer for nothing: %+v", still) + } + after, _ := inv.SecretFor(ctx, "plex-api", "consumer", "kometa", "provider", "") + if after.ForConsumer != before.ForConsumer { + t.Fatal("the same value given again changed what the consumer holds") + } + + changed, err := inv.KeepIssuedSecret(ctx, "plex-api", "provider", "a-token-plex-changed") + if err != nil { + t.Fatal(err) + } + if !changed.Changed { + t.Fatal("a changed value was not sealed") + } + got, _ := inv.SecretFor(ctx, "plex-api", "consumer", "tautulli", "provider", "") + if opened, ok := consumerOpens(got.ForConsumer); !ok || string(opened) != "a-token-plex-changed" { + t.Fatal("the changed value did not reach the holder") + } + + if _, err := inv.KeepIssuedSecret(ctx, "plex-api", "provider", " "); err == nil { + t.Fatal("an empty value was kept") + } +} + +// A consumer whose sealing key changed holds a value the mesh made in place of the issued one, and the next ask +// seals the issued value to it again; so does a pair a person accepted over it (novox/hq ADR 0305 §5, the review +// of mesh-controller #239). +func TestAnIssuedValueIsSealedAgainAfterAKeyChangeOrAnAcceptance(t *testing.T) { + inv, ctx := twoNodesWithKeys(t) + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "kometa", Version: "1", Requires: []string{"plex-api"}, + Secrets: map[string]string{"plex-api": "/run/plex-api.secret"}}, Source{}); err != nil { + t.Fatal(err) + } + if _, err := inv.SecretFor(ctx, "plex-api", "consumer", "kometa", "provider", ""); err != nil { + t.Fatal(err) + } + if kept, err := inv.KeepIssuedSecret(ctx, "plex-api", "provider", "the-servers-own-token"); err != nil || !kept.Changed { + t.Fatalf("the first ask did not seal: %+v %v", kept, err) + } + + // The consumer rejoins with a new key: the mesh remakes the pair, a value the software refuses. + newKey, opens := aSealingKey(t) + cn, _ := inv.NodeByName(ctx, "consumer") + if err := inv.RecordSealingKey(ctx, cn.ID, newKey); err != nil { + t.Fatal(err) + } + remade, err := inv.SecretFor(ctx, "plex-api", "consumer", "kometa", "provider", "") + if err != nil { + t.Fatal(err) + } + if remade.Origin != OriginMade { + t.Fatalf("a value the mesh remade says it came from %q", remade.Origin) + } + again, err := inv.KeepIssuedSecret(ctx, "plex-api", "provider", "the-servers-own-token") + if err != nil { + t.Fatal(err) + } + if !again.Changed || len(again.Sealed) != 1 { + t.Fatalf("the issued value was not sealed again after the key changed: %+v", again) + } + got, _ := inv.SecretFor(ctx, "plex-api", "consumer", "kometa", "provider", "") + if opened, ok := opens(got.ForConsumer); !ok || string(opened) != "the-servers-own-token" { + t.Fatal("the consumer does not hold the issued value under its new key") + } + + // A person accepts another value over it: the next ask seals the software's again. + if err := inv.AcceptSecretForPair(ctx, "plex-api", "consumer", "kometa", "provider", "", "a-typed-copy"); err != nil { + t.Fatal(err) + } + after, err := inv.KeepIssuedSecret(ctx, "plex-api", "provider", "the-servers-own-token") + if err != nil { + t.Fatal(err) + } + if !after.Changed { + t.Fatal("an accepted value over the issued one was read as still holding it") + } +} diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index bd03edc5..f90999a8 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -111,6 +111,12 @@ type PlanBatch struct { // a merge never shares a batch with one that waits for mesh-delivery's (ADR 0276, decided during the // build, 2026-10-10). Own bool `json:"own,omitempty"` + // Closed says the window was closed by a stop that walks a stopped walk's merges again in this batch: they + // were heard before, and an older commit never waits behind a newer one (novox/hq ADR 0299). Withheld are + // the modules a stopped merge moves: built at the branch, which holds that merge, they would deliver it, so + // the batch's walk does not build them. + Closed bool `json:"closed,omitempty"` + Withheld []string `json:"withheld,omitempty"` } // OwnPath says the record is a batch of merges on the controller's own path. @@ -187,6 +193,17 @@ type PlanDelivery struct { // TakenOverBy names the walk a walk folded before it started was taken over by: its merges are that // walk's now. TakenOverBy string `json:"taken_over_by,omitempty"` + // WalkedAgainBy names, on a walk stopped without some of its merges, the batch that walks its other merges + // again, and LeftOut the merges it left out (novox/hq issue 459, ADR 0299): a delivery of a merge left out + // ends stopped with the walk, one of a merge walked again follows that batch's walk. + WalkedAgainBy string `json:"walked_again_by,omitempty"` + LeftOut []PlanLeftOut `json:"left_out,omitempty"` + // Kept are the modules the stopped walk already sent and nothing walks again: the machines named keep running + // the stopped walk's build (novox/hq ADR 0299). + Kept []PlanKept `json:"kept,omitempty"` + // Withheld are, on a walk cut from a batch a stop walked merges again in, the modules it did not build: a + // stopped merge moves them, and built only as dependents at the branch they would deliver it (ADR 0299). + Withheld []string `json:"withheld,omitempty"` // Batch is the window of a batch; nil once it is cut into a walk. Batch *PlanBatch `json:"batch,omitempty"` // Alone says the walk walks one merge on its own commit, after a failed walk carried it in a later one: @@ -194,6 +211,26 @@ type PlanDelivery struct { Alone bool `json:"alone,omitempty"` } +// PlanLeftOut is a merge a stop left out of the batch that walks a stopped walk's merges again (novox/hq ADR +// 0299): a merge stopped, or a later merge of its repository's branch, which contains the stopped one and so +// cannot be walked without delivering it. +type PlanLeftOut struct { + Repository string `json:"repository"` + Commit string `json:"commit"` + // Contains is the stopped merge a later merge contains, or whose modules it shares; empty for a merge + // stopped itself. + Contains string `json:"contains,omitempty"` + // Shares are the modules it moves that a stopped merge moves too: built at the branch, which holds the stopped + // merge, they would deliver it. + Shares []string `json:"shares,omitempty"` +} + +// PlanKept is a module a stopped walk sent and nothing walks again, and the machines it reached. +type PlanKept struct { + Module string `json:"module"` + Machines []string `json:"machines"` +} + // Waiting is whether the walk waits for its delivery's word. func (p Plan) Waiting() bool { return p.Delivery != nil && p.Delivery.Awaits != "" && p.Delivery.Go == nil @@ -265,7 +302,10 @@ type PlanGate struct { Sent map[string]SentDeclaration `json:"sent,omitempty"` // Passes counts the consecutive judgings that found it healthy, LastPass the newest; a judging that // does not resets them. - Passes int `json:"passes,omitempty"` + Passes int `json:"passes,omitempty"` + // Wants is how many passes the send needs: a core module's three, a leaf module's one (novox/hq ADR 0282 + // decision 4 as ADR 0304 decision 3 changed it). Zero in a gate judged before it was said: three. + Wants int `json:"wants,omitempty"` LastPass *time.Time `json:"last_pass,omitempty"` // Last is what the newest judging found wanting, while it still may pass. Last string `json:"last,omitempty"` @@ -305,6 +345,18 @@ type PlanGate struct { // Readings are the judging's readings with their times (novox/hq ADR 0282 decision 6): every reading that // counted a pass, and the first that did not after one that did. At most maxReadings, the newest kept. Readings []GateReading `json:"readings,omitempty"` + // Grants is the grants step taken before this gate's send (novox/hq issue 490): the bus's user list sent + // alone to the machine holding the bus, every build there kept, so what the send newly grants is on the + // bus before the gate judges it. Nil when the list did not change, or the gate's machine holds the bus. + Grants *GrantsStep `json:"grants,omitempty"` +} + +// GrantsStep is the bus's user list sent to the machine holding the bus ahead of a gated send (novox/hq +// issue 490): to which machine, when, and, when it could not be sent, why — the send went on without it. +type GrantsStep struct { + Node string `json:"node"` + At *time.Time `json:"at,omitempty"` + Failed string `json:"failed,omitempty"` } // GateReading is one reading of a first-node gate. @@ -385,39 +437,28 @@ func (p Plan) Open() bool { return p.State == PlanBuilding || p.State == PlanRol // with ErrPlanMoved otherwise; and only by a process that may act (ActsUnder), whose epoch it records. // On success p's revision and epoch are the ones written, so the caller may save it again. func (i *Inventory) SavePlan(ctx context.Context, p *Plan) error { + return i.SavePlansMoving(ctx, []*Plan{p}, nil) +} + +// MergeMove is a merge given to the plan that answers it now (novox/hq ADR 0276). +type MergeMove struct { + Repository, Commit, Plan string + Alone bool +} + +// SavePlansMoving writes plans, as SavePlan does each, and gives merges to the plans that answer them, all in one +// transaction: every write and move, or none (novox/hq ADR 0299 decision 7 — a stop that walks merges again never +// leaves a merge answered by two plans, nor a walk stopped whose merges were not moved). A plan written by +// somebody else since it was read refuses the whole act with ErrPlanMoved. Each plan is said (PlanSaved) once +// the transaction is committed, in order. +func (i *Inventory) SavePlansMoving(ctx context.Context, plans []*Plan, moves []MergeMove) error { epoch, err := i.actingEpoch(ctx) if err != nil { - return fmt.Errorf("the plan for %s %s is not written: %w", p.Repository, p.Commit, err) - } - tiers, err := json.Marshal(p.Tiers) - if err != nil { + if len(plans) > 0 { + return fmt.Errorf("the plan for %s %s is not written: %w", plans[0].Repository, plans[0].Commit, err) + } return err } - modules, err := json.Marshal(p.Modules) - if err != nil { - return err - } - var release, delivery, commits, times []byte - if p.Times != nil { - if times, err = json.Marshal(p.Times); err != nil { - return err - } - } - if len(p.Commits) > 0 { - if commits, err = json.Marshal(p.Commits); err != nil { - return err - } - } - if p.Release != nil { - if release, err = json.Marshal(p.Release); err != nil { - return err - } - } - if p.Delivery != nil { - if delivery, err = json.Marshal(p.Delivery); err != nil { - return err - } - } // **And how long the tier it left took** (novox/hq to-be 45 Phase 0): measured here, where the // plan moves, in the same transaction as the move, so no save can move a tier unmeasured or // measure one twice. @@ -426,9 +467,77 @@ func (i *Inventory) SavePlan(ctx context.Context, p *Plan) error { return err } defer func() { _ = tx.Rollback(ctx) }() + type written struct { + revision int64 + entered time.Time + } + done := make([]written, len(plans)) + for n, p := range plans { + revision, entered, err := writePlan(ctx, tx, p, epoch) + if err != nil { + return err + } + done[n] = written{revision, entered} + } + for _, m := range moves { + if _, err := tx.Exec(ctx, + `update batched_merge set plan_id = nullif($3, ''), alone = $4 where repository = lower($1) and commit_hash = $2`, + m.Repository, m.Commit, m.Plan, m.Alone); err != nil { + return err + } + } + if err := tx.Commit(ctx); err != nil { + return err + } + for n, p := range plans { + p.Revision, p.TierEntered = done[n].revision, done[n].entered + if epoch != nil { + p.Epoch = uint64(*epoch) + } else { + p.Epoch = 0 + } + if PlanSaved != nil { + PlanSaved(*p) + } + } + return nil +} + +// writePlan writes one plan within a transaction, by compare-and-set on its revision: its new revision, and when +// it entered its tier. +func writePlan(ctx context.Context, tx pgx.Tx, p *Plan, epoch *int64) (int64, time.Time, error) { + tiers, err := json.Marshal(p.Tiers) + if err != nil { + return 0, time.Time{}, err + } + modules, err := json.Marshal(p.Modules) + if err != nil { + return 0, time.Time{}, err + } + var release, delivery, commits, times []byte + if p.Times != nil { + if times, err = json.Marshal(p.Times); err != nil { + return 0, time.Time{}, err + } + } + if len(p.Commits) > 0 { + if commits, err = json.Marshal(p.Commits); err != nil { + return 0, time.Time{}, err + } + } + if p.Release != nil { + if release, err = json.Marshal(p.Release); err != nil { + return 0, time.Time{}, err + } + } + if p.Delivery != nil { + if delivery, err = json.Marshal(p.Delivery); err != nil { + return 0, time.Time{}, err + } + } entered, err := planTierLeft(ctx, tx, *p, time.Now()) if err != nil { - return err + return 0, time.Time{}, err } var revision int64 err = tx.QueryRow(ctx, @@ -449,24 +558,9 @@ func (i *Inventory) SavePlan(ctx context.Context, p *Plan) error { // The row is there and at another revision — moved since this was read, or there already // when this one is new: either way not this writer's to overwrite. (A plan saved before plans // had revisions is at zero, and its first save here is from a read at zero.) - return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved) + return 0, time.Time{}, fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved) } - if err != nil { - return err - } - if err := tx.Commit(ctx); err != nil { - return err - } - p.Revision, p.TierEntered = revision, entered - if epoch != nil { - p.Epoch = uint64(*epoch) - } else { - p.Epoch = 0 - } - if PlanSaved != nil { - PlanSaved(*p) - } - return nil + return revision, entered, err } // short is a commit as a person reads it. diff --git a/internal/inventory/secrets.go b/internal/inventory/secrets.go index 80f002b5..34ee9cb1 100644 --- a/internal/inventory/secrets.go +++ b/internal/inventory/secrets.go @@ -58,7 +58,9 @@ const ( // **Remade when either end's sealing key changes.** A node that rejoined generated a new key and // can no longer open what was sealed to the old one, so keeping the blob would deliver something // unreadable for ever. The new secret reaches both ends in the same push, which is the only -// moment they can be changed together. +// moment they can be changed together. A remade value is the mesh's, whatever the row held: its +// origin says made and it carries no stamp, so a value a provider's software issued is sealed to it +// again at the provider's next ask (novox/hq ADR 0305 §5) rather than read as still held. func (i *Inventory) SecretFor(ctx context.Context, name, consumer, consumerModule, provider, local string) ( Secret, error) { consumerKey, err := i.SealingKeyOf(ctx, consumer) @@ -121,7 +123,8 @@ func (i *Inventory) SecretFor(ctx context.Context, name, consumer, consumerModul for_consumer = excluded.for_consumer, for_provider = excluded.for_provider, consumer_key = excluded.consumer_key, provider_key = excluded.provider_key, created_at = now(), - operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key`, + operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key, + origin = 'made', generation = null`, name, consumerNode.ID, consumerModule, providerNode.ID, made.ForConsumer, made.ForProvider, made.ConsumerKey, made.ProviderKey, forOperator, operatorKey, local) if err != nil { @@ -202,7 +205,7 @@ func (i *Inventory) AcceptSecretForPair(ctx context.Context, name, consumer, con on conflict (name, local, consumer, consumer_module, provider) do update set for_consumer = excluded.for_consumer, for_provider = excluded.for_provider, consumer_key = excluded.consumer_key, provider_key = excluded.provider_key, - created_at = now(), origin = excluded.origin, + created_at = now(), origin = excluded.origin, generation = null, operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key`, name, consumerNode.ID, consumerModule, providerNode.ID, sealed.ForConsumer, sealed.ForProvider, sealed.ConsumerKey, sealed.ProviderKey, diff --git a/internal/inventory/sends.go b/internal/inventory/sends.go index 8a6efa36..edba4ac8 100644 --- a/internal/inventory/sends.go +++ b/internal/inventory/sends.go @@ -100,6 +100,23 @@ type Send struct { CounterRaisedTo int64 // Recorded is false for a refusal of a sequence the mesh has no send for: its sender is unknown. Recorded bool + // Answer is what the machine last reported of this send; filled only where a last send is read back + // (LastSend, LastSends), the zero answer elsewhere. + Answer SendAnswer +} + +// SendAnswer is what a machine reported of one send, read from its last report when that report is about the +// send's declaration (novox/hq issue 485): whether it applied it, failed part of it or refused it, and why. +// The zero answer is a machine that has not reported on this send — yet, or since a later report replaced it. +type SendAnswer struct { + // Outcome is applied, failed or refused, as the machine reported it; empty when it has not reported on it. + Outcome string + // Refused is the machine's own words when it refused the declaration whole. + Refused string + // Failed is how many of its resources failed, when some did. + Failed int + // At is when the machine reported it; nil when it has not. + At *time.Time } // unknownSender is the sender of a refused sequence the mesh has no record of sending. @@ -209,20 +226,56 @@ func scanSends(rows pgx.Rows) ([]Send, error) { return out, rows.Err() } -// LastSend is the last declaration a machine was sent, by its name; false when none was recorded. A refusal -// of a sequence the mesh has no send for is not a send, and is not it. +// LastSend is the last declaration a machine was sent, by its name, with what the machine answered of it; false +// when none was recorded. A refusal of a sequence the mesh has no send for is not a send, and is not it. func (i *Inventory) LastSend(ctx context.Context, name string) (Send, bool, error) { - rows, err := i.store.Pool().Query(ctx, `select `+sendColumns+` - from declaration_send s join node n on n.id = s.node - where n.name = $1 and s.recorded order by s.id desc limit 1`, name) + sends, err := i.lastSends(ctx, `n.name = $1`, name) if err != nil { return Send{}, false, err } - sends, err := scanSends(rows) - if err != nil || len(sends) == 0 { - return Send{}, false, err + s, found := sends[name] + return s, found, nil +} + +// LastSends is every machine's last send, by machine name, each with what the machine answered of it: what +// `status` says per machine (novox/hq issue 485). A machine never sent anything has no entry. +func (i *Inventory) LastSends(ctx context.Context) (map[string]Send, error) { + return i.lastSends(ctx, `true`) +} + +// lastSends reads the newest recorded send of each machine matching where, beside the machine's last report +// when that report is about the send (novox/hq issue 485). It names the send's declaration (its digest), and: +// - from an ordered node-engine, one that reports the epoch and sequence it acted on, those are the send's +// own. A periodic report about an identical earlier send names the same digest and may arrive after a later +// send the machine never received; only the sequence tells them apart. It also holds for a fast machine +// whose report lands before the send's row is stamped. +// - from an engine that reports no order, the report came after the send — both times the store's own — so +// a machine sent again a declaration it once applied does not read as having applied the new send. +func (i *Inventory) lastSends(ctx context.Context, where string, args ...any) (map[string]Send, error) { + rows, err := i.store.Pool().Query(ctx, `select distinct on (n.name) `+sendColumns+`, + coalesce(r.outcome, ''), coalesce(r.refused, ''), case when jsonb_typeof(r.failed) = 'array' then jsonb_array_length(r.failed) else 0 end, r.at + from declaration_send s join node n on n.id = s.node + left join node_report r on r.node = s.node and r.declared <> '' and r.declared = s.digest + and case when r.reported_sequence is not null + then r.reported_sequence = s.sequence and coalesce(r.reported_epoch, 0) = coalesce(s.epoch, 0) + else r.at >= s.sent_at end + where s.recorded and `+where+` + order by n.name, s.id desc`, args...) + if err != nil { + return nil, err } - return sends[0], true, nil + defer rows.Close() + out := map[string]Send{} + for rows.Next() { + var s Send + if err := rows.Scan(&s.Node, &s.NodeName, &s.Sequence, &s.Epoch, &s.Sender, &s.Generation, &s.Digest, + &s.Modules, &s.SentAt, &s.RefusedAt, &s.RefusedApplied, &s.CounterRaisedTo, &s.Recorded, + &s.Answer.Outcome, &s.Answer.Refused, &s.Answer.Failed, &s.Answer.At); err != nil { + return nil, err + } + out[s.NodeName] = s + } + return out, rows.Err() } // RefusedSend keeps a machine's refusal of the send of a sequence for the generation it came from, and diff --git a/internal/inventory/sends_test.go b/internal/inventory/sends_test.go index 96c98d15..37aaafd6 100644 --- a/internal/inventory/sends_test.go +++ b/internal/inventory/sends_test.go @@ -205,3 +205,144 @@ func TestWhetherAMachineReadsAGenerationIsItsLatestWord(t *testing.T) { } } } + +// **The last send says what the machine answered of it** (novox/hq issue 485): applied, refused with its +// words, or not answered yet — read from the machine's report only when that report is about the send's +// declaration and came after it, so a report about an earlier send is never this one's answer. And every +// machine's last send is read at once, for status, by name; a machine never sent anything has none. +func TestTheLastSendSaysWhatTheMachineAnswered(t *testing.T) { + inv, node := aNodeWithModules(t) + ctx := t.Context() + record, err := inv.NodeByName(ctx, node) + if err != nil { + t.Fatal(err) + } + if _, err := inv.AddNode(ctx, "laptop"); err != nil { + t.Fatal(err) + } + send := func(sequence int64, digest string) { + t.Helper() + if err := inv.RecordSentWith(ctx, record.ID, digest, nil, 57, 40, Send{Sequence: sequence, Epoch: 57, + Sender: "the controller's daemon (pid 7 on anchor)", Generation: 40, Digest: digest, + Modules: []string{"docker", "sudo"}}); err != nil { + t.Fatal(err) + } + } + report := func(d Doing) { + t.Helper() + if _, err := inv.RecordDoing(ctx, record.ID, d); err != nil { + t.Fatal(err) + } + } + answer := func() SendAnswer { + t.Helper() + last, found, err := inv.LastSend(ctx, node) + if err != nil || !found { + t.Fatalf("the last send is not read back: %v, %v", found, err) + } + return last.Answer + } + + send(11, "d11") + if a := answer(); a.Outcome != "" || a.At != nil { + t.Fatalf("a send not reported on reads as answered: %+v", a) + } + report(Doing{Outcome: OutcomeApplied, Declared: "d11", Applied: 3}) + if a := answer(); a.Outcome != OutcomeApplied || a.At == nil { + t.Fatalf("a send the machine reported applying reads %+v", a) + } + + send(12, "d12") + if a := answer(); a.Outcome != "" { + t.Fatalf("the report about the send before is taken for this one's answer: %+v", a) + } + report(Doing{Outcome: OutcomeRefused, Declared: "d12", Refused: "unknown field \"generation\"\nand more"}) + if a := answer(); a.Outcome != OutcomeRefused || a.Refused != "unknown field \"generation\"\nand more" { + t.Fatalf("a send the machine refused reads %+v", a) + } + + // Sent again a declaration it reported applying before the send: not answered until it reports again. + report(Doing{Outcome: OutcomeApplied, Declared: "d12", Applied: 3}) + send(13, "d12") + if a := answer(); a.Outcome != "" { + t.Fatalf("a report from before the send is taken for its answer: %+v", a) + } + + all, err := inv.LastSends(ctx) + if err != nil { + t.Fatal(err) + } + if len(all) != 1 || all[node].Sequence != 13 || all[node].Epoch != 57 || all[node].Generation != 40 || + !slices.Equal(all[node].Modules, []string{"docker", "sudo"}) { + t.Fatalf("every machine's last send reads %+v", all) + } + if _, found := all["laptop"]; found { + t.Error("a machine never sent anything has a last send") + } +} + +// **A report about another declaration, or another send of the same one, is not the last send's answer** +// (novox/hq issue 485, review of mesh-controller #234). A report naming an earlier send's digest that arrives +// after a later send is not the later send's answer. And from an ordered node-engine, a periodic report about +// an identical earlier send — the same digest, arriving after the later send — is not its answer either: only +// the report's epoch and sequence say which send it is about. +func TestAReportIsTheLastSendsAnswerOnlyWhenItIsAboutThatSend(t *testing.T) { + inv, node := aNodeWithModules(t) + ctx := t.Context() + record, err := inv.NodeByName(ctx, node) + if err != nil { + t.Fatal(err) + } + send := func(sequence int64, digest string) { + t.Helper() + if err := inv.RecordSentWith(ctx, record.ID, digest, nil, 57, 40, Send{Sequence: sequence, Epoch: 57, + Sender: "the controller's daemon (pid 7 on anchor)", Generation: 40, Digest: digest}); err != nil { + t.Fatal(err) + } + } + ordered := func(epoch, sequence int64, digest string) { + t.Helper() + if _, err := inv.RecordOrderedDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: digest, Applied: 1}, + ReportOrder{Epoch: epoch, Sequence: sequence}, func(ReportOrder) bool { return false }); err != nil { + t.Fatal(err) + } + } + answer := func() string { + t.Helper() + last, found, err := inv.LastSend(ctx, node) + if err != nil || !found { + t.Fatalf("the last send is not read back: %v, %v", found, err) + } + return last.Answer.Outcome + } + + // An engine that reports no order: the digest decides. + send(11, "d11") + send(12, "d12") + if _, err := inv.RecordDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: "d11"}); err != nil { + t.Fatal(err) + } + if a := answer(); a != "" { + t.Fatalf("a report about d11, after d12 was sent, reads as d12's answer: %q", a) + } + + // An ordered engine: the epoch and sequence decide, whatever the digest and the time. + send(20, "d20") + ordered(57, 20, "d20") + if a := answer(); a != OutcomeApplied { + t.Fatalf("an ordered report about sequence 20 is not its answer: %q", a) + } + send(21, "d20") + ordered(57, 20, "d20") + if a := answer(); a != "" { + t.Fatalf("a periodic report about sequence 20, after 21 was sent with the same digest, reads as 21's answer: %q", a) + } + ordered(56, 21, "d20") + if a := answer(); a != "" { + t.Fatalf("a report about sequence 21 of another epoch reads as this send's answer: %q", a) + } + ordered(57, 21, "d20") + if a := answer(); a != OutcomeApplied { + t.Fatalf("an ordered report about sequence 21 is not its answer: %q", a) + } +} diff --git a/internal/link/builds_nats_test.go b/internal/link/builds_nats_test.go index b3661388..6b405d42 100644 --- a/internal/link/builds_nats_test.go +++ b/internal/link/builds_nats_test.go @@ -123,15 +123,24 @@ func TestNatsABuildIsTakenAndItsOutcomeReachesEverybody(t *testing.T) { t.Fatalf("line %d came back as %s (%v)", want, msg.Data, err) } } - // And it is in the stream for a reader who comes later. - info, err := js.Context().StreamInfo(broker.EventsStream, &nats.StreamInfoRequest{SubjectsFilter: BuildLog("b-1")}) - if err != nil { - t.Fatal(err) + // And it is in the stream for a reader who comes later. **Waited for, with a bound, rather + // than read once** (novox/hq issue 507): a line is said with a plain publish, so the server + // hands it to a live subscriber and stores it in the stream independently, and under load the + // subscriber above had both lines while the stream still held one, or none. + var held map[string]uint64 + for until := time.Now().Add(5 * time.Second); ; { + info, err := js.Context().StreamInfo(broker.EventsStream, &nats.StreamInfoRequest{SubjectsFilter: BuildLog("b-1")}) + if err != nil { + t.Fatal(err) + } + held = info.State.Subjects + if held[BuildLog("b-1")] >= 2 || time.Now().After(until) { + break + } + time.Sleep(10 * time.Millisecond) } - if info.State.Subjects[BuildLog("b-1")] != 2 { - t.Fatalf("the stream holds %v under the build's subject, want 2", info.State.Subjects) - } - if err == nil { + if held[BuildLog("b-1")] != 2 { + t.Fatalf("the stream holds %v under the build's subject, want 2", held) } if result.ID != "b-1" || result.Commit != "abc1234" { t.Fatalf("the asker got %+v", result) diff --git a/internal/link/retirement.go b/internal/link/retirement.go index 2f580c1c..c0bd8330 100644 --- a/internal/link/retirement.go +++ b/internal/link/retirement.go @@ -173,6 +173,25 @@ func ModuleToolOn(module, tool, node string) string { return ToolSubject(module, // AskModuleToolOn asks one machine's instance of a module one of its tools and reads its answer. func AskModuleToolOn(ctx context.Context, conn *nats.Conn, module, tool, node string, args any, + timeout time.Duration) (Answer, error) { + return askModuleToolAt(ctx, conn, ModuleToolOn(module, tool, node), module, tool, node, args, timeout) +} + +// AskModuleToolOnAs is AskModuleToolOn in the caller's own name: on the subject that names its caller (novox/hq +// issue 365), which the bus lets only that user publish, so the tool knows who asks from the subject alone. The +// one way a tool can answer the controller and nobody else (novox/hq ADR 0305 §2). +func AskModuleToolOnAs(ctx context.Context, conn *nats.Conn, caller, module, tool, node string, args any, + timeout time.Duration) (Answer, error) { + subject := broker.CalledSubject(ModuleToolOn(module, tool, node), caller) + if subject == "" { + return Answer{}, fmt.Errorf("%q cannot ask %s.%s on %s in its own name: it is not a bus user's name", caller, + module, tool, node) + } + return askModuleToolAt(ctx, conn, subject, module, tool, node, args, timeout) +} + +// askModuleToolAt asks one machine's instance of a module one of its tools on subject and reads its answer. +func askModuleToolAt(ctx context.Context, conn *nats.Conn, subject, module, tool, node string, args any, timeout time.Duration) (Answer, error) { body, err := json.Marshal(args) if err != nil { @@ -180,7 +199,6 @@ func AskModuleToolOn(ctx context.Context, conn *nats.Conn, module, tool, node st } asking, cancel := context.WithTimeout(ctx, timeout) defer cancel() - subject := ModuleToolOn(module, tool, node) refused, stop := refusalsOf(conn, subject) defer stop() type replied struct { diff --git a/merge-check.sh b/merge-check.sh index 296fe271..e1e85687 100755 --- a/merge-check.sh +++ b/merge-check.sh @@ -18,6 +18,34 @@ set -eu export GOFLAGS=-mod=vendor GOPROXY=off +# judge_store_tests says each listed test of one package as `go test -v` reported it, and counts in $missing each +# that did not pass: $1 the package, $2 go test's exit status, $3 the file holding its output, then the names. +# A result is found anywhere in a line, not only at its start: when a test's last output leaves a line open, +# `go test -v` writes `--- PASS: (…)` on that same line (novox/hq issue 515). A package whose go test +# exits non-zero fails even when each listed test said PASS — a TestMain or a leak checker can fail after them. +judge_store_tests() { + pkg=$1 status=$2 out=$3 + shift 3 + for name in "$@"; do + line=$(grep -oE -- "--- (PASS|FAIL|SKIP): $name \([^)]*\)" "$out" | tail -n 1 || true) + echo " $pkg ${line:-"--- NOT RUN: $name"}" + case "$line" in "--- PASS: "*) ;; *) missing=$((missing + 1)) ;; esac + done + if [ "$status" -ne 0 ]; then + echo " $pkg FAIL: go test exited $status" + missing=$((missing + 1)) + fi +} +# `sh merge-check.sh judge-store-tests …` runs only the judging, to try it on a +# recorded `go test -v` output. +if [ "${1:-}" = judge-store-tests ]; then + shift + missing=0 + judge_store_tests "$@" + [ "$missing" -eq 0 ] + exit +fi + unformatted=$(gofmt -l cmd internal examples) if [ -n "$unformatted" ]; then echo "not gofmt'd:" @@ -32,3 +60,57 @@ else echo "NOT RACE-CHECKED: the toolchain holds no C compiler; the suite runs without the race detector" CGO_ENABLED=0 go test -count=1 -timeout 30m ./... fi + +# **The store tests listed ran** (novox/hq issue 459): a test that needs the store skips without one, and a skip +# passes unseen. With the store this check is given, every test named in testdata/store-tests must have run against +# it and passed, each said by name; without one, that is said. A pull request adds its own store tests to the list. + + +listed=$(grep -v '^[[:space:]]*\(#\|$\)' testdata/store-tests || true) +# Each line is exactly ` `: a line of one field ran nothing and said nothing (novox/hq issue 515). +malformed=$(printf '%s\n' "$listed" | awk 'NF != 2 && NF != 0' || true) +if [ -n "$malformed" ]; then + echo "testdata/store-tests: a line is not :" + echo "$malformed" + exit 1 +fi +# **A change does not shorten the list it is judged by** (novox/hq issue 515): the list is in the change's own tree, +# so an entry main lists stays listed while its test function exists; removing the function removes the entry. +if ! git rev-parse --verify --quiet origin/main^{commit} >/dev/null; then + echo "origin/main cannot be read, and its testdata/store-tests says which store tests must stay listed" + exit 1 +fi +dropped=0 +for entry in $(git show origin/main:testdata/store-tests 2>/dev/null | grep -v '^[[:space:]]*\(#\|$\)' | + awk 'NF == 2 {print $1 "|" $2}'); do + pkg=${entry%%|*} name=${entry#*|} + printf '%s\n' "$listed" | awk -v p="$pkg" -v n="$name" '$1 == p && $2 == n {f = 1} END {exit !f}' && continue + if cat "$pkg"/*_test.go 2>/dev/null | grep -q "^func $name("; then + echo "testdata/store-tests: $pkg $name is listed on main and its test still exists, but the change drops it" + dropped=$((dropped + 1)) + fi +done +[ "$dropped" -eq 0 ] || exit 1 + +if [ -z "$listed" ]; then + echo "store tests: none listed in testdata/store-tests" +elif [ -n "${MESH_TEST_POSTGRES:-}" ]; then + echo "store tests, each run against the store:" + missing=0 + out=$(mktemp) + trap 'rm -f "$out"' EXIT + for pkg in $(printf '%s\n' "$listed" | awk '{print $1}' | sort -u); do + names=$(printf '%s\n' "$listed" | awk -v p="$pkg" '$1 == p {print $2}') + pattern="^($(printf '%s\n' "$names" | paste -sd '|' -))\$" + status=0 + CGO_ENABLED=0 go test -count=1 -v -run "$pattern" "$pkg" >"$out" 2>&1 || status=$? + # shellcheck disable=SC2086 # the names are one word each, split on purpose + judge_store_tests "$pkg" "$status" "$out" $names + done + if [ "$missing" -ne 0 ]; then + echo "$missing store test(s) listed in testdata/store-tests did not run and pass against the store" + exit 1 + fi +else + echo "NOT STORE-CHECKED: no MESH_TEST_POSTGRES, so the store tests listed in testdata/store-tests skipped" +fi diff --git a/mesh-controller b/mesh-controller deleted file mode 100755 index b8fa8cd1..00000000 Binary files a/mesh-controller and /dev/null differ diff --git a/module.json b/module.json index ba7c0ad5..f3d79f41 100644 --- a/module.json +++ b/module.json @@ -77,6 +77,7 @@ "drill", "warranted", "secret-ask", + "issued", "hand-acts", "durations", "conditions", diff --git a/postgres-provisioner b/postgres-provisioner deleted file mode 100755 index 99c7af14..00000000 Binary files a/postgres-provisioner and /dev/null differ diff --git a/redis-provisioner b/redis-provisioner deleted file mode 100755 index 96b9ea05..00000000 Binary files a/redis-provisioner and /dev/null differ diff --git a/testdata/beside/CAPTURED b/testdata/beside/CAPTURED index f922ff40..29b367c5 100644 --- a/testdata/beside/CAPTURED +++ b/testdata/beside/CAPTURED @@ -25,3 +25,11 @@ and write the commits here. The SDK is captured at the commit go.mod pins for gi rm -rf testdata/beside/mesh-sdk && mkdir -p testdata/beside/mesh-sdk git -C ../mesh-sdk archive conformance/events | tar -x -C testdata/beside/mesh-sdk + +And from mesh-host at the same commit as its line above, the node-engine's placement guard, which +internal/catalogue's engine_guard_test.go holds the controller's resolved-path rules to (novox/hq issue 496), +kept as .captured so no Go tool reads it as this repository's code: + + mkdir -p testdata/beside/mesh-host/internal/apply + git -C ../mesh-host show :internal/apply/placement_guard.go \ + > testdata/beside/mesh-host/internal/apply/placement_guard.go.captured diff --git a/testdata/beside/mesh-host/internal/apply/placement_guard.go.captured b/testdata/beside/mesh-host/internal/apply/placement_guard.go.captured new file mode 100644 index 00000000..49c1d83e --- /dev/null +++ b/testdata/beside/mesh-host/internal/apply/placement_guard.go.captured @@ -0,0 +1,549 @@ +package apply + +// Where the node-engine places nothing and mounts nothing, whoever asks (novox/hq issue 339). +// +// A directory names its path, and the controller resolves part of that path from what was set for the +// module: its `places` setting moves a directory anywhere, with an owner it names, and its `accesses` setting +// says which of the machine's paths are mounted into its container. The engine runs as root, so a path it +// accepts blindly is a path anyone who could change those settings hands to any account: a directory at /etc +// owned by a caller's account gives it /etc, and an access at / mounts the machine's root into a container. +// The controller refuses both where a setting is made; the engine refuses them again where it applies, +// because a guard in one place is a guard one change away from gone. Whatever the declaration says: +// +// 1. **No directory, access or mount source is one of the machine's own roots, or holds one**: /, /etc, +// /usr, /var, /var/lib, /home, /run and the rest of protectedRoots. Modules place directories BELOW /etc +// or /var/lib, never the root itself; owning one is owning everything in it. +// 2. **Nothing is placed in /proc, /sys, /dev, /boot, /root, /var/spool, /opt or the container runtimes' data +// (/var/lib/docker, /var/lib/containers, /var/lib/containerd, and where the runtimes' configuration moves +// them: runtimeDataRoots), nor in the node-engine's +// own trees** (its state, its identity, its installed builds) but by its own module; nothing is mounted from +// those but /proc, /sys and /dev. A mount of a kernel file, a device or the clock is the plumbing +// systemPath names. +// 2a. **An account's .ssh is never an access or a mount**, and is a directory only below its account's home, +// as that account's (rule 4). +// 3. **A directory below /etc, /usr or /run is root's.** The machine's configuration and programs are read +// as root's word; a directory there owned by another account is that account writing root's word. An +// access is never there at all: the operator's data is not the machine's configuration. +// 4. **Below a person's or an agent's home, a directory is that account's.** Root's or another account's +// directory there is one the account does not control in a tree whose every parent it does. A home +// itself may be a module's directory (a backup repository kept as an account's home is one), and as +// every directory the mesh did not make, it is used as found: never chowned or chmodded (applyDirectory). +// 5. **A mount source that is a refused directory or a refused access is refused with it**: the container +// would otherwise bind the very path the engine would not place, and the runtime creates a missing one +// as root. +// +// Each is a failed resource with its reason in the node's report; nothing is touched. Paths are judged as +// declared and again with every link in them resolved, so a link at /srv/x pointing at /etc places nothing. + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "sync" + "time" + + "github.com/novox/mesh-host/internal/declaration" +) + +// protectedRoots are paths no directory, access or mount source may be, nor hold. +var protectedRoots = []string{"/", "/bin", "/boot", "/dev", "/etc", "/home", "/lib", "/lib32", "/lib64", + "/media", "/mnt", "/opt", "/proc", "/root", "/run", "/sbin", "/srv", "/sys", "/tmp", "/usr", "/usr/bin", + "/usr/lib", "/usr/lib64", "/usr/local", "/usr/local/bin", "/usr/local/lib", "/usr/local/sbin", "/usr/sbin", + "/usr/share", "/var", "/var/cache", "/var/lib", "/var/lib/mesh", "/var/log", "/var/run", "/var/tmp", + "/var/spool"} + +// forbiddenBelow are trees nothing is placed in or mounted from: the kernel's, the boot loader's and root's +// home. engineTrees are the node-engine's own, which only its own module places in. +var ( + forbiddenBelow = []string{"/proc", "/sys", "/dev", "/boot", "/root", "/var/spool", "/opt", "/var/lib/docker", + "/var/lib/containers", "/var/lib/containerd"} + // forbiddenBelowMount are the trees no container mounts from: a mount of the kernel's files and devices is + // the plumbing a container may need (systemPath); root's home and the boot loader's are no plumbing. + forbiddenBelowMount = []string{"/boot", "/root", "/var/spool", "/opt", "/var/lib/docker", "/var/lib/containers", + "/var/lib/containerd"} + engineTrees = []string{"/var/lib/mesh-host", "/usr/lib/nox-mesh-host"} + // rootsOnly are trees a directory below is root's, and an access is never in. + rootsOnly = []string{"/etc", "/usr", "/run", "/var/run"} +) + +// runtimeFiles are where the container runtimes say where they keep their data: dockerd's daemon.json, its +// unit and the unit's drop-ins (an ExecStart with --data-root, or the older -g/--graph, continued over lines, +// quoted, through Environment= or EnvironmentFile=, or a --config-file naming another daemon.json), and podman's +// storage.conf (graphroot, a basic or a literal string). The running runtimes are asked first. containerd keeps its own under /var/lib/containerd, which forbiddenBelow names; a +// containerd configured elsewhere, and podman's rootless stores under each account's home, are not read: the +// first is no runtime this mesh runs, and the second is below a home, which rule 4 already keeps for its +// account. A variable so a test names its own. +type runtimeFiles struct { + daemonJSON string + units []string + dropInDirs []string + storageConf string +} + +var runtimeConfigs = runtimeFiles{ + daemonJSON: "/etc/docker/daemon.json", + units: []string{"/etc/systemd/system/docker.service", "/usr/lib/systemd/system/docker.service", "/lib/systemd/system/docker.service"}, + dropInDirs: []string{"/etc/systemd/system/docker.service.d", "/run/systemd/system/docker.service.d", "/usr/lib/systemd/system/docker.service.d"}, + storageConf: "/etc/containers/storage.conf", +} + +var ( + dataRootFlag = regexp.MustCompile(`(?:--data-root|--graph|-g)(?:=|\s+)(\S+)`) + configFlag = regexp.MustCompile(`--config-file(?:=|\s+)(\S+)`) + graphRoot = regexp.MustCompile(`(?m)^\s*graphroot\s*=\s*(?:"([^"]+)"|'([^']+)')`) + envVar = regexp.MustCompile(`\$\{([A-Za-z_][A-Za-z0-9_]*)\}|\$([A-Za-z_][A-Za-z0-9_]*)`) +) + +// runtimeRoots is where the container runtimes keep their data, as the last apply read it (readRuntimeRoots); +// nil until an apply has read it, when the guard reads the files itself. +var ( + runtimeRootsMu sync.Mutex + runtimeRoots []string +) + +// AskRuntimes is how the engine asks the running container runtimes where they keep their data: set by the engine +// to run the commands, nil in a test, which then reads the files alone. Its own, never the apply's runner, whose +// commands a test reads back as what the apply did. +var AskRuntimes Runner + +// refreshRuntimeRoots reads where the runtimes keep their data once, at the start of an apply. +func refreshRuntimeRoots(ctx context.Context) { + roots := readRuntimeRoots(ctx, AskRuntimes) + runtimeRootsMu.Lock() + runtimeRoots = roots + runtimeRootsMu.Unlock() +} + +// runtimeDataRoots is every place the container runtimes keep their data, beyond the default trees forbiddenBelow +// names: every container's filesystem is there. +func runtimeDataRoots() []string { + runtimeRootsMu.Lock() + roots := runtimeRoots + runtimeRootsMu.Unlock() + if roots != nil { + return roots + } + return readRuntimeRoots(context.Background(), nil) +} + +// readRuntimeRoots asks the running runtimes where they keep their data, when run is given (`docker info`, +// `podman info`), and reads their configuration besides: an answer from a runtime that is running is what it +// really does, and the files say what it will do when it starts again. Both count. A runtime that is not running +// or not installed answers nothing, which is no error. +func readRuntimeRoots(ctx context.Context, run Runner) []string { + seen := map[string]bool{} + var out []string + add := func(p string) { + p = strings.Trim(strings.TrimSpace(p), `"'`) + if !filepath.IsAbs(p) { + return + } + p = filepath.Clean(p) + if !seen[p] { + seen[p] = true + out = append(out, p) + } + } + if run != nil { + // Each runtime has its own ten seconds: one that hangs costs the other nothing. + for _, q := range [][]string{{"docker", "info", "--format", "{{.DockerRootDir}}"}, + {"podman", "info", "--format", "{{.Store.GraphRoot}}"}} { + ask, cancel := context.WithTimeout(ctx, 10*time.Second) + if root, err := run(ask, q[0], q[1:]...); err == nil { + add(root) + } + cancel() + } + } + daemonJSON := func(path string) { + raw, err := os.ReadFile(path) + if err != nil { + return + } + var c struct { + DataRoot string `json:"data-root"` + Graph string `json:"graph"` + } + if json.Unmarshal(raw, &c) == nil { + add(c.DataRoot) + add(c.Graph) + } + } + daemonJSON(runtimeConfigs.daemonJSON) + units := append([]string(nil), runtimeConfigs.units...) + for _, dir := range runtimeConfigs.dropInDirs { + matches, _ := filepath.Glob(filepath.Join(dir, "*.conf")) + units = append(units, matches...) + } + // The unit and its drop-ins are one unit to systemd: a variable set in one is seen by an ExecStart in another. + env := map[string]string{} + var execs []string + for _, u := range units { + raw, err := os.ReadFile(u) + if err != nil { + continue + } + e, x := unitLines(string(raw)) + for k, v := range e { + env[k] = v + } + execs = append(execs, x...) + } + for _, line := range execs { + line = envVar.ReplaceAllStringFunc(line, func(ref string) string { + m := envVar.FindStringSubmatch(ref) + if v, ok := env[m[1]+m[2]]; ok { + return v + } + return ref + }) + for _, m := range dataRootFlag.FindAllStringSubmatch(line, -1) { + add(m[1]) + } + for _, m := range configFlag.FindAllStringSubmatch(line, -1) { + daemonJSON(strings.Trim(m[1], `"'`)) + } + } + if raw, err := os.ReadFile(runtimeConfigs.storageConf); err == nil { + for _, m := range graphRoot.FindAllStringSubmatch(string(raw), -1) { + add(m[1] + m[2]) + } + } + return out +} + +// unitLines reads a unit file as systemd does for what matters here: a line ending in a backslash continues on the +// next, Environment= sets variables (quoted or not, several to a line), EnvironmentFile= (a leading - says it may +// be missing) reads KEY=value lines, and every ExecStart line is returned whole. +func unitLines(text string) (map[string]string, []string) { + var joined []string + var cur strings.Builder + for _, line := range strings.Split(text, "\n") { + trimmed := strings.TrimRight(line, " \t") + if strings.HasSuffix(trimmed, "\\") { + cur.WriteString(strings.TrimSuffix(trimmed, "\\") + " ") + continue + } + cur.WriteString(line) + joined = append(joined, cur.String()) + cur.Reset() + } + if cur.Len() > 0 { + joined = append(joined, cur.String()) + } + env := map[string]string{} + setPairs := func(s string) { + for _, f := range splitQuoted(s) { + if k, v, ok := strings.Cut(f, "="); ok { + env[strings.TrimSpace(k)] = strings.Trim(strings.TrimSpace(v), `"'`) + } + } + } + var execs []string + for _, line := range joined { + l := strings.TrimSpace(line) + switch { + case strings.HasPrefix(l, "Environment="): + setPairs(strings.TrimPrefix(l, "Environment=")) + case strings.HasPrefix(l, "EnvironmentFile="): + path := strings.TrimPrefix(strings.TrimSpace(strings.TrimPrefix(l, "EnvironmentFile=")), "-") + if raw, err := os.ReadFile(path); err == nil { + for _, kv := range strings.Split(string(raw), "\n") { + kv = strings.TrimSpace(kv) + if kv == "" || strings.HasPrefix(kv, "#") { + continue + } + setPairs(kv) + } + } + case strings.HasPrefix(l, "ExecStart"): + execs = append(execs, l) + } + } + return env, execs +} + +// splitQuoted splits on blanks outside double or single quotes, keeping the quotes' contents whole. +func splitQuoted(s string) []string { + var out []string + var cur strings.Builder + var quote rune + for _, r := range s { + switch { + case quote != 0 && r == quote: + quote = 0 + case quote == 0 && (r == '"' || r == '\''): + quote = r + case quote == 0 && (r == ' ' || r == '\t'): + if cur.Len() > 0 { + out = append(out, cur.String()) + cur.Reset() + } + default: + cur.WriteRune(r) + } + } + if cur.Len() > 0 { + out = append(out, cur.String()) + } + return out +} + +// engineModule is the module whose resources may place in the engine's own trees. +const engineModule = "mesh-host" + +// passwdFile is the user database homes are read from. A variable so a test names its own. +var passwdFile = "/etc/passwd" + +// PlacementRefusedError is a resource the engine will not place, or mount, where it says. +type PlacementRefusedError struct { + Path, Why string +} + +func (e *PlacementRefusedError) Error() string { + return fmt.Sprintf("%s is not placed: %s (novox/hq issue 339); nothing was touched", e.Path, e.Why) +} + +// homeAccount is a person's or an agent's account and its home. +type homeAccount struct { + Name string + UID int +} + +// accountsOfHomes is each person's or agent's home and the account it belongs to, from the user database: an +// account with a uid of 1000 or more, or a home under /home. A variable so a test names its own. +var accountsOfHomes = func() map[string]homeAccount { + f, err := os.Open(passwdFile) + if err != nil { + return nil + } + defer f.Close() + out := map[string]homeAccount{} + sc := bufio.NewScanner(f) + for sc.Scan() { + fields := strings.Split(sc.Text(), ":") + if len(fields) < 6 { + continue + } + uid, err := strconv.Atoi(fields[2]) + if err != nil { + continue + } + home := filepath.Clean(fields[5]) + if home == "/" || home == "." || home == "" || home == "/nonexistent" { + continue + } + if (uid >= 1000 && uid != 65534) || strings.HasPrefix(home, "/home/") { + out[home] = homeAccount{Name: fields[0], UID: uid} + } + } + return out +} + +// below says whether path is strictly below dir. +func below(path, dir string) bool { + if dir == "/" { + return path != "/" + } + return strings.HasPrefix(path, dir+"/") +} + +// atOrBelow says whether path is dir or below it. +func atOrBelow(path, dir string) bool { return path == dir || below(path, dir) } + +// resolved is a path with every link in it followed, as far as the path exists, and the rest as declared. +func resolved(path string) string { + rest := "" + for p := path; ; p = filepath.Dir(p) { + if real, err := filepath.EvalSymlinks(p); err == nil { + return filepath.Clean(filepath.Join(real, rest)) + } + if filepath.Dir(p) == p { + return path + } + rest = filepath.Join(filepath.Base(p), rest) + } +} + +// ownedByAccount says whether a declared owner is that account: by name, or by its uid ("1001", "1001:1001"). +func ownedByAccount(owner string, a homeAccount) bool { + if owner == a.Name { + return true + } + user, _, _ := strings.Cut(owner, ":") + if uid, err := strconv.Atoi(user); err == nil { + return uid == a.UID + } + return false +} + +// rootOwner says whether a declared owner is root: none, "root", or uid 0. +func rootOwner(owner string) bool { + if owner == "" || owner == "root" { + return true + } + user, _, _ := strings.Cut(owner, ":") + return user == "0" +} + +// what a guarded path is, for the words of a refusal. +type placing int + +const ( + placingDirectory placing = iota + placingAccess + placingMount +) + +// refusePath says why a path is not placed or mounted; nil when it may be. module is the resource's module, +// owner a directory's declared owner. +func refusePath(path string, kind placing, module, owner string) error { + clean := filepath.Clean(path) + if !filepath.IsAbs(clean) { + return nil // the declaration refuses a relative path already; a named volume is not a path + } + for _, p := range []string{clean, resolved(clean)} { + if err := refuseOne(p, kind, module, owner); err != nil { + if p != clean { + err.Why = fmt.Sprintf("through a link, it is %s, and %s", p, err.Why) + err.Path = clean + } + return err + } + } + return nil +} + +func refuseOne(path string, kind placing, module, owner string) *PlacementRefusedError { + for _, root := range protectedRoots { + if path == root || below(root, path) { + return &PlacementRefusedError{Path: path, Why: root + " is one of the machine's own directories, " + + "and owning or mounting it would be owning or mounting everything in it"} + } + } + trees := append([]string(nil), forbiddenBelow...) + if kind == placingMount { + // A mount is the machine's plumbing as often as a module's data — the clock, a kernel file, /dev/null + // (systemPath) — and what a setting can mount at all is a directory or an access, refused above it. + trees = append([]string(nil), forbiddenBelowMount...) + } + // The container runtimes' data, wherever the machine keeps it: every container's filesystem is there. + trees = append(trees, runtimeDataRoots()...) + for _, tree := range trees { + if atOrBelow(path, tree) { + return &PlacementRefusedError{Path: path, Why: "nothing is placed in or mounted from " + tree} + } + } + if module != engineModule { + for _, tree := range engineTrees { + if atOrBelow(path, tree) { + return &PlacementRefusedError{Path: path, Why: tree + " is the node-engine's own, placed in by " + + "its own module alone"} + } + } + } + for _, tree := range rootsOnly { + if !below(path, tree) { + continue + } + switch { + case kind == placingAccess: + return &PlacementRefusedError{Path: path, Why: "an access is the operator's data, and " + tree + + " is the machine's own"} + case kind == placingDirectory && !rootOwner(owner): + return &PlacementRefusedError{Path: path, Why: fmt.Sprintf("a directory below %s is root's, and this "+ + "one is declared %s's", tree, owner)} + } + } + ssh := false + for _, part := range strings.Split(path, "/") { + ssh = ssh || part == ".ssh" + } + if ssh && kind != placingDirectory { + return &PlacementRefusedError{Path: path, Why: "it is an account's .ssh, which holds its keys and who may " + + "log in as it, and is never an access or a mount"} + } + if kind != placingDirectory { + return nil + } + homes := accountsOfHomes() + var deepest string + for home := range homes { + if below(path, home) && len(home) > len(deepest) { + deepest = home + } + } + if ssh && deepest == "" { + return &PlacementRefusedError{Path: path, Why: "a .ssh directory is placed only below its account's home, " + + "as that account's"} + } + if deepest != "" { + if a := homes[deepest]; !ownedByAccount(owner, a) { + if owner == "" { + owner = "root" + } + return &PlacementRefusedError{Path: path, Why: fmt.Sprintf("it is below %s's home and declared %s's; "+ + "below a home only that account's directories are placed", a.Name, owner)} + } + } + return nil +} + +// moduleOfID is the module a resource id names, or "". +func moduleOfID(id string) string { + module, _ := moduleOf(id) + return module +} + +// refusedPlaces judges every directory and access of a declaration before anything is applied, and answers +// each refusal by path: what is refused is refused again as a container's mount source. +func refusedPlaces(resources []declaration.Resource) map[string]error { + out := map[string]error{} + for _, r := range resources { + var err error + switch res := r.(type) { + case *declaration.Directory: + err = refusePath(res.Path, placingDirectory, moduleOfID(res.ID), res.Owner) + case *declaration.Access: + err = refusePath(res.Path, placingAccess, moduleOfID(res.ID), "") + default: + continue + } + if err != nil { + out[filepath.Clean(r.Target())] = err + } + } + return out +} + +// refuseMounts says why a container's mounts are not made; nil when they may be. +func refuseMounts(c *declaration.Container, refused map[string]error) error { + for _, v := range c.Volumes { + src := mountSource(v) + if !strings.HasPrefix(src, "/") { + continue // a named volume, which the runtime keeps in its own tree + } + src = filepath.Clean(src) + for path, why := range refused { + if atOrBelow(src, path) { + var refusal *PlacementRefusedError + if errors.As(why, &refusal) { + return &PlacementRefusedError{Path: src, Why: "it is mounted from " + path + + ", which is refused: " + refusal.Why} + } + return why + } + } + if err := refusePath(src, placingMount, moduleOfID(c.ID), ""); err != nil { + return err + } + } + return nil +} diff --git a/testdata/store-tests b/testdata/store-tests new file mode 100644 index 00000000..d604f6d9 --- /dev/null +++ b/testdata/store-tests @@ -0,0 +1,14 @@ +# The tests that need the throwaway store and must have run against it (novox/hq issue 459): merge-check.sh runs +# each one named here, by package, and fails unless every one passed — a test that needs the store skips without +# one, and a skip passes unseen. One per line: . Lines starting with # are said nothing of. +./cmd/mesh-controller TestAStopWithoutAMergeWalksTheOthersAgain +./cmd/mesh-controller TestAStopWithoutAMergeEndsTheMergesThatContainIt +./cmd/mesh-controller TestAStopWalksTheOthersAgainInTheOpenBatch +./cmd/mesh-controller TestAStopNamesWhatStaysOnTheMachinesAndWhatStillCarriesIt +./cmd/mesh-controller TestAStopWithoutAMergeLeavesItOutOfABatch +./cmd/mesh-controller TestSavingPlansAndMovingMergesIsOneAct +./internal/inventory TestTheLastSendSaysWhatTheMachineAnswered +./internal/inventory TestAReportIsTheLastSendsAnswerOnlyWhenItIsAboutThatSend +./cmd/mesh-controller TestNodeShowAndStatusReadTheLastSendFromTheStore +./internal/inventory TestAnIssuedValueIsSealedToEveryHolderOnceAndNotAgain +./internal/inventory TestAnIssuedValueIsSealedAgainAfterAKeyChangeOrAnAcceptance