Compare commits
35
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
37e4219e28 | ||
|
|
97a2d841c7 | ||
|
|
fcdecb4361 | ||
|
|
197e60c97b | ||
|
|
e75c376a7d | ||
|
|
9932adfa00 | ||
|
|
4b2eac702a | ||
|
|
c88474a127 | ||
|
|
3d3ca6e500 | ||
|
|
fbcf44702c | ||
|
|
2941ef3695 | ||
|
|
9f2e16ff70 | ||
|
|
c0bee5d042 | ||
|
|
221fe5a60d | ||
|
|
9aa0332694 | ||
|
|
60d390aded | ||
|
|
c5fe7d4795 | ||
|
|
ae071e2b59 | ||
|
|
233956ef09 | ||
|
|
2f40ae5f76 | ||
|
|
dc36536e19 | ||
|
|
71362c196d | ||
|
|
617881c309 | ||
|
|
661918f4e4 | ||
|
|
def3ccefe3 | ||
|
|
912d76df25 | ||
|
|
3048dccd0e | ||
|
|
f0af09cfc7 | ||
|
|
85b85a3fa9 | ||
|
|
02b542a059 | ||
|
|
7530c2aeac | ||
|
|
f45e0a1de5 | ||
|
|
276f6755ab | ||
|
|
1ce4ab8c2a | ||
|
|
d43fbc202c |
+68
-14
@@ -26,6 +26,7 @@ import (
|
||||
"os/signal"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -272,7 +273,12 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
// role's own events under the build's id (novox/hq ADR 0157) — so whoever asked, and anybody
|
||||
// watching, reads the same lines this container's log holds, live, and after the fact from the
|
||||
// stream. Said first, before anything runs, so a build that hangs is one that visibly started.
|
||||
// One line at a time: a check's builds for its merge say their lines while its tests say theirs (novox/hq
|
||||
// issue 716).
|
||||
var saying sync.Mutex
|
||||
say := func(step, message string) {
|
||||
saying.Lock()
|
||||
defer saying.Unlock()
|
||||
fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message)
|
||||
work.Say(step, message)
|
||||
if mine != nil && step != "run" && step != "output" {
|
||||
@@ -313,25 +319,36 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
registry = r.Address
|
||||
}
|
||||
spec := checkSpecOf(request)
|
||||
// The judge cache the build agent declares as its own data, never in the workspace: only judge builds and
|
||||
// mesh-lab's replays mount it, never a check's containers, so no pull request's script can write what a judge
|
||||
// is built with or kept as, or what the replays run (novox/hq ADR 0358 §6, issue 709).
|
||||
spec.JudgeCache = judgeCache()
|
||||
// What the merge would move, built by the check for the walk to take (novox/hq ADR 0350 §1): into the
|
||||
// scratch namespace. **Never with the package registry's credential**: this is code nobody has approved
|
||||
// yet, and a recipe could read the credential it is handed (issue 462). A module that resolves packages
|
||||
// at build time has no reuse key anyway, so nothing is lost by refusing it here.
|
||||
if isRegistry && len(spec.Builds) > 0 {
|
||||
spec.BuildWith = &builder.CheckBuilding{Publish: publisher, Scratch: registryHere, Held: request.Held,
|
||||
Seats: request.Seats}
|
||||
Running: request.Check.Running, Seats: request.Seats, Named: namedOf(request.Check.Named)}
|
||||
}
|
||||
var v builder.CheckVerdict
|
||||
v, err = builder.Check(building, builder.Command, spec, workspace, registry, forgeFrom(), say)
|
||||
if err == nil {
|
||||
result.Check = &link.CheckOutcome{Verdict: v.Verdict, Summary: v.Summary, Report: v.Report,
|
||||
Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo),
|
||||
Head: v.Head, Trunk: v.Trunk, Beside: v.Beside, Built: checkBuiltOf(v.Built)}
|
||||
Head: v.Head, Trunk: v.Trunk, Beside: v.Beside, Built: checkBuiltOf(v.Built),
|
||||
Release: v.Release, JudgedBy: v.JudgedBy, BesideAt: v.BesideAt, Validator: v.Validator}
|
||||
// What each of a group's heads was judged as (novox/hq ADR 0323 §6).
|
||||
for _, m := range v.Members {
|
||||
result.Check.Members = append(result.Check.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo,
|
||||
Commit: m.Head, Trunk: m.Trunk})
|
||||
}
|
||||
// A branch tried before its pull request reads its outcome from this log, whole (novox/hq issue 686).
|
||||
if request.Check.Try {
|
||||
if line, err := json.Marshal(triedOf(v)); err == nil {
|
||||
say(link.StepTried, string(line))
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if err == nil {
|
||||
// The package-registry credential is a build input, so it is resolved before the clone: a
|
||||
@@ -346,15 +363,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
reuse.Pulls = append(reuse.Pulls, builder.PullRef{Owner: p.Owner, Repo: p.Repo, Number: p.Number})
|
||||
}
|
||||
// Only what a verdict names is taken (ADR 0350 §2).
|
||||
for _, b := range request.Reuse.Builds {
|
||||
named := builder.NamedBuild{Pull: builder.PullRef{Owner: b.Pull.Owner, Repo: b.Pull.Repo,
|
||||
Number: b.Pull.Number}, Check: b.Check, Module: b.Built.Module, Key: b.Built.Key,
|
||||
Digests: map[string]string{}}
|
||||
for _, m := range b.Built.Made {
|
||||
named.Digests[m.Name] = builder.DigestOf(m.Reference)
|
||||
}
|
||||
reuse.Named = append(reuse.Named, named)
|
||||
}
|
||||
reuse.Named = namedOf(request.Reuse.Builds)
|
||||
built, err = builder.BuildReusing(building, builder.Labelled(builder.Command, request.ID), publisher, reuse,
|
||||
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
||||
forgeFrom(), say, request.Seats)
|
||||
@@ -418,6 +427,11 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
for _, s := range built.Sources {
|
||||
result.Sources = append(result.Sources, link.BuildSource{Repository: s.Repository, Ref: s.Ref, Paths: s.Paths})
|
||||
}
|
||||
// The list of predefined keys the manifest was judged against, for the controller to publish
|
||||
// (novox/hq ADR 0348 §7).
|
||||
if p := built.Predefined; p != nil {
|
||||
result.Predefined = &link.PredefinedList{Keys: p.Keys, Generation: p.Generation}
|
||||
}
|
||||
say("built", built.Manifest.Module+" from "+short(built.Commit))
|
||||
}
|
||||
}
|
||||
@@ -457,9 +471,10 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
|
||||
spec := builder.CheckSpec{ID: request.ID, Repository: request.Repository, Ref: request.Ref,
|
||||
Owner: c.Owner, Repo: c.Repo, Number: c.Number, Paths: c.Paths, Beside: map[string]builder.Beside{},
|
||||
Modules: c.Modules, New: c.New, Manifests: c.Manifests, Judge: c.Judge, Base: c.Base,
|
||||
Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held)}
|
||||
Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held),
|
||||
Tests: c.Tests, StoreOnly: c.StoreOnly, Release: c.Release}
|
||||
for dir, b := range c.Beside {
|
||||
spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref}
|
||||
spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref, Else: b.Else}
|
||||
}
|
||||
for _, b := range c.Builds {
|
||||
spec.Builds = append(spec.Builds, builder.CheckBuildSpec{Module: b.Module, Repository: b.Repository, Path: b.Path,
|
||||
@@ -472,11 +487,50 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
|
||||
return spec
|
||||
}
|
||||
|
||||
// judgeCache is the judge cache the build agent's manifest declares (MESH_JUDGE_CACHE, novox/hq ADR 0358 §6): a
|
||||
// directory mounted at the same path on the host, so a judge build's mounts name what the agent sees. Empty when the
|
||||
// agent declares none, and every judge is then built cold under its check's root.
|
||||
func judgeCache() string { return os.Getenv("MESH_JUDGE_CACHE") }
|
||||
|
||||
// triedOf is a try's outcome as its log line says it (novox/hq issue 686): the repository's own check's verdict,
|
||||
// or the check's where that did not run — a head that does not merge — and the lines naming what failed.
|
||||
func triedOf(v builder.CheckVerdict) link.Tried {
|
||||
t := link.Tried{Verdict: v.Verdict, Summary: v.Summary, Took: v.Took.Round(time.Second).String(),
|
||||
Head: v.Head, Trunk: v.Trunk}
|
||||
if v.Repo != nil && (v.Gate == nil || v.Gate.Verdict == "pass") {
|
||||
t.Verdict, t.Summary = v.Repo.Verdict, v.Repo.Summary
|
||||
}
|
||||
if v.Repo != nil && v.Repo.Failed != "" {
|
||||
t.Failed = strings.Split(v.Repo.Failed, "\n")
|
||||
}
|
||||
if t.Verdict == "" {
|
||||
t.Verdict = "error"
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// namedOf are the builds the seat's verdicts name, as the builder believes a scratch record by them (novox/hq ADR
|
||||
// 0350 §2, issue 703): each module's key with every artifact's digest.
|
||||
func namedOf(builds []link.CheckedBuild) []builder.NamedBuild {
|
||||
var out []builder.NamedBuild
|
||||
for _, b := range builds {
|
||||
named := builder.NamedBuild{Pull: builder.PullRef{Owner: b.Pull.Owner, Repo: b.Pull.Repo,
|
||||
Number: b.Pull.Number}, Check: b.Check, Module: b.Built.Module, Key: b.Built.Key,
|
||||
Digests: map[string]string{}}
|
||||
for _, m := range b.Built.Made {
|
||||
named.Digests[m.Name] = builder.DigestOf(m.Reference)
|
||||
}
|
||||
out = append(out, named)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// checkBuiltOf is a check's builds for its merge as the outcome carries them (novox/hq ADR 0350 §1).
|
||||
func checkBuiltOf(built []builder.CheckBuilt) []link.CheckBuilt {
|
||||
var out []link.CheckBuilt
|
||||
for _, b := range built {
|
||||
c := link.CheckBuilt{Module: b.Module, Key: b.Key, NotBuilt: b.NotBuilt, Took: b.Took.Round(time.Second).String()}
|
||||
c := link.CheckBuilt{Module: b.Module, Key: b.Key, NotBuilt: b.NotBuilt, Took: b.Took.Round(time.Second).String(),
|
||||
Kept: b.Kept, Same: b.Same, SameAs: b.SameAs}
|
||||
for _, m := range b.Made {
|
||||
c.Made = append(c.Made, link.MadeArtifact{Name: m.Name, Kind: m.Kind, Reference: m.Reference})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A pull request into a release branch reaches the check as one** (novox/hq ADR 0358): the build seat hands the
|
||||
// check the release branch and, beside it, the trunk each repository is read at when it has no branch of that name;
|
||||
// and it hands the check the judge cache the build agent declares (MESH_JUDGE_CACHE), never a path of its own.
|
||||
func TestTheBuildSeatHandsTheCheckItsReleaseBranch(t *testing.T) {
|
||||
spec := checkSpecOf(link.BuildRequest{ID: "b", Repository: "r", Ref: "abc", Check: &link.CheckRequest{
|
||||
Owner: "novox", Repo: "mesh-catalog", Base: "feat/57", Release: "feat/57",
|
||||
Beside: map[string]link.CheckedOut{"mesh-host": {Repository: "h", Ref: "feat/57", Else: "main"}}}})
|
||||
if spec.Release != "feat/57" || spec.Beside["mesh-host"].Ref != "feat/57" || spec.Beside["mesh-host"].Else != "main" {
|
||||
t.Fatalf("the check was handed release %q and mesh-host %+v", spec.Release, spec.Beside["mesh-host"])
|
||||
}
|
||||
t.Setenv("MESH_JUDGE_CACHE", "/var/lib/mesh/build-agent/judge-cache")
|
||||
if got := judgeCache(); got != "/var/lib/mesh/build-agent/judge-cache" {
|
||||
t.Fatalf("the judge cache the manifest declares is read as %q", got)
|
||||
}
|
||||
t.Setenv("MESH_JUDGE_CACHE", "")
|
||||
if got := judgeCache(); got != "" {
|
||||
t.Fatalf("with no judge cache declared, %q is used", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
)
|
||||
|
||||
// A try's outcome, as the build seat says it in the build's log (novox/hq issue 686): the repository's own
|
||||
// check's verdict, how long it took, and each line naming what failed.
|
||||
func TestATrysOutcomeNamesWhatFailed(t *testing.T) {
|
||||
tried := triedOf(builder.CheckVerdict{Verdict: "pass", Summary: "the change touches no module", Took: 303 * time.Second,
|
||||
Head: "c0ffee", Trunk: "ca3f43c", Gate: &builder.Layer{Verdict: "pass"},
|
||||
Repo: &builder.Layer{Verdict: "fail", Summary: "its merge-check.sh failed: TestX",
|
||||
Failed: "--- FAIL: TestX (0.01s)\n x_test.go:12: got 1, want 2"}})
|
||||
if tried.Verdict != "fail" || tried.Took != "5m3s" || len(tried.Failed) != 2 || tried.Trunk != "ca3f43c" ||
|
||||
tried.Summary != "its merge-check.sh failed: TestX" {
|
||||
t.Fatalf("a failed try: %+v", tried)
|
||||
}
|
||||
// A head that does not merge fails both layers: the check's own word.
|
||||
conflict := triedOf(builder.CheckVerdict{Verdict: "fail", Summary: "does not merge", Gate: &builder.Layer{Verdict: "fail"},
|
||||
Repo: &builder.Layer{Verdict: "fail", Summary: "does not merge"}})
|
||||
if conflict.Verdict != "fail" || conflict.Summary != "does not merge" {
|
||||
t.Fatalf("a try that does not merge: %+v", conflict)
|
||||
}
|
||||
// No verdict is an error, never a pass.
|
||||
if none := triedOf(builder.CheckVerdict{}); none.Verdict != "error" {
|
||||
t.Fatalf("a try with no verdict: %+v", none)
|
||||
}
|
||||
}
|
||||
@@ -26,8 +26,11 @@ import (
|
||||
//
|
||||
// - **The serving controller** takes it before it does anything else — before it asserts the bus's
|
||||
// objects, which are the controller's to write — waiting while another holds it, and renews it.
|
||||
// A renewal refused or failed is the lease lost: the gate closes at once and the process exits, so
|
||||
// its service manager restarts it as a candidate (serve, in push.go).
|
||||
// Another holder seen, or the key's age passed with no renewal made, is the lease lost: the gate
|
||||
// closes at once and the process exits, so its service manager restarts it as a candidate (serve,
|
||||
// in push.go). A renewal the bus does not answer in time is not a loss (novox/hq issue 477): the
|
||||
// lease is kept, no new act starts once half the key's age passed unrenewed, and the next renewal
|
||||
// made opens the gate again.
|
||||
// - **A command run at a shell** — `push` in the installer, the lab, a person repairing a mesh whose
|
||||
// controller is down (issue 201) — acts **under the holder's epoch** when a controller holds the
|
||||
// lease: it is the same mesh's word, composed and sent under the store's hold of each machine like
|
||||
|
||||
@@ -140,7 +140,7 @@ func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testin
|
||||
}
|
||||
|
||||
// And a replay to the catalogue says where the store is now.
|
||||
announced, err := following{open}.Announceable(ctx)
|
||||
announced, err := following{open: open}.Announceable(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -424,8 +424,9 @@ func comparisonOf(module string, c comparison, opts takeOptions) (preview string
|
||||
}
|
||||
// And its settings on this machine, composed against its definition (rule 1, rule 6).
|
||||
for _, layer := range c.layers {
|
||||
keys := make([]string, 0, len(layer.Values))
|
||||
for k := range layer.Values {
|
||||
said := layer.Whole()
|
||||
keys := make([]string, 0, len(said))
|
||||
for k := range said {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
@@ -38,15 +38,61 @@ func watchApplyFailing(f *signalFacts) []conditions.Observation {
|
||||
}
|
||||
}
|
||||
var out []conditions.Observation
|
||||
said := map[string]bool{}
|
||||
for _, d := range f.doing {
|
||||
if !d.Wrong() || d.Since == nil || f.now.Sub(*d.Since) <= applyEvery || !heard[d.Node] {
|
||||
continue
|
||||
}
|
||||
out = append(out, applyFailing(d, f.now))
|
||||
said[d.Node] = true
|
||||
}
|
||||
// **A module's files the provisioner failed to place are a failed apply too** (novox/hq ADR 0362 §3): its client's
|
||||
// newest word, judged by the same bound — the client says a failure once, and fills again on every change —
|
||||
// on a machine whose node-engine applied what it was sent.
|
||||
failed := map[string][]inventory.ProvReport{}
|
||||
for _, r := range f.provFailed {
|
||||
if r.Status != inventory.ProvFailed || f.now.Sub(r.At) <= applyEvery || !heard[r.Node] || said[r.Node] {
|
||||
continue
|
||||
}
|
||||
failed[r.Node] = append(failed[r.Node], r)
|
||||
}
|
||||
for _, node := range sortedNames(failed) {
|
||||
out = append(out, provisioningFailing(node, failed[node], f.now))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// provisioningFailing is one machine's S21 condition for the modules whose files its provisioner client failed to
|
||||
// place.
|
||||
func provisioningFailing(m string, failed []inventory.ProvReport, now time.Time) conditions.Observation {
|
||||
sort.Slice(failed, func(a, b int) bool { return failed[a].Module < failed[b].Module })
|
||||
var names, words []string
|
||||
since := failed[0].At
|
||||
for _, r := range failed {
|
||||
names = append(names, r.Module)
|
||||
words = append(words, r.Module+": "+oneLine(orNoReason(r.Reason)))
|
||||
if r.At.Before(since) {
|
||||
since = r.At
|
||||
}
|
||||
}
|
||||
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m,
|
||||
Severity: conditions.Warning, Said: strings.Join(words, "; "),
|
||||
Resolved: m + " applies what it is sent again"}
|
||||
o.Summary = fmt.Sprintf("%s's provisioner has failed to place the files of %s since %s (bound %s)", m,
|
||||
strings.Join(names, ", "), since.UTC().Format("2006-01-02 15:04 MST"), ago(applyEvery))
|
||||
o.Headline = m + " is not applying its update"
|
||||
o.Explanation = fmt.Sprintf("The provisioner on %s has not been able to write the files of %s for %s. The "+
|
||||
"details say why.", m, strings.Join(names, ", "), humanDuration(now.Sub(since)))
|
||||
return o
|
||||
}
|
||||
|
||||
func orNoReason(s string) string {
|
||||
if s == "" {
|
||||
return "no reason given"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// applyFailing is one machine's S21 condition.
|
||||
func applyFailing(d inventory.Doing, now time.Time) conditions.Observation {
|
||||
m := d.Node
|
||||
|
||||
@@ -688,7 +688,7 @@ func spelledOf(m inventory.BatchedMerge) string {
|
||||
func namedOf(m inventory.BatchedMerge, repository string) inventory.PlanMerge {
|
||||
k := announcedOf(m)
|
||||
return inventory.PlanMerge{Repository: repository, Commit: m.Commit, Number: k.Number, Title: k.Title, Moves: k.Moves,
|
||||
Merged: m.Merged, Heard: m.Heard}
|
||||
Head: k.HeadSHA, Merged: m.Merged, Heard: m.Heard}
|
||||
}
|
||||
|
||||
// laterMerge says a was merged after b: by the forge's merge time, then by when each was heard.
|
||||
@@ -1000,10 +1000,15 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
|
||||
plan.Delivery.Merges = named
|
||||
plan.Delivery.Alone = batch.Delivery != nil && batch.Delivery.Alone
|
||||
plan.Delivery.Withheld = withheld
|
||||
// **Left where it is, a module whose program did not change** (novox/hq issue 721): its check found it the same as
|
||||
// the build the mesh holds.
|
||||
for n, why := range leaveSameAsRunning(ctx, inv, &plan, moved, append(append([]inventory.Edge{}, edges...), ordered...)) {
|
||||
unchanged[n] = why
|
||||
}
|
||||
plan.Delivery.Unchanged = unchangedLeft(unchanged, plan.Modules)
|
||||
// **Its moments and its class** (novox/hq ADR 0282 decision 6): measured, never acted on.
|
||||
plan.Times = walkTimesAtCut(*batch, plan, entries, now)
|
||||
if len(moved) == 0 {
|
||||
if len(moved) == 0 || len(plan.Modules) == 0 {
|
||||
plan.State = inventory.PlanDone
|
||||
plan.Tiers = [][]string{}
|
||||
plan.Note = "moved nothing the mesh holds: nothing to walk"
|
||||
@@ -1044,7 +1049,7 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
|
||||
return nil
|
||||
}
|
||||
// Written and its first tier asked as one act on the plans (novox/hq issue 213), under the caller's hold.
|
||||
if err := askTier(ctx, inv, &plan); err != nil {
|
||||
if err := askTier(ctx, inv, &plan, plan.Tiers[plan.Tier]); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
|
||||
@@ -758,7 +758,7 @@ func TestTheCatchUpKeepsWhatACutMadeHistory(t *testing.T) {
|
||||
return entries, read, err
|
||||
}
|
||||
if err := catchUpOnMerges(ctx, time.Now(), unheardMerges{announcedList(lost), open.inventory}, catalogued,
|
||||
(following{open}).SourceMoved, t.Logf); err != nil {
|
||||
(following{open: open}).SourceMoved, t.Logf); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
||||
|
||||
@@ -183,8 +183,10 @@ func buildFrom(result link.BuildResult) inventory.Build {
|
||||
kept.Manifest = recordedManifest(result.Manifest, announced)
|
||||
// The module name comes from the manifest, which only exists when the build got that far.
|
||||
if len(kept.Manifest) > 0 {
|
||||
if m, err := catalogue.ParseManifest(kept.Manifest); err == nil {
|
||||
kept.Module = m.Module
|
||||
if list, err := judgedWith(result); err == nil {
|
||||
if m, err := catalogue.ParseManifestWith(kept.Manifest, list); err == nil {
|
||||
kept.Module = m.Module
|
||||
}
|
||||
}
|
||||
}
|
||||
return kept
|
||||
@@ -603,6 +605,11 @@ func saysWhenThePolicyActs(ctx context.Context, inv *inventory.Inventory, module
|
||||
// written with the same values.
|
||||
func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResult) (
|
||||
catalogue.Manifest, inventory.Build, error) {
|
||||
// The list of predefined keys as the store keeps it now: another controller process may have kept a newer one.
|
||||
if err := loadPredefined(ctx, inv); err != nil {
|
||||
return catalogue.Manifest{}, inventory.Build{}, fmt.Errorf("the list of predefined keys kept cannot be "+
|
||||
"read: %w", err)
|
||||
}
|
||||
kept := buildFrom(result)
|
||||
if err := inv.RecordBuild(ctx, kept); err != nil {
|
||||
return catalogue.Manifest{}, kept, err
|
||||
@@ -613,7 +620,15 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
||||
return catalogue.Manifest{}, kept, fmt.Errorf("%s could not build %s:\n%s",
|
||||
result.On, result.Repository, result.Failed)
|
||||
}
|
||||
manifest, err := catalogue.ParseManifest(kept.Manifest)
|
||||
// Judged first against the list the build was judged against: the catalogue's at the commit built (novox/hq
|
||||
// ADR 0348 §7), so a module naming a key its own commit adds is read; and again below against the list the
|
||||
// mesh publishes, before it is registered.
|
||||
list, err := judgedWith(result)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On,
|
||||
result.Repository, short(result.Commit), err)
|
||||
}
|
||||
manifest, err := catalogue.ParseManifestWith(kept.Manifest, list)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, kept, fmt.Errorf("%s built %s and what came back is not a manifest: %w",
|
||||
result.On, result.Repository, err)
|
||||
@@ -687,6 +702,23 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
||||
"back: it is recorded and not registered again — a newer build is", result.On, manifest.Module,
|
||||
short(result.Commit))
|
||||
}
|
||||
// **The newest trunk build's list of predefined keys is published** (novox/hq ADR 0348 §7): a build of the
|
||||
// catalogue that may be registered keeps its list when its commit is newer than the one kept. Then the module
|
||||
// is judged against the list the mesh publishes, as every manifest read again is: one naming a key that list
|
||||
// does not hold, or of another type, is refused here and never enters the catalogue.
|
||||
if carried, err := carriedList(result); err != nil {
|
||||
return manifest, kept, err
|
||||
} else if carried != nil {
|
||||
if err := publishPredefined(ctx, inv, result, carried); err != nil {
|
||||
fmt.Printf("%s: the list of predefined keys it carried is not published: %v\n", result.ID, err)
|
||||
}
|
||||
}
|
||||
if manifest.NamesPredefined() {
|
||||
if _, err := catalogue.ParseManifest(kept.Manifest); err != nil {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: judged against the "+
|
||||
"list of predefined keys the mesh publishes, %w", result.On, manifest.Module, short(result.Commit), err)
|
||||
}
|
||||
}
|
||||
// **A build whose source is unchanged is never a move** (novox/hq issue 280): a rebuild made from
|
||||
// what the build the mesh stands on was made from registers that build's artifacts at the new
|
||||
// commit, so no machine is sent a new digest for a source nobody changed — an image is not
|
||||
@@ -820,6 +852,9 @@ type answers struct {
|
||||
sources map[string]inventory.Source
|
||||
// waiting is every machine not running what the mesh would send it.
|
||||
waiting []inventory.Machine
|
||||
// recordedWaits is, per machine, every move of a recorded module that waits there for a person (novox/hq ADR
|
||||
// 0377 §6): the machine is behind for these, and `push <node>` sends them.
|
||||
recordedWaits map[string][]string
|
||||
// reported is every machine's last word beside when it was last sent a declaration — the
|
||||
// pair that answers "has it caught up", which waiting alone cannot (the sent digest is
|
||||
// recorded at send, not at apply).
|
||||
@@ -899,6 +934,34 @@ type answers struct {
|
||||
//
|
||||
// Routed through the artifact store as the network reaches it now (novox/hq 04-ISSUES/102): a
|
||||
// base is recorded by digest and path, and a build machine needs something it can pull.
|
||||
// runningBy is, keyed "<module>/<artifact>", each artifact of the build the mesh has registered for its module, as a
|
||||
// builder reaches the artifact store (novox/hq issue 721): what a check compares its builds with. Empty when it cannot
|
||||
// be read, said, and then nothing is said unchanged.
|
||||
func runningBy(ctx context.Context) map[string]string {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read what this mesh runs, so the check compares nothing: %v\n", err)
|
||||
// empty-on-error: said above; a check given nothing to compare with says no module unchanged, and all move
|
||||
return nil
|
||||
}
|
||||
defer open.Close()
|
||||
running, err := open.inventory.RegisteredHeld(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read what this mesh runs, so the check compares nothing: %v\n", err)
|
||||
// empty-on-error: said above; a check given nothing to compare with says no module unchanged, and all move
|
||||
return nil
|
||||
}
|
||||
address, err := whereABuilderReachesTheStore(ctx, open.inventory)
|
||||
if err != nil || address == "" {
|
||||
return running
|
||||
}
|
||||
routed := make(map[string]string, len(running))
|
||||
for key, reference := range running {
|
||||
routed[key] = catalogue.Rerouted(reference, address)
|
||||
}
|
||||
return routed
|
||||
}
|
||||
|
||||
func heldBy(ctx context.Context) map[string]string {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **Every user's slow consumer is S9's, not the controller's alone** (novox/hq ADR 0357, issue 270). On
|
||||
// 2026-10-07 the bus closed two clients on the laptop as slow consumers and nothing raised a condition:
|
||||
// the controller heard only its own connection. Said by the bus module and read back, a user the bus
|
||||
// closes is one condition naming the module and its machine, raised at the first look and cleared an hour
|
||||
// after the last closure.
|
||||
func TestAUserTheBusClosedAsASlowConsumerIsOneConditionNamingItsModuleAndMachine(t *testing.T) {
|
||||
closed := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC)
|
||||
now := closed.Add(20 * time.Second)
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
||||
t.Cleanup(func() { k.Close(t.Context()) })
|
||||
|
||||
advisories := link.BusClosures([]link.BusClosure{
|
||||
{User: "laptop.claude-code", CID: 10, Reason: "Slow Consumer (Write Deadline)", Opened: closed.Add(-time.Hour),
|
||||
Closed: closed.Add(-10 * time.Minute)},
|
||||
{User: "laptop.claude-code", CID: 44, Reason: "Slow Consumer (Write Deadline)", Opened: closed.Add(-time.Minute),
|
||||
Closed: closed},
|
||||
{User: "anchor.desk", CID: 7, Reason: "Maximum Message Payload Exceeded", Opened: closed.Add(-time.Hour),
|
||||
Closed: closed},
|
||||
}, []string{link.AdvisorySlowConsumer, link.AdvisorySlowConsumer, link.AdvisoryConnectionClosed}, closed.Add(-time.Hour))
|
||||
look := func() {
|
||||
t.Helper()
|
||||
f := &signalFacts{now: now, host: "anchor", advisories: advisories}
|
||||
if err := k.Reconcile(t.Context(), "S9", watchAdvisories(f)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
look()
|
||||
|
||||
slow, found, err := conditions.ReadOne(t.Context(), store, "bus.laptop.claude-code.slow-consumer")
|
||||
if err != nil || !found {
|
||||
t.Fatalf("no slow-consumer condition for the user: %v %v", found, err)
|
||||
}
|
||||
if slow.Subject.Machine != "laptop" || slow.Observations != 2 || !slow.LastObserved.Equal(closed) {
|
||||
t.Fatalf("the slow consumer is said on %q, %d closure(s), last %s: %+v", slow.Subject.Machine, slow.Observations,
|
||||
slow.LastObserved, slow)
|
||||
}
|
||||
if !strings.Contains(slow.Summary, "claude-code on laptop") || strings.Contains(slow.Summary, "laptop.claude-code") {
|
||||
t.Fatalf("the summary does not name the module in words, or carries the dotted user: %q", slow.Summary)
|
||||
}
|
||||
if !strings.Contains(slow.Headline, "Claude-code on laptop") {
|
||||
t.Fatalf("the headline is %q", slow.Headline)
|
||||
}
|
||||
cut, found, err := conditions.ReadOne(t.Context(), store, "bus.anchor.desk.connection-closed")
|
||||
if err != nil || !found || cut.Subject.Machine != "anchor" || !strings.Contains(cut.Summary, "maximum message payload exceeded") {
|
||||
t.Fatalf("the user cut off for breaking the bus's rules: %v %v %+v", found, err, cut)
|
||||
}
|
||||
|
||||
// An hour after the newest closure without another, both clear.
|
||||
now = closed.Add(time.Hour + time.Minute)
|
||||
look()
|
||||
for _, key := range []string{"bus.laptop.claude-code.slow-consumer", "bus.anchor.desk.connection-closed"} {
|
||||
if _, found, _ := k.Get(t.Context(), key); found {
|
||||
t.Fatalf("%s is still open an hour after the last closure", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **A read of the bus module's events that fails does not blind S9** (review of #326): the controller's own
|
||||
// signals are still said, and the failed read is a condition of its own.
|
||||
func TestABusModuleReadThatFailsKeepsTheControllersOwnSignals(t *testing.T) {
|
||||
at := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC)
|
||||
f := &signalFacts{now: at.Add(time.Minute), host: "anchor",
|
||||
advisories: []link.Advisory{{Kind: link.AdvisorySlowConsumer, ID: "controller", Said: "the controller fell behind",
|
||||
First: at, Last: at, Count: 1}},
|
||||
busFaultsErr: errors.New("nats: timeout")}
|
||||
keys := map[string]bool{}
|
||||
for _, o := range watchAdvisories(f) {
|
||||
keys[o.Key()] = true
|
||||
}
|
||||
if !keys["bus.controller.slow-consumer"] || !keys["bus.bus-module.bus-faults-unread"] {
|
||||
t.Fatalf("observed %v: the controller's own slow consumer and the unread events, both", keys)
|
||||
}
|
||||
for _, row := range signalsTable {
|
||||
if row.Row == "S9" && row.needs(f) != nil {
|
||||
t.Fatal("the S9 row went blind for a read of the bus module's events")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -49,10 +49,12 @@ func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Ra
|
||||
// composed from, so a module the mesh grants a consumer's subjects has that consumer waiting.
|
||||
// Done on every raise, not only when a credential is issued: every module moved onto this bus
|
||||
// by the rollout was issued on the old one, and came up with nothing to bind to (2026-09-28).
|
||||
consumers, err := moduleConsumers(ctx, inv)
|
||||
consumers, records, err := moduleConsumersAndRecords(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// And a grant of a restricted stream that composes nothing, said on every raise and send (novox/hq ADR 0376 §2).
|
||||
sayWithheld(records, " ")
|
||||
// And the work queues of seats that name their caller or their kind, with each holder's worker
|
||||
// (novox/hq ADR 0259 §3): an ask queues until the router takes it, a channel's work until that kind
|
||||
// takes it.
|
||||
@@ -60,9 +62,19 @@ func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Ra
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// And every module's own stream (novox/hq ADR 0363), from the catalogue, before the consumers on it.
|
||||
modStreams, err := inv.DeclaredStreams(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Every one tried, and every failure named: one module's consumer the bus refuses is no reason
|
||||
// the modules after it in the list hear nothing (novox/hq issue 208, where this runs on each send).
|
||||
var failed []error
|
||||
for _, s := range broker.ModuleStreams(modStreams) {
|
||||
if err := r.EnsureStream(s); err != nil {
|
||||
failed = append(failed, fmt.Errorf("the module stream %s: %w", s.Name, err))
|
||||
}
|
||||
}
|
||||
for _, s := range trafficStreams {
|
||||
if err := r.EnsureStream(s); err != nil {
|
||||
failed = append(failed, fmt.Errorf("the work queue %s: %w", s.Name, err))
|
||||
@@ -73,11 +85,30 @@ func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Ra
|
||||
failed = append(failed, fmt.Errorf("the worker %s on %s: %w", c.Name, c.Stream, err))
|
||||
}
|
||||
}
|
||||
restricted := map[string]bool{}
|
||||
for _, s := range modStreams {
|
||||
restricted[s.Name] = s.Restricted
|
||||
}
|
||||
for _, c := range consumers {
|
||||
if err := r.EnsureConsumer(c.Consumer); err != nil {
|
||||
if err := broker.EnsureHeard(r, c.Consumer, restricted[c.Consumer.Stream]); err != nil {
|
||||
failed = append(failed, fmt.Errorf("how %s on %s hears what it consumes: %w", c.Module, c.Node, err))
|
||||
}
|
||||
}
|
||||
// **And a consumer no longer granted loses what it reads through** (novox/hq ADR 0363 §6): every consumer
|
||||
// on a module's stream the composition no longer asks for is removed, on this very send.
|
||||
if p, prunes := r.(broker.Pruner); prunes && len(modStreams) > 0 {
|
||||
var wanted []broker.Consumer
|
||||
for _, c := range consumers {
|
||||
wanted = append(wanted, c.Consumer)
|
||||
}
|
||||
removed, err := broker.PruneHeard(p, modStreams, wanted)
|
||||
if err != nil {
|
||||
failed = append(failed, err)
|
||||
}
|
||||
for _, gone := range removed {
|
||||
fmt.Printf(" removed the consumer %s: its module is no longer granted that stream (novox/hq ADR 0363)\n", gone)
|
||||
}
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
return nil, errors.Join(failed...)
|
||||
}
|
||||
@@ -136,15 +167,21 @@ func unassertedObservation(err error) conditions.Observation {
|
||||
|
||||
// moduleConsumers is every module's durable consumer, from the records the user list is composed from.
|
||||
func moduleConsumers(ctx context.Context, inv *inventory.Inventory) ([]broker.ModuleConsumer, error) {
|
||||
consumers, _, err := moduleConsumersAndRecords(ctx, inv)
|
||||
return consumers, err
|
||||
}
|
||||
|
||||
// moduleConsumersAndRecords is moduleConsumers with the records they were composed from.
|
||||
func moduleConsumersAndRecords(ctx context.Context, inv *inventory.Inventory) ([]broker.ModuleConsumer, broker.Records, error) {
|
||||
records, err := inv.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, broker.Records{}, err
|
||||
}
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, broker.Records{}, err
|
||||
}
|
||||
return broker.ConsumersOf(users), nil
|
||||
return broker.ConsumersOf(users), records, nil
|
||||
}
|
||||
|
||||
// seatTrafficObjects is the work queues and workers of seats that name their caller or their kind, from
|
||||
|
||||
@@ -73,6 +73,11 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
|
||||
"(issue 280)", busUpgradeNeeded, name, busUpgradeVerb, strings.Join(e.On, ", ")))
|
||||
case name == "nats" || catalogue.ProvidesBus(e.Manifest):
|
||||
p.Steps = append(p.Steps, "a planned bus step: the bus is upgraded by `bus upgrade`, never by an ordinary send")
|
||||
case waits && len(e.On) > 0 && (u.From == catalogue.FromModule || u.From == catalogue.FromData):
|
||||
// Judged per machine by the walk, against the build each was last sent (novox/hq ADR 0377).
|
||||
p.Steps = append(p.Steps, fmt.Sprintf("%s waits for a person: its policy records (%s) — unless on a machine "+
|
||||
"its move recreates none of its containers and moves none of its data, which the walk judges and then sends",
|
||||
name, orNone(u.From)))
|
||||
case waits && len(e.On) > 0:
|
||||
p.Steps = append(p.Steps, fmt.Sprintf("%s waits for a person: its policy records (%s)", name,
|
||||
orNone(u.From)))
|
||||
|
||||
@@ -47,6 +47,24 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
shelf := catalogue.Shelf{}
|
||||
faulted := map[string]bool{}
|
||||
failed := 0
|
||||
// The list of predefined keys at the root of the repository above each manifest (novox/hq ADR 0348 §7), read
|
||||
// once per repository: a manifest naming a key is judged against the list its own tree holds.
|
||||
lists := map[string]catalogue.PredefinedList{}
|
||||
listOf := func(path string) (catalogue.PredefinedList, error) {
|
||||
root := catalogue.RepositoryRoot(filepath.Dir(path))
|
||||
if root == "" {
|
||||
return nil, nil
|
||||
}
|
||||
if list, read := lists[root]; read {
|
||||
return list, nil
|
||||
}
|
||||
list, err := catalogue.ReadPredefined(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lists[root] = list
|
||||
return list, nil
|
||||
}
|
||||
for _, path := range paths {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
@@ -54,7 +72,18 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
list, err := listOf(path)
|
||||
if err != nil {
|
||||
fmt.Fprintf(out, "%s: %v\n", path, err)
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
m, err := catalogue.ParseManifestWith(raw, list)
|
||||
if err == nil {
|
||||
// Each template's text read from the tree beside the manifest, and judged as inline content is
|
||||
// (novox/hq ADR 0349 §2): a missing template is refused by name.
|
||||
m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(path))
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintf(out, "%s: %v\n", path, err)
|
||||
failed++
|
||||
@@ -215,6 +244,12 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
checks = append(checks, fmt.Sprintf("%v by %s", r["id"], catalogue.HealthWords(h)))
|
||||
}
|
||||
}
|
||||
// And its own runtime's, where it runs as its own account (novox/hq ADR 0366).
|
||||
if own, has := catalogue.OwnRuntimeHealth(m); has {
|
||||
if h, _, _ := catalogue.ReadHealth(own); h.Kind != "" {
|
||||
checks = append(checks, fmt.Sprintf("%v by %s", own["id"], catalogue.HealthWords(h)))
|
||||
}
|
||||
}
|
||||
if len(checks) > 0 {
|
||||
fmt.Fprintf(out, ", ready: %s", strings.Join(checks, "; "))
|
||||
}
|
||||
|
||||
@@ -125,6 +125,7 @@ func TestAWalksBuildSaysWhetherItReusedOrBuilt(t *testing.T) {
|
||||
// build as it judges one the walk built. A scratch reference is refused. Needs the store (MESH_TEST_POSTGRES), and
|
||||
// is named in testdata/store-tests.
|
||||
func TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := g.open.inventory
|
||||
@@ -190,27 +191,90 @@ func TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
// A walk's leave names the builds its own pull requests' verdicts made, and none when they cannot be read: the
|
||||
// builder takes nothing a verdict does not name (novox/hq ADR 0350 §2).
|
||||
// A walk's leave names the builds its own pull requests' verdicts made at the head each merge merged — never an
|
||||
// earlier head's — read once per walk, and none when they cannot be read: the builder takes nothing a verdict does
|
||||
// not name (novox/hq ADR 0350 §2, issue 703).
|
||||
func TestAWalksLeaveNamesTheBuildsItsPullRequestsVerdictsMade(t *testing.T) {
|
||||
was := checkedBuildsOf
|
||||
t.Cleanup(func() { checkedBuildsOf = was })
|
||||
var asked []link.PullRef
|
||||
named := []link.CheckedBuild{{Pull: link.PullRef{Owner: "novox", Repo: "shop", Number: 7}, Check: "build-1",
|
||||
Built: link.CheckBuilt{Module: "app", Key: "r1:1"}}}
|
||||
forgetWalkVerdicts()
|
||||
t.Cleanup(func() { checkedBuildsOf = was; forgetWalkVerdicts() })
|
||||
var asked [][]link.PullRef
|
||||
pull := link.PullRef{Owner: "novox", Repo: "shop", Number: 7}
|
||||
atHead := link.CheckedBuild{Pull: pull, Check: "build-2", Head: "bbbbbbbbbbbb", Built: link.CheckBuilt{Module: "app", Key: "r1:2"}}
|
||||
earlier := link.CheckedBuild{Pull: pull, Check: "build-1", Head: "aaaaaaaaaaaa", Built: link.CheckBuilt{Module: "app", Key: "r1:1"}}
|
||||
checkedBuildsOf = func(_ context.Context, pulls []link.PullRef) ([]link.CheckedBuild, error) {
|
||||
asked = pulls
|
||||
return named, nil
|
||||
asked = append(asked, pulls)
|
||||
return []link.CheckedBuild{atHead, earlier}, nil
|
||||
}
|
||||
pulls := []link.PullRef{{Owner: "novox", Repo: "shop", Number: 7}}
|
||||
if got := reuseWithVerdicts(t.Context(), &link.ReuseAsk{Pulls: pulls}); !reflect.DeepEqual(got.Builds, named) ||
|
||||
!reflect.DeepEqual(asked, pulls) {
|
||||
t.Fatalf("the leave names %+v, read for %+v", got.Builds, asked)
|
||||
pulls := []link.PullRef{pull}
|
||||
heads := map[link.PullRef]string{pull: "bbbbbbbbbbbb0123456789"}
|
||||
for range 3 {
|
||||
if got := reuseWithVerdicts(t.Context(), "plan-1", &link.ReuseAsk{Pulls: pulls}, heads); !reflect.DeepEqual(got.Builds,
|
||||
[]link.CheckedBuild{atHead}) {
|
||||
t.Fatalf("the leave names %+v, not only the merged head's build", got.Builds)
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(asked, [][]link.PullRef{pulls}) {
|
||||
t.Fatalf("the verdicts were read %d times for one walk: %v", len(asked), asked)
|
||||
}
|
||||
// A merge whose head was not said takes nothing.
|
||||
if got := reuseWithVerdicts(t.Context(), "plan-1", &link.ReuseAsk{Pulls: pulls}, nil); got.Builds != nil {
|
||||
t.Fatalf("a merge of no said head names %+v", got.Builds)
|
||||
}
|
||||
checkedBuildsOf = func(context.Context, []link.PullRef) ([]link.CheckedBuild, error) {
|
||||
return nil, errors.New("no bus")
|
||||
}
|
||||
if got := reuseWithVerdicts(t.Context(), &link.ReuseAsk{Pulls: pulls}); got.Builds != nil {
|
||||
if got := reuseWithVerdicts(t.Context(), "plan-2", &link.ReuseAsk{Pulls: pulls}, heads); got.Builds != nil {
|
||||
t.Fatalf("verdicts that could not be read named %+v", got.Builds)
|
||||
}
|
||||
}
|
||||
|
||||
// A check is handed only the builds earlier checks of its own pull request at its own head made: another pull
|
||||
// request's build, or an earlier head's, is never kept (novox/hq issue 703). None when they cannot be read.
|
||||
func TestACheckIsHandedOnlyItsOwnHeadsBuilds(t *testing.T) {
|
||||
was := checkedBuildsOf
|
||||
t.Cleanup(func() { checkedBuildsOf = was })
|
||||
pull := link.PullRef{Owner: "novox", Repo: "shop", Number: 7}
|
||||
mine := link.CheckedBuild{Pull: pull, Check: "build-1", Head: "bbbbbbbbbbbb", Built: link.CheckBuilt{Module: "app", Key: "r1:1"}}
|
||||
earlier := link.CheckedBuild{Pull: pull, Check: "build-0", Head: "aaaaaaaaaaaa", Built: link.CheckBuilt{Module: "app", Key: "r1:1"}}
|
||||
another := link.CheckedBuild{Pull: link.PullRef{Owner: "novox", Repo: "shop", Number: 8}, Check: "build-9",
|
||||
Head: "bbbbbbbbbbbb", Built: link.CheckBuilt{Module: "app", Key: "r1:1"}}
|
||||
var asked [][]link.PullRef
|
||||
checkedBuildsOf = func(_ context.Context, pulls []link.PullRef) ([]link.CheckedBuild, error) {
|
||||
asked = append(asked, pulls)
|
||||
return []link.CheckedBuild{mine, earlier, another}, nil
|
||||
}
|
||||
p := link.PullUpdated{Owner: "novox", Repo: "shop", Number: 7, Commit: "bbbbbbbbbbbb0123456789"}
|
||||
builds := []link.CheckBuild{{Module: "app"}}
|
||||
if got := namedForTheCheck(t.Context(), p, builds); !reflect.DeepEqual(got, []link.CheckedBuild{mine}) ||
|
||||
!reflect.DeepEqual(asked, [][]link.PullRef{{pull}}) {
|
||||
t.Fatalf("the check is handed %+v, read for %v", got, asked)
|
||||
}
|
||||
checkedBuildsOf = func(context.Context, []link.PullRef) ([]link.CheckedBuild, error) {
|
||||
return nil, errors.New("no bus")
|
||||
}
|
||||
if got := namedForTheCheck(t.Context(), p, builds); got != nil {
|
||||
t.Fatalf("verdicts that could not be read named %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func forgetWalkVerdicts() {
|
||||
walkVerdicts.Lock()
|
||||
defer walkVerdicts.Unlock()
|
||||
walkVerdicts.read = map[string]verdictsRead{}
|
||||
}
|
||||
|
||||
// A walk's merged heads are the forge's word for each merge; a merge announced without its head, or a pull request
|
||||
// merged at two heads in one walk, has none, and its walk takes no check's build (novox/hq issue 703).
|
||||
func TestAWalksMergedHeadsAreTheForgesWord(t *testing.T) {
|
||||
merge := func(number int, head string) inventory.PlanMerge {
|
||||
return inventory.PlanMerge{Repository: "novox/shop", Number: number, Head: head}
|
||||
}
|
||||
p := &inventory.Plan{Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{
|
||||
merge(7, "aaaaaaaaaaaa1"), merge(8, ""), merge(9, "bbbbbbbbbbbb"), merge(9, "cccccccccccc")}}}
|
||||
heads := mergedHeadsOf(p)
|
||||
shop := func(n int) link.PullRef { return link.PullRef{Owner: "novox", Repo: "shop", Number: n} }
|
||||
if heads[shop(7)] != "aaaaaaaaaaaa1" || heads[shop(8)] != "" || heads[shop(9)] != "" {
|
||||
t.Fatalf("merged heads %v", heads)
|
||||
}
|
||||
}
|
||||
|
||||
+193
-25
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path"
|
||||
"regexp"
|
||||
@@ -188,6 +189,8 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||
if err := ask.Ask(ctx, request); err != nil {
|
||||
return err
|
||||
}
|
||||
// A new head ends the check of the old one (novox/hq issue 726).
|
||||
endingSupersededChecks(ctx, seat, p, request.ID, f.oneShot)
|
||||
what := "its own merge-check.sh alone: " + noModuleTouched
|
||||
if scope.gated() {
|
||||
what = fmt.Sprintf("the gate over %s (a merge would build %d module(s) in %d tier(s))",
|
||||
@@ -248,31 +251,16 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
|
||||
if err != nil {
|
||||
return link.BuildRequest{}, err
|
||||
}
|
||||
// Beside it, at what the mesh runs: each core repository by the module the mesh builds from it.
|
||||
beside := map[string]link.CheckedOut{}
|
||||
for _, e := range entries {
|
||||
dir, core := coreModules[e.Manifest.Module]
|
||||
if !core || e.Provided || e.Source.Repository == "" {
|
||||
continue
|
||||
}
|
||||
url, err := clone(e.Source)
|
||||
if err != nil {
|
||||
return link.BuildRequest{}, err
|
||||
}
|
||||
refs := besideRefs(dir, current[e.Manifest.Module].Commit)
|
||||
beside[dir] = link.CheckedOut{Repository: url, Ref: refs[dir]}
|
||||
if dir == "mesh-controller" {
|
||||
beside["mesh-controller-main"] = link.CheckedOut{Repository: url, Ref: refs["mesh-controller-main"]}
|
||||
if e.Source.Seat != "" {
|
||||
for _, sibling := range []string{"mesh-lab", "mesh-sdk"} {
|
||||
if url, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository,
|
||||
sibling)}); err == nil {
|
||||
beside[sibling] = link.CheckedOut{Repository: url, Ref: refs[sibling]}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// **A pull request into a release branch is judged by that branch** (novox/hq ADR 0358): asked of the forge once.
|
||||
release, err := releaseBranchOf(ctx, p, scope, entries)
|
||||
if err != nil {
|
||||
return link.BuildRequest{}, err
|
||||
}
|
||||
beside, err := besideOf(entries, func(module string) string { return current[module].Commit }, clone, release)
|
||||
if err != nil {
|
||||
return link.BuildRequest{}, err
|
||||
}
|
||||
builds := checkBuildsOf(p, scope, entries, clone)
|
||||
return link.BuildRequest{
|
||||
ID: link.NewBuildID(time.Now()),
|
||||
Repository: repository,
|
||||
@@ -282,10 +270,188 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
|
||||
Source: sourceOnSeat(source),
|
||||
Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base,
|
||||
Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents, New: scope.New,
|
||||
Manifests: scope.Manifests, Judge: scope.Judge, Builds: checkBuildsOf(p, scope, entries, clone)},
|
||||
Manifests: scope.Manifests, Judge: scope.Judge, Builds: builds, Named: namedForTheCheck(ctx, p, builds),
|
||||
Release: release, Running: runningBy(ctx)},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// besideOf is what a check reads beside its own repository, by the directory it is found under: each core repository
|
||||
// by the module the mesh builds from it, at besideRefs — or, for a pull request into a release branch (novox/hq ADR
|
||||
// 0358 §3), at its branch of that name where it has one and at its trunk otherwise, the controller's main kept for
|
||||
// its judge, and mesh-lab at its trunk whatever the base: its replays run with the container runtime's socket. running
|
||||
// is the commit the mesh runs of a module.
|
||||
func besideOf(entries []inventory.Entry, running func(module string) string,
|
||||
clone func(inventory.Source) (string, error), release string) (map[string]link.CheckedOut, error) {
|
||||
beside := map[string]link.CheckedOut{}
|
||||
for _, e := range entries {
|
||||
dir, core := coreModules[e.Manifest.Module]
|
||||
if !core || e.Provided || e.Source.Repository == "" {
|
||||
continue
|
||||
}
|
||||
url, err := clone(e.Source)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
refs := besideRefs(dir, running(e.Manifest.Module))
|
||||
at := func(dir, trunk string) link.CheckedOut {
|
||||
if release == "" {
|
||||
return link.CheckedOut{Repository: url, Ref: refs[dir]}
|
||||
}
|
||||
if trunk == "" {
|
||||
trunk = "main"
|
||||
}
|
||||
return link.CheckedOut{Repository: url, Ref: release, Else: trunk}
|
||||
}
|
||||
beside[dir] = at(dir, followedBranch(e.Source.Ref))
|
||||
if dir == "mesh-controller" {
|
||||
// The controller's main stays its main: it judges only a check the running controller predates.
|
||||
beside["mesh-controller-main"] = link.CheckedOut{Repository: url, Ref: refs["mesh-controller-main"]}
|
||||
if release != "" {
|
||||
// mesh-controller has the release branch: that is what makes it one (ADR 0358 §1).
|
||||
beside[dir] = link.CheckedOut{Repository: url, Ref: release}
|
||||
}
|
||||
if e.Source.Seat != "" {
|
||||
for _, sibling := range []string{"mesh-lab", "mesh-sdk"} {
|
||||
if url, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository,
|
||||
sibling)}); err == nil {
|
||||
b := link.CheckedOut{Repository: url, Ref: refs[sibling]}
|
||||
// **mesh-lab is read at its trunk, always** (ADR 0358): its replays run with the container
|
||||
// runtime's socket, so they are only ever reviewed code — never a branch anyone may create.
|
||||
if release != "" && sibling != "mesh-lab" {
|
||||
b = link.CheckedOut{Repository: url, Ref: release, Else: "main"}
|
||||
}
|
||||
beside[sibling] = b
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return beside, nil
|
||||
}
|
||||
|
||||
// releaseBranchOf is the release branch a pull request merges into (novox/hq ADR 0358 §1): its base, when that is not
|
||||
// its repository's trunk and mesh-controller has a branch of that name; empty otherwise, and the pull request is
|
||||
// checked as before it. **A base that is a trunk asks nothing**: the branch a module built from the repository follows
|
||||
// (main when it records none), or mesh-controller's own trunk. Any other base is asked of the forge once, and an
|
||||
// answer it cannot give is an error, never read as "no release branch".
|
||||
func releaseBranchOf(ctx context.Context, p link.PullUpdated, scope checkScope, entries []inventory.Entry) (string, error) {
|
||||
base := p.Base
|
||||
if base == "" {
|
||||
return "", nil
|
||||
}
|
||||
if scope.From != nil && base == trunkOf(scope.From.Source) {
|
||||
return "", nil
|
||||
}
|
||||
var controller *inventory.Entry
|
||||
for i, e := range entries {
|
||||
if e.Manifest.Module == "mesh-controller" && !e.Provided && e.Source.Repository != "" {
|
||||
controller = &entries[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if controller == nil || base == trunkOf(controller.Source) {
|
||||
return "", nil
|
||||
}
|
||||
if !tryRef.MatchString(base) || strings.Contains(base, "..") {
|
||||
return "", nil // no branch the forge could hold: checked as before
|
||||
}
|
||||
owner, repo := sourceOwner(controller.Source.Repository), sourceName(controller.Source.Repository)
|
||||
if owner == "" || repo == "" {
|
||||
return "", nil
|
||||
}
|
||||
has, err := forgeHasBranch(ctx, owner, repo, base)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("whether %s/%s#%d merges into a release branch cannot be told — the forge did not say "+
|
||||
"whether %s/%s has a branch %s (novox/hq ADR 0358): %w", p.Owner, p.Repo, p.Number, owner, repo, base, err)
|
||||
}
|
||||
if !has {
|
||||
return "", nil
|
||||
}
|
||||
return base, nil
|
||||
}
|
||||
|
||||
// trunkOf is the branch a module's source follows: main when it records none, or records a commit.
|
||||
func trunkOf(s inventory.Source) string {
|
||||
if t := strings.TrimSpace(followedBranch(s.Ref)); t != "" {
|
||||
return t
|
||||
}
|
||||
return "main"
|
||||
}
|
||||
|
||||
// sourceName is the name of a recorded repository, a path on the git seat or a URL: novox/mesh-host → mesh-host.
|
||||
func sourceName(repository string) string {
|
||||
parts := strings.Split(strings.Trim(strings.TrimSuffix(repository, ".git"), "/"), "/")
|
||||
if len(parts) < 2 {
|
||||
return ""
|
||||
}
|
||||
return parts[len(parts)-1]
|
||||
}
|
||||
|
||||
// forgeHasBranch asks the forge, through its module's tools on the bus, whether a repository has a branch. A
|
||||
// variable so a test needs no forge.
|
||||
var forgeHasBranch = func(ctx context.Context, owner, repo, branch string) (bool, error) {
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer js.Close()
|
||||
raw, _ := json.Marshal(map[string]any{"path": "/repos/" + owner + "/" + repo + "/branches/" + branch})
|
||||
answer, err := link.Ask(ctx, link.OverNATS{Conn: js.Conn()}, "gitea", "gitea_api", raw, 30*time.Second)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return branchAnswered(answer, branch)
|
||||
}
|
||||
|
||||
// branchAnswered reads the forge's answer about a branch: a 404 is "no such branch"; an answer naming the branch is
|
||||
// "it has it"; anything else — another error, an unreadable answer, one naming no branch or another — is an error,
|
||||
// never read as "no release branch".
|
||||
func branchAnswered(answer link.Answer, branch string) (bool, error) {
|
||||
if answer.Error != "" {
|
||||
if strings.Contains(answer.Error, " 404 ") {
|
||||
return false, nil
|
||||
}
|
||||
return false, errors.New(answer.Error)
|
||||
}
|
||||
var found struct {
|
||||
Result struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"result"`
|
||||
}
|
||||
if err := json.Unmarshal(answer.Result, &found); err != nil {
|
||||
return false, fmt.Errorf("the forge's answer about branch %s cannot be read: %w", branch, err)
|
||||
}
|
||||
if found.Result.Name != branch {
|
||||
return false, fmt.Errorf("the forge answered about branch %s naming %q", branch, found.Result.Name)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// namedForTheCheck are the builds earlier checks of the same pull request at the same head named, read once for the
|
||||
// check (novox/hq issue 703): asked again because only the trunk moved, the check keeps what they built of a key
|
||||
// still the same. Another pull request's build, or one of an earlier head, is never kept. None when they cannot be
|
||||
// read, and the check builds, as before.
|
||||
func namedForTheCheck(ctx context.Context, p link.PullUpdated, builds []link.CheckBuild) []link.CheckedBuild {
|
||||
if len(builds) == 0 || p.Commit == "" {
|
||||
return nil
|
||||
}
|
||||
pull := link.PullRef{Owner: p.Owner, Repo: p.Repo, Number: p.Number}
|
||||
read, err := checkedBuildsOf(ctx, []link.PullRef{pull})
|
||||
if err != nil {
|
||||
fmt.Printf(" the earlier checks' builds of %s/%s#%d could not be read, so the check builds: %v\n",
|
||||
p.Owner, p.Repo, p.Number, err)
|
||||
// empty-on-error: said above; a check handed no named build builds every module, as before issue 703
|
||||
return nil
|
||||
}
|
||||
var named []link.CheckedBuild
|
||||
for _, b := range read {
|
||||
if b.Pull == pull && link.SameHead(b.Head, p.Commit) {
|
||||
named = append(named, b)
|
||||
}
|
||||
}
|
||||
return named
|
||||
}
|
||||
|
||||
// checkBuildsOf are the modules a merge of the change would build, in the walk's order, as the check builds them
|
||||
// for the walk to take (novox/hq ADR 0350 §1): a module built from the repository checked, at the branch it merges
|
||||
// into, from the merge the check makes; any other — a module standing on one — from its own repository at the
|
||||
@@ -432,6 +598,8 @@ func checkedOf(result link.BuildResult) link.Checked {
|
||||
// repositories read beside it.
|
||||
c.Trunk, c.Beside, c.Took = result.Check.Trunk, result.Check.Beside, result.Check.Took
|
||||
c.Built = result.Check.Built
|
||||
c.Release, c.JudgedBy, c.BesideAt = result.Check.Release, result.Check.JudgedBy, result.Check.BesideAt
|
||||
c.Validator = result.Check.Validator
|
||||
if c.Gate == nil {
|
||||
// A build seat from before the layers: its verdict is the gate's.
|
||||
c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
|
||||
|
||||
@@ -0,0 +1,616 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **The move of config** (novox/hq ADR 0348 §9 and its consequences, design 57 K5, issue 664): every config value a
|
||||
// settings layer holds is set in mesh-config, one of the moves of data a release runs once after its delivery, after
|
||||
// the move of placement. A module's layer on a node becomes its own value on that node (`module.<node>.<module>.<key>`).
|
||||
// A whole-mesh layer has no module value for the whole mesh (ADR 0348 §3): its value becomes mesh config for the
|
||||
// predefined key it falls back to where that module alone falls back to the key (claude-code's managed settings
|
||||
// become `agent-managed-settings`), and otherwise the module's own value on each node it is assigned to, laid under
|
||||
// that node's layer as composition lays it. A key a layer still says by its file's name (nodered's `timeZone`) is
|
||||
// set under its config key (`time-zone`, issue 751).
|
||||
//
|
||||
// **Every node composes byte for byte as before.** The move sets the values, reads every module's resolved values
|
||||
// back through mesh-config and compares them with what the layers lay today; only when they agree does it empty the
|
||||
// layers it moved (each kept whole in its history first) and turn on the switch (inventory.FinishConfigMove), after
|
||||
// which the controller reads mesh-config where it read the layers. A module still in the settings form keeps its
|
||||
// layers and is said so. Anything that cannot move as it composes refuses the whole move, by name, before anything
|
||||
// is set. Asked again, it sets nothing that is set already.
|
||||
|
||||
// configLevel is where mesh-config keeps a value: mesh config, node config, or a module's own value on a node.
|
||||
type configLevel struct {
|
||||
Level string `json:"level"`
|
||||
Node string `json:"node,omitempty"`
|
||||
Module string `json:"module,omitempty"`
|
||||
}
|
||||
|
||||
func (l configLevel) valueKey(key string) string {
|
||||
switch l.Level {
|
||||
case "node":
|
||||
return "node." + l.Node + "." + key
|
||||
case "module":
|
||||
return "module." + l.Node + "." + l.Module + "." + key
|
||||
}
|
||||
return "mesh." + key
|
||||
}
|
||||
|
||||
func (l configLevel) String() string {
|
||||
switch l.Level {
|
||||
case "node":
|
||||
return "node config on " + l.Node
|
||||
case "module":
|
||||
return l.Module + "'s own value on " + l.Node
|
||||
}
|
||||
return "mesh config"
|
||||
}
|
||||
|
||||
// configSet is one value the move sets in mesh-config, and the layers it comes from.
|
||||
type configSet struct {
|
||||
configLevel
|
||||
Key string `json:"key"`
|
||||
Value any `json:"value"`
|
||||
Trusted bool `json:"trusted,omitempty"`
|
||||
// From is the layers it comes from, as `<module> on <where>: <key as the layer says it>`.
|
||||
From []string `json:"from"`
|
||||
// Already is true when mesh-config holds this value at this level already: nothing is set.
|
||||
Already bool `json:"already,omitempty"`
|
||||
}
|
||||
|
||||
// configMoveAnswer is what the move of config says it did, or would do.
|
||||
type configMoveAnswer struct {
|
||||
// DryRun is true when nothing was set: asked without --confirm.
|
||||
DryRun bool `json:"dry-run"`
|
||||
Sets []configSet `json:"sets"`
|
||||
// Moved is each layer whose values the move sets, emptied once they read back.
|
||||
Moved []string `json:"moved"`
|
||||
// Stays is each layer of a module still in the settings form: kept, and composed from as before.
|
||||
Stays []string `json:"stays,omitempty"`
|
||||
// Unread is each layer that composes nothing (its module is not assigned where it says): kept in the history,
|
||||
// emptied, and set nowhere.
|
||||
Unread []string `json:"unread,omitempty"`
|
||||
// Problems is why the move refuses, each by name; nothing is set while there is one.
|
||||
Problems []string `json:"problems,omitempty"`
|
||||
// Switched is true once the controller reads mesh-config where it read the layers.
|
||||
Switched bool `json:"switched"`
|
||||
}
|
||||
|
||||
// configMovePlan is the move worked out from the layers, before anything is asked of mesh-config.
|
||||
type configMovePlan struct {
|
||||
answer configMoveAnswer
|
||||
// moved is the layers the move empties once the values read back.
|
||||
moved []inventory.LayerToMove
|
||||
// expected is, for each module the move reaches, keyed `<node>.<module>`, every value composition lays over
|
||||
// its files today: what mesh-config must resolve for it from mesh config, node config or its own value.
|
||||
expected map[string]map[string]any
|
||||
// modules is each module in the config form, by the nodes it is assigned to: whose resolved values are read back.
|
||||
modules map[string][]string
|
||||
}
|
||||
|
||||
func whereOf(node string) string {
|
||||
if node == "" {
|
||||
return "the whole mesh"
|
||||
}
|
||||
return node
|
||||
}
|
||||
|
||||
// planConfigMove works out the move from every layer that holds a value, the catalogue, and where each module is
|
||||
// assigned.
|
||||
func planConfigMove(layers []inventory.LayerToMove, shelf map[string]catalogue.Manifest,
|
||||
on map[string][]string) configMovePlan {
|
||||
p := configMovePlan{answer: configMoveAnswer{Sets: []configSet{}, Moved: []string{}},
|
||||
expected: map[string]map[string]any{}, modules: map[string][]string{}}
|
||||
problem := func(format string, a ...any) {
|
||||
p.answer.Problems = append(p.answer.Problems, fmt.Sprintf(format, a...))
|
||||
}
|
||||
for name, m := range shelf {
|
||||
if m.DeclaresConfig() {
|
||||
p.modules[name] = on[name]
|
||||
}
|
||||
}
|
||||
|
||||
byModule := map[string]*layersOfModule{}
|
||||
var names []string
|
||||
for _, l := range layers {
|
||||
said := fmt.Sprintf("%s on %s", l.Module, whereOf(l.Node))
|
||||
m, known := shelf[l.Module]
|
||||
if !known || !m.DeclaresConfig() {
|
||||
p.answer.Stays = append(p.answer.Stays, said+" (its module is in the settings form)")
|
||||
continue
|
||||
}
|
||||
if _, placement := catalogue.SplitPlacement(l.Values); len(placement) > 0 {
|
||||
problem("%s still holds placement (%s): run move-placement first", said, strings.Join(movedKeys(placement), ", "))
|
||||
continue
|
||||
}
|
||||
assigned := on[l.Module]
|
||||
if (l.Node == "" && len(assigned) == 0) || (l.Node != "" && !containsString(assigned, l.Node)) {
|
||||
p.answer.Unread = append(p.answer.Unread, said+" (the module is not assigned there, so it composes nothing)")
|
||||
p.moved = append(p.moved, l)
|
||||
continue
|
||||
}
|
||||
// The layer's keys as config keys: a key the file names otherwise is set under its config key.
|
||||
as := catalogue.TakenAs(m)
|
||||
values := map[string]any{}
|
||||
from := map[string]string{}
|
||||
for _, k := range movedKeys(l.Values) {
|
||||
key := k
|
||||
if _, declared := m.Settings[k]; !declared {
|
||||
if renamed, ok := as[k]; ok {
|
||||
key = renamed
|
||||
} else {
|
||||
problem("%s says %q, which %s does not declare: declare it, or clear it from the layer", said, k, l.Module)
|
||||
continue
|
||||
}
|
||||
}
|
||||
if _, both := values[key]; both {
|
||||
// The layer says the key and the file's name for it: the config key wins, as composition has it (issue 751).
|
||||
if key != k {
|
||||
continue
|
||||
}
|
||||
}
|
||||
values[key] = l.Values[k]
|
||||
from[key] = said + ": " + k
|
||||
}
|
||||
om := byModule[l.Module]
|
||||
if om == nil {
|
||||
om = &layersOfModule{nodes: map[string]map[string]any{}, saidAt: map[string]map[string]string{}}
|
||||
byModule[l.Module] = om
|
||||
names = append(names, l.Module)
|
||||
}
|
||||
om.saidAt[l.Node] = from
|
||||
if l.Node == "" {
|
||||
om.mesh = values
|
||||
} else {
|
||||
om.nodes[l.Node] = values
|
||||
}
|
||||
p.moved = append(p.moved, l)
|
||||
p.answer.Moved = append(p.answer.Moved, said)
|
||||
}
|
||||
sort.Strings(names)
|
||||
|
||||
// Which predefined key each module falls back to, so a whole-mesh value goes to mesh config only where no other
|
||||
// module's value moves with it.
|
||||
fallingBack := map[string][]string{}
|
||||
for name, m := range shelf {
|
||||
for _, d := range m.Settings {
|
||||
if d.Predefined != "" && !containsString(fallingBack[d.Predefined], name) {
|
||||
fallingBack[d.Predefined] = append(fallingBack[d.Predefined], name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, name := range names {
|
||||
m, om := shelf[name], byModule[name]
|
||||
asked := catalogue.AskedKeys(m)
|
||||
keys := map[string]bool{}
|
||||
for k := range om.mesh {
|
||||
keys[k] = true
|
||||
}
|
||||
for _, values := range om.nodes {
|
||||
for k := range values {
|
||||
keys[k] = true
|
||||
}
|
||||
}
|
||||
for _, key := range movedKeys(keys) {
|
||||
d := m.Settings[key]
|
||||
meshValue, meshSays := om.mesh[key]
|
||||
alone := d.Predefined != "" && len(fallingBack[d.Predefined]) == 1
|
||||
if meshSays && alone {
|
||||
p.set(configLevel{Level: "mesh"}, d.Predefined, meshValue, d, om.said("", key))
|
||||
}
|
||||
for _, node := range on[name] {
|
||||
nodeValue, nodeSays := om.nodes[node][key]
|
||||
value := meshValue
|
||||
switch {
|
||||
case nodeSays && meshSays:
|
||||
_, under := meshValue.(map[string]any)
|
||||
_, over := nodeValue.(map[string]any)
|
||||
if under && over && asked[key] {
|
||||
problem("%s on %s lays an object for %q over the whole mesh's, and a placeholder takes it whole "+
|
||||
"where its file merges the two: say the value on %s whole, then move", name, node, key, node)
|
||||
continue
|
||||
}
|
||||
value = catalogue.LaidOver(meshValue, nodeValue)
|
||||
case nodeSays:
|
||||
value = nodeValue
|
||||
case !meshSays:
|
||||
continue
|
||||
}
|
||||
p.expect(node, name, key, value)
|
||||
if nodeSays || !alone {
|
||||
saidBy := om.said(node, key)
|
||||
if meshSays {
|
||||
saidBy = append(om.said("", key), saidBy...)
|
||||
}
|
||||
p.set(configLevel{Level: "module", Node: node, Module: name}, key, value, d, saidBy)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(p.answer.Problems)
|
||||
return p
|
||||
}
|
||||
|
||||
// layersOfModule is one module's layers as the move reads them: its whole-mesh layer's config values, each node's,
|
||||
// and where each key was said, by node ("" for the whole mesh).
|
||||
type layersOfModule struct {
|
||||
mesh map[string]any
|
||||
nodes map[string]map[string]any
|
||||
saidAt map[string]map[string]string
|
||||
}
|
||||
|
||||
func (om *layersOfModule) said(node, key string) []string {
|
||||
if s, ok := om.saidAt[node][key]; ok {
|
||||
return []string{s}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// set is one value the move sets, judged against its declaration as mesh-config will judge it.
|
||||
func (p *configMovePlan) set(level configLevel, key string, value any, d catalogue.SettingDeclaration, from []string) {
|
||||
p.answer.Sets = append(p.answer.Sets, configSet{configLevel: level, Key: key, Value: value, Trusted: d.Trusted,
|
||||
From: from})
|
||||
for _, wrong := range catalogue.ValueProblems(d, value) {
|
||||
p.answer.Problems = append(p.answer.Problems, fmt.Sprintf("%s at %s: %s", key, level, wrong))
|
||||
}
|
||||
}
|
||||
|
||||
func (p *configMovePlan) expect(node, module, key string, value any) {
|
||||
k := node + "." + module
|
||||
if p.expected[k] == nil {
|
||||
p.expected[k] = map[string]any{}
|
||||
}
|
||||
p.expected[k][key] = value
|
||||
}
|
||||
|
||||
func movedKeys[V any](m map[string]V) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// configService is mesh-config as the move asks it: what it holds at each level, what it resolved, and a value set
|
||||
// as the controller carries it from its terminal.
|
||||
type configService interface {
|
||||
// Values is every value set, by its key in mesh-config_values (`mesh.<key>`, `node.<node>.<key>`,
|
||||
// `module.<node>.<module>.<key>`).
|
||||
Values(ctx context.Context) (map[string]json.RawMessage, error)
|
||||
// Resolved is every resolved value.
|
||||
Resolved(ctx context.Context) ([]resolvedEntry, error)
|
||||
// Set sets one value at its level, by whom and why.
|
||||
Set(ctx context.Context, level configLevel, key string, value any, by, why string) error
|
||||
}
|
||||
|
||||
// runConfigMove is the move against a store and mesh-config: a dry run says what it would set.
|
||||
func runConfigMove(ctx context.Context, inv *inventory.Inventory, config configService, apply bool, by,
|
||||
why string) (configMoveAnswer, error) {
|
||||
layers, holding, err := inv.LayersToMove(ctx)
|
||||
if err != nil {
|
||||
return configMoveAnswer{}, err
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return configMoveAnswer{}, err
|
||||
}
|
||||
shelf := map[string]catalogue.Manifest{}
|
||||
on := map[string][]string{}
|
||||
for _, e := range entries {
|
||||
shelf[e.Manifest.Module] = e.Manifest
|
||||
on[e.Manifest.Module] = e.On
|
||||
}
|
||||
p := planConfigMove(layers, shelf, on)
|
||||
if len(holding) > 0 && len(p.answer.Problems) == 0 {
|
||||
p.answer.Problems = append(p.answer.Problems, "a layer still holds placement ("+strings.Join(holding, "; ")+
|
||||
"): run move-placement first")
|
||||
}
|
||||
moved, err := inv.ConfigMoved(ctx)
|
||||
if err != nil {
|
||||
return configMoveAnswer{}, err
|
||||
}
|
||||
p.answer.Switched = moved.Moved
|
||||
if config == nil {
|
||||
return p.answer, errors.New("mesh-config cannot be asked, so nothing was moved")
|
||||
}
|
||||
held, err := config.Values(ctx)
|
||||
if err != nil {
|
||||
return p.answer, fmt.Errorf("what mesh-config holds cannot be read, so nothing was moved: %w", err)
|
||||
}
|
||||
for i := range p.answer.Sets {
|
||||
s := &p.answer.Sets[i]
|
||||
if was, ok := held[s.valueKey(s.Key)]; ok && sameJSON(was, s.Value) {
|
||||
s.Already = true
|
||||
}
|
||||
}
|
||||
p.answer.DryRun = !apply
|
||||
if !apply || len(p.answer.Problems) > 0 {
|
||||
if apply {
|
||||
return p.answer, fmt.Errorf("the move of config refuses, and nothing was set: %s",
|
||||
strings.Join(p.answer.Problems, "; "))
|
||||
}
|
||||
return p.answer, nil
|
||||
}
|
||||
if len(p.moved) == 0 && moved.Moved {
|
||||
return p.answer, nil
|
||||
}
|
||||
for _, s := range p.answer.Sets {
|
||||
if s.Already {
|
||||
continue
|
||||
}
|
||||
said := "moved from the settings layer of " + strings.Join(s.From, " and ") + " (issue 664): " + why
|
||||
if err := config.Set(ctx, s.configLevel, s.Key, s.Value, by, said); err != nil {
|
||||
return p.answer, fmt.Errorf("%s at %s was not set, and no layer was emptied; ask the move again: %w",
|
||||
s.Key, s.configLevel, err)
|
||||
}
|
||||
}
|
||||
// Read back through mesh-config before anything points at it (design 57): every module the move reaches, and
|
||||
// every other in the config form, must resolve exactly what its layers lay today.
|
||||
if differs, err := readsBack(ctx, config, p); err != nil {
|
||||
return p.answer, err
|
||||
} else if len(differs) > 0 {
|
||||
return p.answer, fmt.Errorf("mesh-config does not resolve what the layers lay today, so no layer was emptied and "+
|
||||
"the controller still composes from the layers: %s", strings.Join(differs, "; "))
|
||||
}
|
||||
if err := inv.FinishConfigMove(ctx, p.moved, by, why); err != nil {
|
||||
return p.answer, err
|
||||
}
|
||||
p.answer.Switched = true
|
||||
return p.answer, nil
|
||||
}
|
||||
|
||||
// readsBack compares what mesh-config resolved with what the layers lay, for every module in the config form on
|
||||
// every node it is assigned to, and says each difference. A value the layers do not lay and mesh-config resolves from
|
||||
// a level would be laid once the switch is on: a difference too.
|
||||
func readsBack(ctx context.Context, config configService, p configMovePlan) ([]string, error) {
|
||||
var resolved []resolvedEntry
|
||||
var err error
|
||||
// mesh-config writes the resolved values with each set; a pass that missed one is put right by its next
|
||||
// resolution, so a difference is read again for a little while before it is said.
|
||||
for wait := 0; ; wait++ {
|
||||
if resolved, err = config.Resolved(ctx); err != nil {
|
||||
return nil, fmt.Errorf("mesh-config's resolved values cannot be read back, so no layer was emptied: %w", err)
|
||||
}
|
||||
differs, err := differences(resolved, p)
|
||||
if err != nil || len(differs) == 0 || wait >= readBackTries {
|
||||
return differs, err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(readBackEvery):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// How often, and how many times, the move reads mesh-config's resolved values back before it says they differ.
|
||||
var (
|
||||
readBackEvery = 2 * time.Second
|
||||
readBackTries = 15
|
||||
)
|
||||
|
||||
func differences(resolved []resolvedEntry, p configMovePlan) ([]string, error) {
|
||||
var differs []string
|
||||
for _, module := range movedKeys(p.modules) {
|
||||
for _, node := range p.modules[module] {
|
||||
laid, err := configLaid(resolved, node, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
want := p.expected[node+"."+module]
|
||||
if want == nil {
|
||||
want = map[string]any{}
|
||||
}
|
||||
for _, k := range movedKeys(mergedKeys(laid, want)) {
|
||||
got, has := laid[k]
|
||||
w, wants := want[k]
|
||||
switch {
|
||||
case !wants:
|
||||
differs = append(differs, fmt.Sprintf("%s on %s resolves %s from a level no layer says; clear it "+
|
||||
"there, or set it in a layer first", module, node, k))
|
||||
case !has:
|
||||
differs = append(differs, fmt.Sprintf("%s on %s does not resolve %s from the value moved", module, node, k))
|
||||
case !reflect.DeepEqual(normalJSON(got), normalJSON(w)):
|
||||
differs = append(differs, fmt.Sprintf("%s on %s resolves %s as %s, and its layers lay %s", module,
|
||||
node, k, compactJSON(got), compactJSON(w)))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return differs, nil
|
||||
}
|
||||
|
||||
func mergedKeys(a, b map[string]any) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for k := range a {
|
||||
out[k] = true
|
||||
}
|
||||
for k := range b {
|
||||
out[k] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func normalJSON(v any) any {
|
||||
raw, _ := json.Marshal(v)
|
||||
var out any
|
||||
_ = json.Unmarshal(raw, &out)
|
||||
return out
|
||||
}
|
||||
|
||||
func compactJSON(v any) string {
|
||||
raw, _ := json.Marshal(v)
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
func sameJSON(raw json.RawMessage, v any) bool {
|
||||
var held any
|
||||
if err := json.Unmarshal(raw, &held); err != nil {
|
||||
return false
|
||||
}
|
||||
return reflect.DeepEqual(held, normalJSON(v))
|
||||
}
|
||||
|
||||
// busConfig is mesh-config on this controller's bus: its buckets read directly, a value set through its verb in
|
||||
// the controller's own name, so mesh-config takes a trusted value as given at the controller's terminal (ADR 0348 §8).
|
||||
type busConfig struct{}
|
||||
|
||||
func (busConfig) Values(ctx context.Context) (map[string]json.RawMessage, error) {
|
||||
out := map[string]json.RawMessage{}
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
|
||||
defer cancel()
|
||||
kv, err := js.KeyValue(ctx, broker.BucketName("mesh-config", "values"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w, err := kv.WatchAll(ctx, jetstream.IgnoreDeletes())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = w.Stop() }()
|
||||
for e := range w.Updates() {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
var v struct {
|
||||
Value json.RawMessage `json:"value"`
|
||||
}
|
||||
if err := json.Unmarshal(e.Value(), &v); err != nil {
|
||||
return fmt.Errorf("mesh-config holds %q, which is not a value: %w", e.Key(), err)
|
||||
}
|
||||
out[e.Key()] = v.Value
|
||||
}
|
||||
return errors.New("mesh-config's values stopped answering")
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
func (busConfig) Resolved(ctx context.Context) ([]resolvedEntry, error) { return readResolved(ctx) }
|
||||
|
||||
func (busConfig) Set(ctx context.Context, level configLevel, key string, value any, by, why string) error {
|
||||
args := map[string]any{"action": "set", "key": key, "value": value, "why": why, "by": by}
|
||||
if level.Node != "" {
|
||||
args["node"] = level.Node
|
||||
}
|
||||
if level.Module != "" {
|
||||
args["module"] = level.Module
|
||||
}
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
answer, err := link.AskMeshSeatToolAs(ctx, conn, broker.ControllerName, "mesh-config", "config", args, 30*time.Second)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return errors.New(answer.Error)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// moveConfigCommand is `move-config`: the move of config (issue 664).
|
||||
func moveConfigCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("move-config", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "answer as JSON")
|
||||
confirm := set.Bool("confirm", false, "set the values and switch, rather than only say what would move")
|
||||
f := addHandActFlags(set)
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 0 {
|
||||
return errors.New("move-config [--json] [--confirm --why <text>]")
|
||||
}
|
||||
if !*confirm && f.given() {
|
||||
return errors.New("move-config takes --why only with --confirm: without it nothing is moved. Nothing was done")
|
||||
}
|
||||
if *confirm {
|
||||
if err := f.require("move-config"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
if *confirm {
|
||||
f.record(ctx, "move-config", nil)
|
||||
}
|
||||
why := ""
|
||||
if *confirm {
|
||||
why = strings.TrimSpace(*f.why)
|
||||
}
|
||||
answer, err := runConfigMove(ctx, open.inventory, busConfig{}, *confirm, "the move of config, run by "+link.Caller(), why)
|
||||
if *asJSON {
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
if encErr := encoder.Encode(answer); encErr != nil {
|
||||
return encErr
|
||||
}
|
||||
return err
|
||||
}
|
||||
fmt.Print(describeConfigMove(answer))
|
||||
return err
|
||||
}
|
||||
|
||||
// describeConfigMove says the answer as a person reads it.
|
||||
func describeConfigMove(a configMoveAnswer) string {
|
||||
var b strings.Builder
|
||||
verb := "set"
|
||||
if a.DryRun {
|
||||
verb = "would set"
|
||||
}
|
||||
for _, s := range a.Sets {
|
||||
state := verb
|
||||
if s.Already {
|
||||
state = "holds already"
|
||||
}
|
||||
fmt.Fprintf(&b, "%s %s = %s at %s (from %s)\n", state, s.Key, compactJSON(s.Value), s.configLevel,
|
||||
strings.Join(s.From, " and "))
|
||||
}
|
||||
for _, s := range a.Unread {
|
||||
fmt.Fprintf(&b, "emptied, set nowhere: %s\n", s)
|
||||
}
|
||||
for _, s := range a.Stays {
|
||||
fmt.Fprintf(&b, "stays: %s\n", s)
|
||||
}
|
||||
for _, s := range a.Problems {
|
||||
fmt.Fprintf(&b, "refused: %s\n", s)
|
||||
}
|
||||
switch {
|
||||
case len(a.Problems) > 0:
|
||||
b.WriteString("nothing was set: the move refuses until each is put right\n")
|
||||
case a.DryRun:
|
||||
fmt.Fprintf(&b, "%d layer(s), %d value(s); nothing was set — with --confirm --why <text> they are\n",
|
||||
len(a.Moved), len(a.Sets))
|
||||
case a.Switched:
|
||||
fmt.Fprintf(&b, "%d layer(s) moved into mesh-config, each kept whole in its history first; the controller "+
|
||||
"reads mesh-config where it read them\n", len(a.Moved))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,785 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
snapshot "github.com/novox/mesh-controller/internal/facts"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// configuredMesh is every live settings layer (issue 664) — read with the settings verb on 2026-10-15, on the four
|
||||
// machines as anchor, laptop, workstation and desktop, the whole mesh's with no node; personal values stood in for,
|
||||
// and the placement words left out, which move-placement moves first — with each module in the config form shaped
|
||||
// as its definition on the feature branch: its config, its files (templates inlined) and directories, placeholders
|
||||
// of kinds this test's mesh does not hold stood in for. A module still in the settings form (the media catalogue's,
|
||||
// and kernel) stands in as a mergeable file that takes every key. `on` is where each is assigned, as live.
|
||||
type configuredMesh struct {
|
||||
Modules []json.RawMessage `json:"modules"`
|
||||
Layers []configuredLayer `json:"layers"`
|
||||
On map[string][]string `json:"on"`
|
||||
// Predefined is the catalogue's list of predefined keys, which a module falls back to.
|
||||
Predefined json.RawMessage `json:"predefined"`
|
||||
}
|
||||
|
||||
type configuredLayer struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Values map[string]any `json:"values"`
|
||||
}
|
||||
|
||||
var fourMachines = []string{"anchor", "laptop", "workstation", "desktop"}
|
||||
|
||||
// aConfiguredMesh is aMesh with two machines more, every module of the configured mesh registered and assigned, and
|
||||
// its layers held as the store holds them once move-placement has run.
|
||||
func aConfiguredMesh(t *testing.T) (*stores, configuredMesh, map[string]map[string]any) {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
for i, name := range []string{"workstation", "desktop"} {
|
||||
record, err := inv.AddNode(ctx, name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.SetPlace(ctx, name, name+".example:51820", "here", false, fmt.Sprintf("10.77.0.%d", i+3)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordProfile(ctx, record.ID, map[string]any{"capabilities": []any{
|
||||
map[string]any{"name": "container-runtime", "present": true},
|
||||
map[string]any{"name": "systemd", "present": true}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, name, overlay.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "config-moved", "mesh.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var w configuredMesh
|
||||
if err := json.Unmarshal(raw, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
list, err := catalogue.ParsePredefined(w.Predefined)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
catalogue.UsePredefined(list)
|
||||
served := map[string]map[string]any{}
|
||||
for _, text := range w.Modules {
|
||||
var shape map[string]json.RawMessage
|
||||
if err := json.Unmarshal(text, &shape); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var name string
|
||||
_ = json.Unmarshal(shape["module"], &name)
|
||||
if s, ok := shape["serves-checked"]; ok {
|
||||
var serves map[string]any
|
||||
if err := json.Unmarshal(s, &serves); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
served[name] = serves
|
||||
delete(shape, "serves-checked")
|
||||
}
|
||||
text, err := json.Marshal(shape)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(text)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
register(t, open, m)
|
||||
}
|
||||
for _, l := range w.Layers {
|
||||
values, err := json.Marshal(l.Values)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if l.Node == "" {
|
||||
inv.ExecForTest(t, `insert into settings (node, module, values) values (null, $1, $2)`, l.Module, string(values))
|
||||
continue
|
||||
}
|
||||
inv.ExecForTest(t, `insert into settings (node, module, values) select id, $2, $3 from node where name = $1`,
|
||||
l.Node, l.Module, string(values))
|
||||
}
|
||||
for _, module := range movedKeys(w.On) {
|
||||
for _, node := range w.On[module] {
|
||||
if _, err := assign(ctx, open, node, module); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", node, module, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return open, w, served
|
||||
}
|
||||
|
||||
// standInMeshConfig is mesh-config as this test holds it: each value at its level, and resolved from the catalogue
|
||||
// the store holds as mesh-config resolves (novox/hq ADR 0348 §2): the module's own value on the node, else node
|
||||
// config then mesh config for the predefined key it falls back to, else its default; a required value nothing sets is
|
||||
// unset.
|
||||
type standInMeshConfig struct {
|
||||
open *stores
|
||||
values map[string]any
|
||||
sets []string
|
||||
// asked is each call of its verb: the action and the level's key.
|
||||
asked []string
|
||||
}
|
||||
|
||||
func (c *standInMeshConfig) Values(context.Context) (map[string]json.RawMessage, error) {
|
||||
out := map[string]json.RawMessage{}
|
||||
for k, v := range c.values {
|
||||
raw, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[k] = raw
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *standInMeshConfig) Set(_ context.Context, level configLevel, key string, value any, by, why string) error {
|
||||
if by == "" || why == "" {
|
||||
return fmt.Errorf("%s at %s is set without who or why", key, level)
|
||||
}
|
||||
c.values[level.valueKey(key)] = normalJSON(value)
|
||||
c.sets = append(c.sets, level.valueKey(key))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *standInMeshConfig) Resolved(ctx context.Context) ([]resolvedEntry, error) {
|
||||
entries, err := c.open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []resolvedEntry
|
||||
for _, e := range entries {
|
||||
m := e.Manifest
|
||||
if !m.DeclaresConfig() {
|
||||
continue
|
||||
}
|
||||
for _, node := range e.On {
|
||||
for _, key := range movedKeys(m.Settings) {
|
||||
d := m.Settings[key]
|
||||
r := resolvedEntry{Node: node, Module: m.Module, Key: key, Source: "default"}
|
||||
value, has := c.values["module."+node+"."+m.Module+"."+key]
|
||||
switch {
|
||||
case has:
|
||||
r.Source = "module"
|
||||
case d.Predefined != "" && c.values["node."+node+"."+d.Predefined] != nil:
|
||||
value, r.Source = c.values["node."+node+"."+d.Predefined], "node"
|
||||
case d.Predefined != "" && c.values["mesh."+d.Predefined] != nil:
|
||||
value, r.Source = c.values["mesh."+d.Predefined], "mesh"
|
||||
case d.Required:
|
||||
r.Source = "unset"
|
||||
default:
|
||||
value = d.Default
|
||||
}
|
||||
if r.Source != "unset" {
|
||||
if r.Value, err = json.Marshal(value); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// laid is the stand-in as composition reads it once the switch is on (inventory.ConfigValues).
|
||||
func (c *standInMeshConfig) laid(ctx context.Context, node, module string) (map[string]any, error) {
|
||||
resolved, err := c.Resolved(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return configLaid(resolved, node, module)
|
||||
}
|
||||
|
||||
// snapshotFacts is the facts snapshot as gatherFacts gives it.
|
||||
type snapshotFacts = snapshot.Facts
|
||||
|
||||
// toldEveryMachine is what each of the four machines is told — every resource and the filter's rules — and what
|
||||
// each module that serves something serves, settled from its layers, as JSON a person can diff.
|
||||
func toldEveryMachine(t *testing.T, open *stores, served map[string]map[string]any, on map[string][]string) map[string]string {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
told := map[string]string{}
|
||||
for _, node := range fourMachines {
|
||||
plan, settings, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", node, err)
|
||||
}
|
||||
with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Allocating)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", node, err)
|
||||
}
|
||||
composed, err := plan.Compose(with)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", node, err)
|
||||
}
|
||||
rules, err := plan.Rules(with)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", node, err)
|
||||
}
|
||||
settles := map[string]any{}
|
||||
for module, serves := range served {
|
||||
if !containsString(on[module], node) {
|
||||
continue
|
||||
}
|
||||
layers, err := open.inventory.SettingsFor(ctx, node, module)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, s := range serves {
|
||||
fact, _ := s.(map[string]any)
|
||||
settled, err := catalogue.Settle(fact, catalogue.WithDefaults(shelf[module], layers))
|
||||
if err != nil {
|
||||
settles[module+" serves "+name] = "error: " + err.Error()
|
||||
continue
|
||||
}
|
||||
settles[module+" serves "+name] = settled
|
||||
}
|
||||
}
|
||||
got, err := json.MarshalIndent(map[string]any{"resources": composed.Resources, "left-out": composed.LeftOut,
|
||||
"rules": rules, "served": settles}, "", " ")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
told[node] = string(got)
|
||||
}
|
||||
return told
|
||||
}
|
||||
|
||||
// effectiveEverywhere is each module's every value on each machine it is assigned to, as `settings show` and
|
||||
// `settings preferences` say them (catalogue.Effective): the value, never where it came from, which the move
|
||||
// changes by design; a key a layer says by its file's name is read as its config key.
|
||||
func effectiveEverywhere(t *testing.T, open *stores, w configuredMesh) map[string]any {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := map[string]any{}
|
||||
for _, module := range movedKeys(w.On) {
|
||||
if !shelf[module].DeclaresConfig() {
|
||||
continue
|
||||
}
|
||||
for _, node := range w.On[module] {
|
||||
layers, err := open.inventory.SettingsFor(ctx, node, module)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A layer's old spelling of a key (nodered's timeZone) is listed beside the key it is the file's name
|
||||
// for, and reaches the file as that key does (issue 751): read as that key, which the move sets.
|
||||
as := catalogue.TakenAs(shelf[module])
|
||||
for _, v := range catalogue.Effective(shelf[module], layers) {
|
||||
if key, old := as[v.Key]; old {
|
||||
out[node+"."+module+"."+key] = normalJSON(v.Value)
|
||||
continue
|
||||
}
|
||||
if _, said := out[node+"."+module+"."+v.Key]; said && v.FromDefault {
|
||||
continue
|
||||
}
|
||||
out[node+"."+module+"."+v.Key] = normalJSON(v.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func setAt(a configMoveAnswer, key string) []configSet {
|
||||
var out []configSet
|
||||
for _, s := range a.Sets {
|
||||
if s.valueKey(s.Key) == key {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// **The settings layers move into mesh-config and no machine is told anything new** (novox/hq ADR 0348 §9, design
|
||||
// 57 K5, issue 664). Every live layer is seeded as the store holds it once move-placement has run, on modules shaped
|
||||
// as their definitions, and every machine composed. The move, asked without confirming, says each value it would set
|
||||
// and where from, and sets nothing; confirmed, it sets them in mesh-config, reads them back, empties the layers it
|
||||
// moved and switches. Each machine is then told byte for byte what it was told before, every module serves the same,
|
||||
// and `settings` lists the same values `config` resolves. Asked again, it sets nothing.
|
||||
func TestEveryMachineIsToldTheSameAfterTheMoveOfConfig(t *testing.T) {
|
||||
open, w, served := aConfiguredMesh(t)
|
||||
ctx := t.Context()
|
||||
readBackTries = 0
|
||||
t.Cleanup(func() { readBackTries = 15 })
|
||||
inv := open.inventory
|
||||
config := &standInMeshConfig{open: open, values: map[string]any{}}
|
||||
inv.ReadsConfigFrom(config.laid)
|
||||
before := toldEveryMachine(t, open, served, w.On)
|
||||
listedBefore := effectiveEverywhere(t, open, w)
|
||||
factsBefore, err := gatherFacts(ctx, open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
asked, err := runConfigMove(ctx, inv, config, false, "the move of config, run by a test", "")
|
||||
if err != nil || !asked.DryRun || asked.Switched || len(asked.Problems) > 0 || len(config.sets) > 0 {
|
||||
t.Fatalf("the move, not confirmed, says %+v (%v) and set %v", asked, err, config.sets)
|
||||
}
|
||||
// nodered's live layer says the file's name for its time zone: set under the config key (issue 751).
|
||||
if s := setAt(asked, "module.anchor.nodered.time-zone"); len(s) != 1 || s[0].Value != "Europe/Brussels" ||
|
||||
!strings.HasSuffix(s[0].From[0], ": timeZone") {
|
||||
t.Errorf("nodered's time zone moves as %+v", s)
|
||||
}
|
||||
for _, s := range asked.Sets {
|
||||
if s.Key == "timeZone" {
|
||||
t.Errorf("a key is set under the file's name: %+v", s)
|
||||
}
|
||||
}
|
||||
// claude-code's whole-mesh managed settings become mesh config for agent-managed-settings (ADR 0348's
|
||||
// consequences), and its whole-mesh layer sets no module value.
|
||||
if s := setAt(asked, "mesh.agent-managed-settings"); len(s) != 1 || s[0].From[0] != "claude-code on the whole mesh: managed_settings" {
|
||||
t.Errorf("claude-code's managed settings move as %+v", s)
|
||||
}
|
||||
for _, node := range fourMachines {
|
||||
if s := setAt(asked, "module."+node+".claude-code.managed_settings"); len(s) > 0 {
|
||||
t.Errorf("claude-code's managed settings are also set on %s: %+v", node, s)
|
||||
}
|
||||
}
|
||||
// A whole-mesh layer of a value no predefined key holds goes to each machine it is assigned to, under that
|
||||
// machine's layer: power's on the workstation is the workstation's, its own elsewhere.
|
||||
if s := setAt(asked, "module.workstation.power.handle-power-key"); len(s) != 1 || s[0].Value != "suspend" {
|
||||
t.Errorf("power's key on the workstation moves as %+v", s)
|
||||
}
|
||||
if s := setAt(asked, "module.anchor.power.handle-power-key"); len(s) != 1 || s[0].Value != "poweroff" {
|
||||
t.Errorf("power's key on the anchor moves as %+v", s)
|
||||
}
|
||||
for _, s := range asked.Sets {
|
||||
if s.Level == "node" {
|
||||
t.Errorf("a machine's layer became node config: %+v", s)
|
||||
}
|
||||
}
|
||||
var stays []string
|
||||
for _, s := range asked.Stays {
|
||||
stays = append(stays, strings.Fields(s)[0])
|
||||
}
|
||||
sort.Strings(stays)
|
||||
if want := []string{"bazarr", "bazarr", "bookshelf", "jackett", "kernel", "kometa", "lidarr", "nzbget", "ombi",
|
||||
"plex", "qbittorrent", "radarr", "sonarr", "tautulli"}; len(stays) < 10 || !containsString(stays, "kernel") {
|
||||
t.Errorf("the layers that stay are %v, want those of %v", stays, want)
|
||||
}
|
||||
if after := toldEveryMachine(t, open, served, w.On); !reflect.DeepEqual(after, before) {
|
||||
t.Fatal("a dry run changed what a machine is told")
|
||||
}
|
||||
|
||||
moved, err := runConfigMove(ctx, inv, config, true, "the move of config, run by a test", "release 57 is delivered")
|
||||
if err != nil || moved.DryRun || !moved.Switched {
|
||||
t.Fatalf("the move says %+v (%v)", moved, err)
|
||||
}
|
||||
if len(config.sets) != len(asked.Sets) {
|
||||
t.Fatalf("the move set %d values, and said it would set %d", len(config.sets), len(asked.Sets))
|
||||
}
|
||||
if held := inv.QueryForTest(t, `select module from settings s where values <> '{}'
|
||||
and module in (select name from module where manifest ? 'config')`); len(held) != 0 {
|
||||
t.Fatalf("a layer of a module in the config form still holds values: %v", held)
|
||||
}
|
||||
after := toldEveryMachine(t, open, served, w.On)
|
||||
for _, node := range fourMachines {
|
||||
if after[node] != before[node] {
|
||||
t.Errorf("%s is told something else after the move:\n%s\nwas:\n%s", node, after[node], before[node])
|
||||
}
|
||||
}
|
||||
if listedAfter := effectiveEverywhere(t, open, w); !reflect.DeepEqual(listedAfter, listedBefore) {
|
||||
t.Errorf("settings lists other values after the move:\n%v\nwas:\n%v", listedAfter, listedBefore)
|
||||
}
|
||||
// And config lists what settings listed: each module's resolved value on each machine is the value settings
|
||||
// said before the move.
|
||||
resolved, err := config.Resolved(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range resolved {
|
||||
said, listed := listedBefore[r.Node+"."+r.Module+"."+r.Key]
|
||||
if r.Source == "unset" {
|
||||
if listed && said != nil {
|
||||
t.Errorf("%s of %s on %s is unset in config, and settings said %v", r.Key, r.Module, r.Node, said)
|
||||
}
|
||||
continue
|
||||
}
|
||||
var v any
|
||||
_ = json.Unmarshal(r.Value, &v)
|
||||
if !listed || !reflect.DeepEqual(v, said) {
|
||||
t.Errorf("config resolves %s of %s on %s as %v, and settings said %v", r.Key, r.Module, r.Node, v, said)
|
||||
}
|
||||
}
|
||||
|
||||
// The facts snapshot carries mesh-config's values for the merge gate, as each machine's layer of each module, and
|
||||
// each machine's declaration composes to the digest it did before the move.
|
||||
factsAfter, err := gatherFacts(ctx, open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Machines are named in a snapshot by stand-ins of its own; each is found by what it composes.
|
||||
digests := func(f snapshotFacts) ([]string, string) {
|
||||
var out []string
|
||||
zone := ""
|
||||
for _, m := range f.Machines {
|
||||
out = append(out, m.Declaration.Digest)
|
||||
for _, l := range m.Settings {
|
||||
if l.Module == "nodered" {
|
||||
zone = compactJSON(l.Values["time-zone"])
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, zone
|
||||
}
|
||||
was, _ := digests(factsBefore)
|
||||
now, zone := digests(factsAfter)
|
||||
if zone != `"Europe/Brussels"` {
|
||||
t.Errorf("the facts snapshot does not carry nodered's time zone from mesh-config: %s", zone)
|
||||
}
|
||||
if len(now) != len(fourMachines) || now[0] == "" || !reflect.DeepEqual(now, was) {
|
||||
t.Errorf("the declarations in the facts snapshot are %v after the move, %v before", now, was)
|
||||
}
|
||||
for _, s := range factsAfter.Settings {
|
||||
if _, isConfig := map[string]bool{"betterbird": true, "claude-code": true, "power": true, "xdg": true}[s.Module]; isConfig &&
|
||||
len(s.Values) > 0 {
|
||||
t.Errorf("the facts snapshot still carries the whole mesh's layer of %s: %v", s.Module, s.Values)
|
||||
}
|
||||
}
|
||||
|
||||
config.sets = nil
|
||||
again, err := runConfigMove(ctx, inv, config, true, "the move of config, run by a test", "again")
|
||||
if err != nil || len(config.sets) != 0 || len(again.Moved) != 0 {
|
||||
t.Fatalf("asked again, the move set %v and says %+v (%v)", config.sets, again, err)
|
||||
}
|
||||
if after := toldEveryMachine(t, open, served, w.On); !reflect.DeepEqual(after, before) {
|
||||
t.Fatal("the move asked again changed what a machine is told")
|
||||
}
|
||||
}
|
||||
|
||||
// **The move refuses what it cannot move as it composes, and sets nothing** (issue 664): a layer that still holds
|
||||
// placement (move-placement has not run), a key its module does not declare, and a value its declaration refuses —
|
||||
// each said by name; and a value mesh-config resolves from a level no layer says (it would be laid once the switch
|
||||
// is on) leaves every layer and the switch as they were.
|
||||
func TestTheMoveOfConfigRefusesWhatWouldChangeAMachine(t *testing.T) {
|
||||
open, _, _ := aConfiguredMesh(t)
|
||||
ctx := t.Context()
|
||||
readBackTries = 0
|
||||
t.Cleanup(func() { readBackTries = 15 })
|
||||
inv := open.inventory
|
||||
config := &standInMeshConfig{open: open, values: map[string]any{}}
|
||||
inv.ReadsConfigFrom(config.laid)
|
||||
|
||||
inv.ExecForTest(t, `update settings set values = values || '{"endpoints": {"web": {"reach": "public"}}}'
|
||||
where module = 'nodered'`)
|
||||
inv.ExecForTest(t, `update settings set values = values || '{"colour": "blue"}' where module = 'unifi'`)
|
||||
inv.ExecForTest(t, `update settings set values = '{"font-size": 900}' where module = 'dunst'`)
|
||||
answer, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered")
|
||||
if err == nil || answer.Switched || len(config.sets) > 0 {
|
||||
t.Fatalf("the move went ahead: %+v (%v), set %v", answer, err, config.sets)
|
||||
}
|
||||
for _, want := range []string{"nodered on anchor still holds placement (endpoints): run move-placement first",
|
||||
`unifi on anchor says "colour", which unifi does not declare`, "font-size at dunst's own value on workstation"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not say %q: %v", want, err)
|
||||
}
|
||||
}
|
||||
inv.ExecForTest(t, `update settings set values = values - 'endpoints' where module = 'nodered'`)
|
||||
inv.ExecForTest(t, `update settings set values = values - 'colour' where module = 'unifi'`)
|
||||
inv.ExecForTest(t, `update settings set values = '{"font-size": 13, "width": 500}' where module = 'dunst'`)
|
||||
|
||||
// Set in mesh-config before the move, on a module whose layers say nothing of it: the switch would lay it.
|
||||
config.values["module.anchor.grafana."+movedKeys(func() map[string]catalogue.SettingDeclaration {
|
||||
shelf, _ := inv.Catalogue(ctx)
|
||||
return shelf["grafana"].Settings
|
||||
}())[0]] = "set before the move"
|
||||
answer, err = runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered")
|
||||
if err == nil || !strings.Contains(err.Error(), "grafana on anchor resolves") || answer.Switched {
|
||||
t.Fatalf("a value no layer says was laid by the switch: %+v (%v)", answer, err)
|
||||
}
|
||||
if moved, _ := inv.ConfigMoved(ctx); moved.Moved {
|
||||
t.Fatal("the switch is on after a refused move")
|
||||
}
|
||||
if held := inv.QueryForTest(t, `select count(*) from settings where values <> '{}'`); held[0][0] == "0" {
|
||||
t.Fatal("a refused move emptied the layers")
|
||||
}
|
||||
}
|
||||
|
||||
// The verb's command line: a dry run unless confirmed, and a confirmed move says why, recorded as a person's step of
|
||||
// the release, never a repair (issue 664).
|
||||
func TestTheMoveOfConfigIsADryRunUnlessConfirmed(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
args map[string]any
|
||||
want string
|
||||
}{
|
||||
{map[string]any{}, "move-config --json"},
|
||||
{map[string]any{"confirm": "true", "why": "release 57 is delivered"}, "move-config --json --confirm --why release 57 is delivered"},
|
||||
} {
|
||||
argv, err := argvFor("move-config", c.args)
|
||||
if err != nil || strings.Join(argv, " ") != c.want {
|
||||
t.Errorf("%v: %q %v, want %q", c.args, argv, err, c.want)
|
||||
}
|
||||
}
|
||||
for _, args := range []map[string]any{{"confirm": "true"}, {"why": "tidy"}, {"node": "anchor"}} {
|
||||
if argv, err := argvFor("move-config", args); err == nil {
|
||||
t.Errorf("%v was composed as %q", args, argv)
|
||||
}
|
||||
}
|
||||
if repairingCommand([]string{"move-config", "--json", "--confirm", "--why", "x"}) != "move-config" {
|
||||
t.Error("a confirmed move through the generic verb would go unrecorded")
|
||||
}
|
||||
if !personsDecision(link.HandAct{Verb: "move-config"}) {
|
||||
t.Error("a move of the release would count toward a healer the mesh lacks")
|
||||
}
|
||||
}
|
||||
|
||||
// ask is mesh-config's verb as the stand-in answers it: list, set, clear and propose.
|
||||
func (c *standInMeshConfig) ask(ctx context.Context, args map[string]any) (json.RawMessage, error) {
|
||||
str := func(k string) string { s, _ := args[k].(string); return s }
|
||||
level := configLevel{Level: "mesh"}
|
||||
switch {
|
||||
case str("module") != "" && str("node") != "":
|
||||
level = configLevel{Level: "module", Node: str("node"), Module: str("module")}
|
||||
case str("node") != "":
|
||||
level = configLevel{Level: "node", Node: str("node")}
|
||||
}
|
||||
c.asked = append(c.asked, str("action")+" "+level.valueKey(str("key")))
|
||||
switch str("action") {
|
||||
case "list":
|
||||
resolved, err := c.Resolved(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]any{"resolved": []resolvedEntry{}, "set": []any{}}
|
||||
var listed []resolvedEntry
|
||||
for _, r := range resolved {
|
||||
if r.Module == str("module") && (str("node") == "" || r.Node == str("node")) {
|
||||
listed = append(listed, r)
|
||||
}
|
||||
}
|
||||
out["resolved"] = listed
|
||||
var set []any
|
||||
for _, k := range movedKeys(c.values) {
|
||||
parts := strings.SplitN(k, ".", 4)
|
||||
if len(parts) == 4 && parts[0] == "module" && parts[2] == str("module") {
|
||||
set = append(set, map[string]any{"level": configLevel{Level: "module", Node: parts[1], Module: parts[2]},
|
||||
"key": parts[3], "value": c.values[k], "by": "a test"})
|
||||
}
|
||||
}
|
||||
out["set"] = set
|
||||
return json.Marshal(out)
|
||||
case "set":
|
||||
if str("why") == "" || str("by") == "" {
|
||||
return nil, fmt.Errorf("a set without why or by: %v", args)
|
||||
}
|
||||
return nil, c.Set(ctx, level, str("key"), args["value"], str("by"), str("why"))
|
||||
case "clear":
|
||||
if _, had := c.values[level.valueKey(str("key"))]; !had {
|
||||
return nil, fmt.Errorf("nothing is set for %s", str("key"))
|
||||
}
|
||||
delete(c.values, level.valueKey(str("key")))
|
||||
return json.RawMessage(`{}`), nil
|
||||
case "propose":
|
||||
return json.RawMessage(`{"proposal":"p-1"}`), nil
|
||||
}
|
||||
return nil, fmt.Errorf("the stand-in does not %s", str("action"))
|
||||
}
|
||||
|
||||
// **`settings` answers by forwarding to `config`** (novox/hq ADR 0348 §1, design 57 K5, issue 664): once the move
|
||||
// has run, a module in the config form is shown from mesh-config, its values set and cleared there as its own on the
|
||||
// machine (placement still beside the assignment), and proposed through it; a trusted value is refused through a
|
||||
// verb, a module value for the whole mesh is refused, and a module still in the settings form keeps today's verb.
|
||||
func TestTheSettingsVerbForwardsToConfigOnceTheValuesMoved(t *testing.T) {
|
||||
open, w, served := aConfiguredMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
readBackTries = 0
|
||||
t.Cleanup(func() { readBackTries = 15 })
|
||||
config := &standInMeshConfig{open: open, values: map[string]any{}}
|
||||
inv.ReadsConfigFrom(config.laid)
|
||||
savedRead, savedAsk := readConfigValues, configAsk
|
||||
readConfigValues, configAsk = config.laid, config.ask
|
||||
t.Cleanup(func() { readConfigValues, configAsk = savedRead, savedAsk })
|
||||
if _, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
shown := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"show", "nodered", "--node", "anchor"}) })
|
||||
for _, want := range []string{`"time-zone": "Europe/Brussels"`, `time-zone = "Europe/Brussels" (module)`,
|
||||
"mesh-config.config", `"mqtt": {`} {
|
||||
if !strings.Contains(shown, want) {
|
||||
t.Errorf("settings show does not say %q:\n%s", want, shown)
|
||||
}
|
||||
}
|
||||
if strings.Contains(shown, "timeZone") {
|
||||
t.Errorf("settings show still says the file's name:\n%s", shown)
|
||||
}
|
||||
|
||||
config.asked = nil
|
||||
said := stdoutOf(t, func() error {
|
||||
return settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12, "width": 500}`, "--node", "workstation"})
|
||||
})
|
||||
if config.values["module.workstation.dunst.font-size"] != float64(12) || !strings.Contains(said, "~ font-size") ||
|
||||
containsString(config.asked, "set module.workstation.dunst.width") {
|
||||
t.Fatalf("settings set reached mesh-config as %v (%v):\n%s", config.values, config.asked, said)
|
||||
}
|
||||
told := toldEveryMachine(t, open, served, w.On)["workstation"]
|
||||
if !strings.Contains(told, "12") {
|
||||
t.Error("the workstation is not told the font size set through settings")
|
||||
}
|
||||
if err := settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12}`, "--node", "workstation"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "would no longer set width") {
|
||||
t.Fatalf("a set dropping a value without --replace: %v", err)
|
||||
}
|
||||
if err := settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12}`}); err == nil ||
|
||||
!strings.Contains(err.Error(), "no own value for the whole mesh") {
|
||||
t.Fatalf("a module value for the whole mesh: %v", err)
|
||||
}
|
||||
|
||||
t.Setenv(verbVar, "settings")
|
||||
for _, c := range []struct{ module, node, values, key, held, why string }{
|
||||
// A file root reads: refused as before the move (novox/hq issue 339).
|
||||
{"hostname", "anchor", `{"hostname": "elsewhere"}`, "module.anchor.hostname.hostname", "Ace", "issue 339"},
|
||||
{"messenger", "laptop", `{"desktop-machines": ["g14", "shanks"], "time-zone": "UTC"}`,
|
||||
"module.laptop.messenger.time-zone", "Europe/Brussels", "issue 339"},
|
||||
} {
|
||||
err := settingsCommand(ctx, []string{"set", c.module, c.values, "--node", c.node})
|
||||
if err == nil || !strings.Contains(err.Error(), c.why) || config.values[c.key] != c.held {
|
||||
t.Fatalf("a trusted value of %s through a verb: %v, holds %v", c.module, err, config.values[c.key])
|
||||
}
|
||||
}
|
||||
// And a trusted value no file makes the terminal's is refused as trusted, since mesh-config takes the controller's
|
||||
// word for it (ADR 0348 §8).
|
||||
if err := refuseTrustedThroughAVerb(ctx, inv, "messenger", "laptop", []string{"time-zone"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "ADR 0348 §8") {
|
||||
t.Fatalf("a trusted value through a verb: %v", err)
|
||||
}
|
||||
if err := refuseTrustedThroughAVerb(ctx, inv, "dunst", "workstation", []string{"width"}); err != nil {
|
||||
t.Fatalf("a value nobody trusts through a verb: %v", err)
|
||||
}
|
||||
stdoutOf(t, func() error {
|
||||
return settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12, "width": 600}`, "--node", "workstation"})
|
||||
})
|
||||
if config.values["module.workstation.dunst.width"] != float64(600) {
|
||||
t.Fatalf("a value nobody trusts was not set through a verb: %v", config.values["module.workstation.dunst.width"])
|
||||
}
|
||||
t.Setenv(verbVar, "")
|
||||
|
||||
stdoutOf(t, func() error { return settingsCommand(ctx, []string{"clear", "dunst", "--node", "workstation"}) })
|
||||
for _, k := range []string{"font-size", "width"} {
|
||||
if _, held := config.values["module.workstation.dunst."+k]; held {
|
||||
t.Errorf("settings clear left dunst's %s on the workstation", k)
|
||||
}
|
||||
}
|
||||
|
||||
config.asked = nil
|
||||
proposed := stdoutOf(t, func() error {
|
||||
return settingsCommand(ctx, []string{"propose", "hostname", `{"hostname": "elsewhere"}`, "--node", "anchor"})
|
||||
})
|
||||
if !containsString(config.asked, "propose module.anchor.hostname.hostname") || !strings.Contains(proposed, "p-1") {
|
||||
t.Fatalf("settings propose reached mesh-config as %v:\n%s", config.asked, proposed)
|
||||
}
|
||||
|
||||
config.asked = nil
|
||||
kept := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"show", "kernel", "--node", "laptop"}) })
|
||||
if !strings.Contains(kept, "transparent-huge-pages") || len(config.asked) > 0 {
|
||||
t.Fatalf("a module in the settings form is shown as %q, asking mesh-config %v", kept, config.asked)
|
||||
}
|
||||
}
|
||||
|
||||
// **An approval for a module whose config has moved is refused, and writes nothing** (issue 830, novox/hq ADR 0348):
|
||||
// a proposal raised while grafana read its layer (it had none, so the layer is as the operator was shown it) and
|
||||
// approved after the move would put its values back into the layer, ReadsConfig would turn false, and the module
|
||||
// would compose from emptied layers everywhere.
|
||||
func TestAnApprovalRaisedBeforeTheMoveOfConfigIsRefusedAfterIt(t *testing.T) {
|
||||
open, _, _ := aConfiguredMesh(t)
|
||||
ctx := t.Context()
|
||||
readBackTries = 0
|
||||
t.Cleanup(func() { readBackTries = 15 })
|
||||
inv := open.inventory
|
||||
config := &standInMeshConfig{open: open, values: map[string]any{}}
|
||||
inv.ReadsConfigFrom(config.laid)
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := movedKeys(func() map[string]catalogue.SettingDeclaration { return shelf["grafana"].Settings }())[0]
|
||||
values := map[string]any{key: "raised before the move"}
|
||||
p := settingsProposal{Module: "grafana", Node: "anchor", Values: values, From: "g14/claude-code", At: time.Now(),
|
||||
Digest: layerDigest(values), BeforeDigest: layerDigest(nil)}
|
||||
if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has {
|
||||
t.Fatalf("grafana has a layer on anchor before the test: %v %v", has, err)
|
||||
}
|
||||
if moved, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil || !moved.Switched {
|
||||
t.Fatalf("the move says %+v (%v)", moved, err)
|
||||
}
|
||||
if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads {
|
||||
t.Fatalf("grafana does not read config after the move: %v %v", reads, err)
|
||||
}
|
||||
_, err = setLayerIn(open)(ctx, p, "s30", "approved by the operator")
|
||||
if err == nil || !strings.Contains(err.Error(), "moved to mesh-config since the operator was shown the change") ||
|
||||
!strings.Contains(err.Error(), "propose it again") {
|
||||
t.Fatalf("the stale approval: %v", err)
|
||||
}
|
||||
if layer, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has && len(layer) > 0 {
|
||||
t.Errorf("the refused approval wrote the layer: %v %v %v", layer, has, err)
|
||||
}
|
||||
if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads {
|
||||
t.Errorf("the refused approval turned ReadsConfig off: %v %v", reads, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **A proposal of placement only is set after the move, as before** (issue 830): forwardPropose sends it down the
|
||||
// old path on purpose, so refusing it would loop the operator (refused, proposed again, refused).
|
||||
func TestAPlacementOnlyApprovalRaisedBeforeTheMoveIsSetAfterIt(t *testing.T) {
|
||||
open, _, _ := aConfiguredMesh(t)
|
||||
ctx := t.Context()
|
||||
readBackTries = 0
|
||||
t.Cleanup(func() { readBackTries = 15 })
|
||||
inv := open.inventory
|
||||
config := &standInMeshConfig{open: open, values: map[string]any{}}
|
||||
inv.ReadsConfigFrom(config.laid)
|
||||
values := map[string]any{catalogue.PlacesSetting: map[string]any{"data": "/srv/grafana-data"}}
|
||||
p := settingsProposal{Module: "grafana", Node: "anchor", Values: values, From: "g14/claude-code", At: time.Now(),
|
||||
Digest: layerDigest(values), BeforeDigest: layerDigest(nil)}
|
||||
if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has {
|
||||
t.Fatalf("grafana has a layer on anchor before the test: %v %v", has, err)
|
||||
}
|
||||
if moved, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil || !moved.Switched {
|
||||
t.Fatalf("the move says %+v (%v)", moved, err)
|
||||
}
|
||||
if _, err := setLayerIn(open)(ctx, p, "s31", "approved by the operator"); err != nil {
|
||||
t.Fatalf("the placement-only approval after the move: %v", err)
|
||||
}
|
||||
if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || !has {
|
||||
t.Errorf("the approval did not set the layer: %v %v", has, err)
|
||||
}
|
||||
if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads {
|
||||
t.Errorf("the approval turned ReadsConfig off: %v %v", reads, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **Where the controller reads config values once the move has run** (novox/hq ADR 0348 §9, design 57 K5, issue 664):
|
||||
// mesh-config_resolved, the bucket mesh-config alone writes, keyed `<node>.<module>.<key>`, each value with the
|
||||
// source it came from. Read only for a module the switch reaches (inventory.ReadsConfig), so a controller before the
|
||||
// move never asks the bus for config. The exception to ADR 0348 §9 ("the controller holds no config value") that
|
||||
// ADR 0349 clause 17's old way needs, until design 57's R1 hands every module to the provisioner.
|
||||
|
||||
// resolvedEntry is one key of mesh-config_resolved as the controller reads it: the value and where it came from.
|
||||
type resolvedEntry struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value,omitempty"`
|
||||
Source string `json:"source"`
|
||||
}
|
||||
|
||||
// The sources a resolved value is laid over a module's files from: mesh config, node config and the module's own
|
||||
// value. Its manifest's default is composition's already, and a taken key's default is never laid over its file;
|
||||
// a required value nothing sets (`unset`) is refused by composition by name, as it is today.
|
||||
var laidSources = map[string]bool{"mesh": true, "node": true, "module": true}
|
||||
|
||||
// configLaid is the values of one module on one node that are laid over its files, from the resolved entries.
|
||||
func configLaid(entries []resolvedEntry, node, module string) (map[string]any, error) {
|
||||
out := map[string]any{}
|
||||
for _, e := range entries {
|
||||
if e.Node != node || e.Module != module || !laidSources[e.Source] {
|
||||
continue
|
||||
}
|
||||
var v any
|
||||
if err := json.Unmarshal(e.Value, &v); err != nil {
|
||||
return nil, fmt.Errorf("mesh-config's value %s of %s on %s cannot be read: %w", e.Key, module, node, err)
|
||||
}
|
||||
out[e.Key] = v
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// resolvedRead is what one read of the whole bucket held, and when it was read.
|
||||
type resolvedRead struct {
|
||||
mu sync.Mutex
|
||||
at time.Time
|
||||
entries []resolvedEntry
|
||||
}
|
||||
|
||||
// resolvedFresh is how long one read of the bucket answers: a plan reads every module of a machine, and one read
|
||||
// serves them all; a value set since is read by the next composition.
|
||||
const resolvedFresh = 5 * time.Second
|
||||
|
||||
var theResolved resolvedRead
|
||||
|
||||
// readConfigValues is where an opened inventory reads config values once the switch is on; a test replaces it.
|
||||
var readConfigValues inventory.ConfigValues = busConfigValues
|
||||
|
||||
// busConfigValues reads one module's config values on one node from mesh-config_resolved (inventory.ConfigValues).
|
||||
func busConfigValues(ctx context.Context, node, module string) (map[string]any, error) {
|
||||
theResolved.mu.Lock()
|
||||
defer theResolved.mu.Unlock()
|
||||
if theResolved.entries == nil || time.Since(theResolved.at) > resolvedFresh {
|
||||
entries, err := readResolved(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
theResolved.entries, theResolved.at = entries, time.Now()
|
||||
}
|
||||
return configLaid(theResolved.entries, node, module)
|
||||
}
|
||||
|
||||
// readResolved is every entry of mesh-config_resolved, read on the bus.
|
||||
func readResolved(ctx context.Context) ([]resolvedEntry, error) {
|
||||
entries := []resolvedEntry{}
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
|
||||
defer cancel()
|
||||
kv, err := js.KeyValue(ctx, broker.ConfigResolvedBucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s cannot be opened: %w", broker.ConfigResolvedBucket, err)
|
||||
}
|
||||
w, err := kv.WatchAll(ctx, jetstream.IgnoreDeletes())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = w.Stop() }()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("%s was not read whole: %w", broker.ConfigResolvedBucket, ctx.Err())
|
||||
case e, open := <-w.Updates():
|
||||
if !open {
|
||||
return errors.New(broker.ConfigResolvedBucket + " stopped answering")
|
||||
}
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
var r resolvedEntry
|
||||
if err := json.Unmarshal(e.Value(), &r); err != nil {
|
||||
return fmt.Errorf("%s holds %q, which is not a resolved value: %w", broker.ConfigResolvedBucket,
|
||||
e.Key(), err)
|
||||
}
|
||||
if r.Node == "" || r.Module == "" || r.Key == "" || !strings.HasPrefix(e.Key(), r.Node+"."+r.Module+".") {
|
||||
return fmt.Errorf("%s holds %q, whose value names another key", broker.ConfigResolvedBucket, e.Key())
|
||||
}
|
||||
entries = append(entries, r)
|
||||
}
|
||||
}
|
||||
})
|
||||
return entries, err
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ecdh"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The core's own credentials, sealed by the vault (novox/hq ADR 0349 §16, issue 755; the lead's decision on issue 755,
|
||||
// notes 4 and 5).
|
||||
//
|
||||
// For a machine whose engine opens the vault's box (link.VaultBoxContract), each own secret of a module on the core's
|
||||
// list (catalogue.CoreCredentialModules) travels as the vault sealed it to that machine's key, which the controller
|
||||
// asks for on the send path (`get`, answered to the controller alone and sealed only to the key that machine's engine
|
||||
// stated: mesh-vault's core.go) and carries without being able to open it. A machine below that contract is sent the
|
||||
// controller's own sealed value, exactly as before.
|
||||
//
|
||||
// **The vault seals every answer afresh**, so the box last got is kept (inventory.CoreBox) and carried again while the
|
||||
// vault's generation and the machine's key are the same: a declaration that changed with every answer would never be
|
||||
// current. Every question but a send reads the kept box and asks nothing.
|
||||
//
|
||||
// **When the vault does not answer, the controller never falls back to its own seal.** A credential with a kept box
|
||||
// keeps it — the machine keeps the file it has — and one with none leaves its module out of the declaration (the
|
||||
// machine keeps what it holds of it, and a module not there yet is not placed). Either way the send raises a
|
||||
// condition naming the vault and the machine, cleared by the first send the vault answers whole.
|
||||
|
||||
// engineOpensVaultBox says whether a machine's node-engine opens the vault's box, by its own newest statement.
|
||||
func engineOpensVaultBox(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) {
|
||||
stated, had, err := inv.HealthOf(ctx, node)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return had && stated.Contract >= link.VaultBoxContract, nil
|
||||
}
|
||||
|
||||
// coreAnswer is the vault's `get` answer as the controller reads it: the name, the generation and the box, never
|
||||
// opened.
|
||||
type coreAnswer struct {
|
||||
Name string `json:"name"`
|
||||
Generation int `json:"generation"`
|
||||
Sealed json.RawMessage `json:"sealed"`
|
||||
}
|
||||
|
||||
// coreAsk asks the vault for one secret sealed to the given key.
|
||||
type coreAsk func(name, publicKey string) (coreAnswer, error)
|
||||
|
||||
// coreBoxes is what the vault's boxes make of one machine's composition.
|
||||
type coreBoxes struct {
|
||||
// sealed is each core credential's box, by module and name (catalogue.Rendering.VaultSealed).
|
||||
sealed map[string]map[string]json.RawMessage
|
||||
// notPlaced is each core module left out, with why (catalogue.Rendering.NotPlaced).
|
||||
notPlaced map[string]string
|
||||
// keep is each box newly answered, to keep.
|
||||
keep []inventory.CoreBox
|
||||
// held is each credential the vault did not answer for whose kept box is carried again; silent is why the vault
|
||||
// did not answer, per credential.
|
||||
held, silent []string
|
||||
}
|
||||
|
||||
// coreModulesOf is every module of the plan on the core's list whose own secrets this machine's declaration carries,
|
||||
// with their plain own secret names, sorted: one handed to the provisioner has its secrets from the vault through the
|
||||
// client, and one issued outside the mesh is never the vault's to seal here.
|
||||
func coreModulesOf(modules []catalogue.Manifest, handed func(catalogue.Manifest) bool) map[string][]string {
|
||||
out := map[string][]string{}
|
||||
for _, m := range modules {
|
||||
if !catalogue.CarriesVaultSealed(m.Module) || (handed != nil && handed(m)) {
|
||||
continue
|
||||
}
|
||||
var names []string
|
||||
for name, own := range m.OwnSecrets.Plain() {
|
||||
if own.IssuedBy != catalogue.IssuedOutside {
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
if len(names) > 0 {
|
||||
sort.Strings(names)
|
||||
out[m.Module] = names
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sealingKeyPEM is a machine's sealing key, recorded raw in base64, as the SPKI PEM the vault seals to.
|
||||
func sealingKeyPEM(raw string) (string, error) {
|
||||
b, err := base64.StdEncoding.DecodeString(strings.TrimSpace(raw))
|
||||
if err != nil || len(b) != 32 {
|
||||
return "", errors.New("the machine's sealing key is not a raw X25519 key")
|
||||
}
|
||||
pub, err := ecdh.X25519().NewPublicKey(b)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
der, err := x509.MarshalPKIXPublicKey(pub)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: der})), nil
|
||||
}
|
||||
|
||||
// coreBoxesOf decides each core credential of a machine: on a send, the vault's answer (the kept box while its
|
||||
// generation and key are the same), the kept box where the vault did not answer, or the module left out where nothing
|
||||
// was kept; on any other question, the kept box, or a stand-in the next send asks for. Pure: what to keep and what to
|
||||
// say is answered, not done.
|
||||
func coreBoxesOf(node, sealingKey string, core map[string][]string, kept map[string]map[string]inventory.CoreBox,
|
||||
choosing Choosing, ask coreAsk) coreBoxes {
|
||||
out := coreBoxes{sealed: map[string]map[string]json.RawMessage{}, notPlaced: map[string]string{}}
|
||||
modules := make([]string, 0, len(core))
|
||||
for m := range core {
|
||||
modules = append(modules, m)
|
||||
}
|
||||
sort.Strings(modules)
|
||||
for _, module := range modules {
|
||||
if sealingKey == "" {
|
||||
out.notPlaced[module] = fmt.Sprintf("%s has no sealing key, so the vault can seal %s's credentials to "+
|
||||
"nothing there (novox/hq ADR 0349 §16)", node, module)
|
||||
continue
|
||||
}
|
||||
key, err := sealingKeyPEM(sealingKey)
|
||||
if err != nil {
|
||||
out.notPlaced[module] = fmt.Sprintf("%s's credentials cannot be asked of the vault: %v", module, err)
|
||||
continue
|
||||
}
|
||||
boxes := map[string]json.RawMessage{}
|
||||
for _, name := range core[module] {
|
||||
k, has := kept[module][name]
|
||||
valid := has && k.SealedTo == sealingKey && len(k.Box) > 0
|
||||
if choosing != Allocating {
|
||||
if valid {
|
||||
boxes[name] = k.Box
|
||||
} else {
|
||||
boxes[name] = catalogue.ForeseenVaultBox
|
||||
}
|
||||
continue
|
||||
}
|
||||
secret := "own/" + node + "/" + module + "/" + name
|
||||
got, err := ask(secret, key)
|
||||
if err == nil && (got.Generation <= 0 || canonicalBox(got.Sealed) == nil) {
|
||||
err = errors.New("it answered no box")
|
||||
}
|
||||
if err != nil {
|
||||
out.silent = append(out.silent, fmt.Sprintf("%s: %v", secret, err))
|
||||
if valid {
|
||||
boxes[name] = k.Box
|
||||
out.held = append(out.held, module+"/"+name)
|
||||
continue
|
||||
}
|
||||
out.notPlaced[module] = fmt.Sprintf("the vault did not give %s's credential %s for %s (%v), and the "+
|
||||
"controller never seals it itself (novox/hq ADR 0349 §16): not placed until the vault answers",
|
||||
module, name, node, err)
|
||||
break
|
||||
}
|
||||
if valid && k.Generation == got.Generation {
|
||||
boxes[name] = k.Box
|
||||
continue
|
||||
}
|
||||
box := canonicalBox(got.Sealed)
|
||||
boxes[name] = box
|
||||
out.keep = append(out.keep, inventory.CoreBox{Module: module, Name: name, Generation: got.Generation,
|
||||
SealedTo: sealingKey, Box: box})
|
||||
}
|
||||
if _, left := out.notPlaced[module]; !left {
|
||||
out.sealed[module] = boxes
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// canonicalBox is the vault's box in the one form the declaration carries — the sealed box's fields, in their order,
|
||||
// and nothing else — or nil when it is not one.
|
||||
func canonicalBox(raw json.RawMessage) json.RawMessage {
|
||||
var b struct {
|
||||
V int `json:"v"`
|
||||
Eph string `json:"eph"`
|
||||
IV string `json:"iv"`
|
||||
Tag string `json:"tag"`
|
||||
Ct string `json:"ct"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &b); err != nil || b.V <= 0 || b.Eph == "" || b.Ct == "" {
|
||||
return nil
|
||||
}
|
||||
out, _ := json.Marshal(b)
|
||||
return out
|
||||
}
|
||||
|
||||
// coreVault is how a send asks the vault for a core credential: its `get`, in the controller's name. A test stands
|
||||
// in for it.
|
||||
var coreVault = func(ctx context.Context) coreAsk {
|
||||
vault := theVault(ctx)
|
||||
return func(name, publicKey string) (coreAnswer, error) {
|
||||
raw, err := vault("get", map[string]any{"name": name, "public_key": publicKey})
|
||||
if err != nil {
|
||||
return coreAnswer{}, err
|
||||
}
|
||||
var a coreAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return coreAnswer{}, fmt.Errorf("the vault's answer is not one: %w", err)
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
}
|
||||
|
||||
// coreBoxesFor is coreBoxesOf for one machine, read from and kept in the store, and said on a send.
|
||||
func coreBoxesFor(ctx context.Context, inv *inventory.Inventory, node string, modules []catalogue.Manifest,
|
||||
handed func(catalogue.Manifest) bool, choosing Choosing) (coreBoxes, error) {
|
||||
core := coreModulesOf(modules, handed)
|
||||
if len(core) == 0 {
|
||||
if choosing == Allocating {
|
||||
settleVaultSilent(ctx, node)
|
||||
}
|
||||
return coreBoxes{}, nil
|
||||
}
|
||||
key, err := inv.SealingKeyOf(ctx, node)
|
||||
if err != nil {
|
||||
return coreBoxes{}, err
|
||||
}
|
||||
kept, err := inv.KeptCoreBoxes(ctx, node)
|
||||
if err != nil {
|
||||
return coreBoxes{}, err
|
||||
}
|
||||
var ask coreAsk
|
||||
if choosing == Allocating {
|
||||
ask = coreVault(ctx)
|
||||
}
|
||||
out := coreBoxesOf(node, key, core, kept, choosing, ask)
|
||||
for _, b := range out.keep {
|
||||
if err := inv.KeepCoreBox(ctx, node, b); err != nil {
|
||||
return coreBoxes{}, err
|
||||
}
|
||||
}
|
||||
if choosing == Allocating {
|
||||
sayVaultSilent(ctx, node, out)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// kindVaultSilent is a send that did not have the vault's box for a core credential of a machine.
|
||||
const kindVaultSilent = "vault-silent"
|
||||
|
||||
// vaultSilentObservation is that, as a condition: `machine.<machine>.vault-silent`.
|
||||
func vaultSilentObservation(node string, b coreBoxes) conditions.Observation {
|
||||
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: kindVaultSilent, Token: kindVaultSilent,
|
||||
Machine: node, Severity: conditions.Warning, Source: coreDowngradeSource}
|
||||
var said []string
|
||||
if len(b.notPlaced) > 0 {
|
||||
o.Severity = conditions.Urgent
|
||||
var left []string
|
||||
for m := range b.notPlaced {
|
||||
left = append(left, m)
|
||||
}
|
||||
sort.Strings(left)
|
||||
said = append(said, "not placed: "+strings.Join(left, ", "))
|
||||
}
|
||||
if len(b.held) > 0 {
|
||||
said = append(said, "kept as last sent: "+strings.Join(b.held, ", "))
|
||||
}
|
||||
o.Summary = fmt.Sprintf("the vault did not answer for the core's credentials on %s; %s", node, strings.Join(said, "; "))
|
||||
o.Said = o.Summary + " — " + strings.Join(b.silent, "; ")
|
||||
return o
|
||||
}
|
||||
|
||||
// sayVaultSilent raises the condition for a send the vault did not answer whole, and clears it for one it did.
|
||||
func sayVaultSilent(ctx context.Context, node string, b coreBoxes) {
|
||||
if len(b.silent) == 0 && len(b.notPlaced) == 0 {
|
||||
settleVaultSilent(ctx, node)
|
||||
return
|
||||
}
|
||||
fmt.Printf("HELD: %s\n", vaultSilentObservation(node, b).Said)
|
||||
if conditionsFrom == nil {
|
||||
return
|
||||
}
|
||||
if _, err := conditionsFrom.Observe(ctx, vaultSilentObservation(node, b)); err != nil {
|
||||
fmt.Printf("the condition for the vault's silence on %s could not be raised: %v\n", node, err)
|
||||
}
|
||||
}
|
||||
|
||||
// settleVaultSilent clears the condition once a send had every core credential of the machine from the vault.
|
||||
func settleVaultSilent(ctx context.Context, node string) {
|
||||
if conditionsFrom == nil {
|
||||
return
|
||||
}
|
||||
key := vaultSilentObservation(node, coreBoxes{}).Key()
|
||||
if _, err := conditionsFrom.Clear(ctx, key, "the vault answered for every core credential of "+node); err != nil {
|
||||
fmt.Printf("the condition %s could not be cleared: %v\n", key, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The core's own credentials travel sealed by the vault to their machine (novox/hq ADR 0349 §16, issue 755; the lead's
|
||||
// decision, issue 755 notes 4 and 5): to an engine that opens the vault's box, the send carries the box the vault
|
||||
// answered the controller and never the controller's own seal; when the vault does not answer, a kept box is carried
|
||||
// again and a module with none is not placed, with a condition naming the vault and the machine. An engine below the
|
||||
// contract composes exactly as before, and the vault is not asked.
|
||||
|
||||
// coreStandIn answers `get` with a box per name and generation, or its silence.
|
||||
type coreStandIn struct {
|
||||
asked []string
|
||||
keys []string
|
||||
generation int
|
||||
silent error
|
||||
}
|
||||
|
||||
func (v *coreStandIn) ask(name, key string) (coreAnswer, error) {
|
||||
v.asked, v.keys = append(v.asked, name), append(v.keys, key)
|
||||
if v.silent != nil {
|
||||
return coreAnswer{}, v.silent
|
||||
}
|
||||
box := `{"v":1,"eph":"ZQ==","iv":"aQ==","tag":"dA==","ct":"` + strings.Repeat("c", len(v.asked)) + `"}`
|
||||
return coreAnswer{Name: name, Generation: v.generation, Sealed: json.RawMessage(box)}, nil
|
||||
}
|
||||
|
||||
func withVault(t *testing.T, v *coreStandIn) {
|
||||
t.Helper()
|
||||
before := coreVault
|
||||
coreVault = func(context.Context) coreAsk { return v.ask }
|
||||
t.Cleanup(func() { coreVault = before })
|
||||
}
|
||||
|
||||
func aCoreMesh(t *testing.T, contract int) (*stores, *coreStandIn) {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
client := aClientModule()
|
||||
register(t, open, client)
|
||||
if _, err := assign(ctx, open, "laptop", client.Module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := issueProvisionerClient(ctx, open.inventory, client, "laptop", broker.Broker{Fingerprint: "ab"},
|
||||
"10.0.0.1:4222"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.RecordHealth(ctx, inventory.NodeHealth{Node: "laptop", Contract: contract,
|
||||
SaidAt: time.Now()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v := &coreStandIn{generation: 1}
|
||||
withVault(t, v)
|
||||
return open, v
|
||||
}
|
||||
|
||||
func clientsBroker(resources []map[string]any) map[string]any {
|
||||
for _, r := range resources {
|
||||
if r["id"] == "mesh-provisioner-client.needs-"+broker.ProvisionerSecret {
|
||||
return r
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func vaultSilence(t *testing.T, node string) (conditions.Condition, bool) {
|
||||
t.Helper()
|
||||
c, found, err := conditionsFrom.Get(t.Context(), vaultSilentObservation(node, coreBoxes{}).Key())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c, found
|
||||
}
|
||||
|
||||
func TestTheSendCarriesTheVaultsBoxForACoreCredentialAndKeepsIt(t *testing.T) {
|
||||
open, vault := aCoreMesh(t, link.VaultBoxContract)
|
||||
key, err := open.inventory.SealingKeyOf(t.Context(), "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := clientsBroker(sentTo(t, open, "laptop"))
|
||||
if f == nil || f["sealed"] != nil || f[catalogue.VaultSealedField] == nil {
|
||||
t.Fatalf("the core credential is sent as %v: the vault's box, never the controller's seal", f)
|
||||
}
|
||||
if want := []string{"own/laptop/mesh-provisioner-client/" + broker.ProvisionerSecret}; !reflect.DeepEqual(vault.asked, want) {
|
||||
t.Fatalf("the vault was asked %v", vault.asked)
|
||||
}
|
||||
if want, _ := sealingKeyPEM(key); vault.keys[0] != want {
|
||||
t.Fatalf("the vault was asked to seal to %q, not the machine's key", vault.keys[0])
|
||||
}
|
||||
first, _ := json.Marshal(f[catalogue.VaultSealedField])
|
||||
|
||||
// The vault answers afresh with the same generation: the kept box is carried again, so the declaration is stable.
|
||||
again, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField])
|
||||
if string(again) != string(first) {
|
||||
t.Fatalf("a fresh box of the same generation changed the declaration: %s, then %s", first, again)
|
||||
}
|
||||
// A rotation in the vault (a newer generation) is carried.
|
||||
vault.generation = 2
|
||||
rotated, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField])
|
||||
if string(rotated) == string(first) {
|
||||
t.Fatal("a newer generation was not carried")
|
||||
}
|
||||
if _, found := vaultSilence(t, "laptop"); found {
|
||||
t.Fatal("a send the vault answered raised its silence")
|
||||
}
|
||||
}
|
||||
|
||||
func TestASilentVaultHoldsTheKeptBoxAndSaysSo(t *testing.T) {
|
||||
open, vault := aCoreMesh(t, link.VaultBoxContract)
|
||||
first, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField])
|
||||
vault.silent = errors.New("no responders")
|
||||
held := clientsBroker(sentTo(t, open, "laptop"))
|
||||
got, _ := json.Marshal(held[catalogue.VaultSealedField])
|
||||
if held == nil || held["sealed"] != nil || string(got) != string(first) {
|
||||
t.Fatalf("a silent vault's credential is sent as %v: the box kept, never the controller's seal", held)
|
||||
}
|
||||
c, found := vaultSilence(t, "laptop")
|
||||
if !found || !strings.Contains(c.Summary, "vault") || !strings.Contains(c.Summary, "laptop") {
|
||||
t.Fatalf("no condition names the vault and the machine: %+v", c)
|
||||
}
|
||||
// The vault answers again: the condition clears.
|
||||
vault.silent = nil
|
||||
sentTo(t, open, "laptop")
|
||||
if _, found := vaultSilence(t, "laptop"); found {
|
||||
t.Fatal("the condition stayed after the vault answered")
|
||||
}
|
||||
}
|
||||
|
||||
func TestACoreModuleFirstPlacedWhileTheVaultIsSilentIsNotPlaced(t *testing.T) {
|
||||
open, vault := aCoreMesh(t, link.VaultBoxContract)
|
||||
vault.silent = errors.New("no responders")
|
||||
plan, settings, err := planFor(t.Context(), open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationFor(t.Context(), open, "laptop", plan, settings)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range declared.Resources {
|
||||
if strings.HasPrefix(r["id"].(string), "mesh-provisioner-client.") {
|
||||
t.Fatalf("a core module was placed though the vault gave nothing for it: %v", r)
|
||||
}
|
||||
}
|
||||
if why := declared.leftOutWhy["mesh-provisioner-client"]; !strings.Contains(why, "never seals it itself") {
|
||||
t.Fatalf("left out %v", declared.leftOutWhy)
|
||||
}
|
||||
c, found := vaultSilence(t, "laptop")
|
||||
if !found || c.Severity != conditions.Urgent || !strings.Contains(c.Summary, "not placed") {
|
||||
t.Fatalf("the condition does not say why: %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// An engine below VaultBoxContract is sent the controller's own seal, as before, and the vault is not asked.
|
||||
func TestAnEngineBelowTheContractIsSentTheControllersSealAsBefore(t *testing.T) {
|
||||
open, vault := aCoreMesh(t, link.HandedContract)
|
||||
f := clientsBroker(sentTo(t, open, "laptop"))
|
||||
if f == nil || f["sealed"] == nil || f[catalogue.VaultSealedField] != nil {
|
||||
t.Fatalf("an older engine is sent %v", f)
|
||||
}
|
||||
if len(vault.asked) != 0 {
|
||||
t.Fatalf("the vault was asked %v for an older engine", vault.asked)
|
||||
}
|
||||
}
|
||||
|
||||
// Every question but a send reads the kept box and asks nothing; with none kept, a stand-in the next send asks for.
|
||||
func TestOnlyASendAsksTheVault(t *testing.T) {
|
||||
core := map[string][]string{"nats": {"broker"}}
|
||||
key := aPublicKey(t)
|
||||
kept := map[string]map[string]inventory.CoreBox{"nats": {"broker": {Module: "nats", Name: "broker",
|
||||
Generation: 1, SealedTo: key, Box: json.RawMessage(`{"v":1}`)}}}
|
||||
never := func(string, string) (coreAnswer, error) {
|
||||
t.Fatal("the vault was asked by a question that is not a send")
|
||||
return coreAnswer{}, nil
|
||||
}
|
||||
for _, choosing := range []Choosing{Reading, Foreseeing} {
|
||||
if got := coreBoxesOf("anchor", key, core, kept, choosing, never); string(got.sealed["nats"]["broker"]) != `{"v":1}` {
|
||||
t.Errorf("read %s", got.sealed["nats"]["broker"])
|
||||
}
|
||||
if got := coreBoxesOf("anchor", key, core, nil, choosing, never); string(got.sealed["nats"]["broker"]) !=
|
||||
string(catalogue.ForeseenVaultBox) {
|
||||
t.Errorf("with none kept, read %s", got.sealed["nats"]["broker"])
|
||||
}
|
||||
}
|
||||
// A box kept for another key of the machine is never carried.
|
||||
other := coreBoxesOf("anchor", aPublicKey(t), core, kept, Allocating,
|
||||
func(string, string) (coreAnswer, error) { return coreAnswer{}, errors.New("silent") })
|
||||
if _, left := other.notPlaced["nats"]; !left {
|
||||
t.Fatalf("a box sealed to the machine's old key was carried: %+v", other)
|
||||
}
|
||||
}
|
||||
|
||||
// The core's modules on a machine: on the list, not handed, with their own secrets but one issued outside.
|
||||
func TestTheCoresModulesAreTheListsWithTheirOwnSecrets(t *testing.T) {
|
||||
modules := []catalogue.Manifest{
|
||||
{Module: "nats", OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/a"}, "given": {Path: "/b",
|
||||
IssuedBy: catalogue.IssuedOutside}}},
|
||||
{Module: "postgres", OwnSecrets: catalogue.OwnSecrets{"superuser": {Path: "/c"}}},
|
||||
{Module: "mesh-provisioner", OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/d"}}},
|
||||
}
|
||||
got := coreModulesOf(modules, func(m catalogue.Manifest) bool { return m.Module == "mesh-provisioner" })
|
||||
if want := map[string][]string{"nats": {"broker"}}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("got %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A dead letter from a restricted stream ages out with its stream (novox/hq ADR 0376 §3, §6).
|
||||
//
|
||||
// DEAD_LETTERS lets such a message go when its stream's maximum age runs out, so a consumer's `max-deliveries`
|
||||
// condition can clear with nobody having delivered it again or dropped it. That is said: the watchdogs remember,
|
||||
// from one look to the next, when each restricted dead letter ages out, count those gone once their time passed,
|
||||
// say each in the controller's log as it goes, and clear the condition with words saying how many aged out
|
||||
// undelivered and from which stream.
|
||||
|
||||
// agingTally is what the watchdogs remember of restricted dead letters between looks.
|
||||
type agingTally struct {
|
||||
// seen is each consumer's restricted dead letters at the last look, by id, with when each ages out.
|
||||
seen map[string]map[uint64]time.Time
|
||||
// agedOut is how many of each consumer's aged out undelivered while its condition was open.
|
||||
agedOut map[string]int
|
||||
}
|
||||
|
||||
// look compares this look's restricted dead letters with the last: newly is how many of each consumer's aged out
|
||||
// since, and cleared how many aged out in all for each consumer DEAD_LETTERS no longer holds anything of, when any
|
||||
// did. held is every consumer's count this look; aging its restricted dead letters.
|
||||
func (t *agingTally) look(now time.Time, held map[string]int, aging map[string][]link.Aging) (newly, cleared map[string]int) {
|
||||
if t.seen == nil {
|
||||
t.seen, t.agedOut = map[string]map[uint64]time.Time{}, map[string]int{}
|
||||
}
|
||||
newly, cleared = map[string]int{}, map[string]int{}
|
||||
for key, before := range t.seen {
|
||||
still := map[uint64]bool{}
|
||||
for _, a := range aging[key] {
|
||||
still[a.ID] = true
|
||||
}
|
||||
for id, agesOut := range before {
|
||||
if !still[id] && !agesOut.After(now) {
|
||||
newly[key]++
|
||||
}
|
||||
}
|
||||
if newly[key] == 0 {
|
||||
delete(newly, key)
|
||||
}
|
||||
t.agedOut[key] += newly[key]
|
||||
}
|
||||
for key, n := range t.agedOut {
|
||||
if held[key] > 0 {
|
||||
continue
|
||||
}
|
||||
if n > 0 {
|
||||
cleared[key] = n
|
||||
}
|
||||
delete(t.agedOut, key)
|
||||
}
|
||||
t.seen = map[string]map[uint64]time.Time{}
|
||||
for key, letters := range aging {
|
||||
t.seen[key] = map[uint64]time.Time{}
|
||||
for _, a := range letters {
|
||||
t.seen[key][a.ID] = a.AgesOut
|
||||
}
|
||||
}
|
||||
return newly, cleared
|
||||
}
|
||||
|
||||
// agedOutResolved is the line a consumer's condition clears with when messages it gave up on aged out undelivered.
|
||||
func agedOutResolved(key string, n int) string {
|
||||
stream, _, _ := strings.Cut(key, ".")
|
||||
messages := "a message"
|
||||
if n > 1 {
|
||||
messages = fmt.Sprintf("%d messages", n)
|
||||
}
|
||||
return fmt.Sprintf("Resolved: %s of the stream %s aged out undelivered", messages, stream)
|
||||
}
|
||||
|
||||
// sayAgedOut says, in the log, what aged out since the last look, and clears the condition of every consumer whose
|
||||
// dead letters are all gone with the words saying how many aged out. Before the rows are reconciled, so the
|
||||
// condition is cleared with these words and not its own.
|
||||
func (w *watchdogs) sayAgedOut(ctx context.Context, f *signalFacts) []string {
|
||||
if f.advisoriesErr != nil || f.deadLettersAging == nil {
|
||||
return nil
|
||||
}
|
||||
newly, cleared := w.aging.look(f.now, f.deadLetters, f.deadLettersAging)
|
||||
var problems []string
|
||||
for _, key := range countedKeys(newly) {
|
||||
stream, consumer, _ := strings.Cut(key, ".")
|
||||
fmt.Printf("%d message(s) %s gave up on aged out of %s undelivered: %s keeps them only for its maximum age "+
|
||||
"(novox/hq ADR 0376)\n", newly[key], link.ConsumerInWords(stream, consumer), "DEAD_LETTERS", stream)
|
||||
}
|
||||
for _, key := range countedKeys(cleared) {
|
||||
stream, consumer, _ := strings.Cut(key, ".")
|
||||
fmt.Printf("%s no longer has a message kept: %d aged out of %s undelivered, so its condition clears "+
|
||||
"(novox/hq ADR 0376)\n", link.ConsumerInWords(stream, consumer), cleared[key], stream)
|
||||
if _, err := w.keeper.ClearSaying(ctx, conditions.Key(conditions.ScopeBus, key, link.AdvisoryMaxDeliveries),
|
||||
fmt.Sprintf("its %d dead letter(s) of %s aged out undelivered", cleared[key], stream),
|
||||
agedOutResolved(key, cleared[key])); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// countedKeys is a count's keys in order.
|
||||
func countedKeys(m map[string]int) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A dead letter that ages out clears its condition and says so** (novox/hq ADR 0376 §6), on a fake clock: while
|
||||
// DEAD_LETTERS holds a restricted stream's messages for a consumer its max-deliveries condition is open; once they
|
||||
// have aged out it clears, with words saying how many aged out undelivered and from which stream. One dropped before
|
||||
// its age is not counted as aged out, and a consumer of EVENTS clears as before.
|
||||
func TestADeadLetterThatAgesOutClearsItsConditionAndSaysSo(t *testing.T) {
|
||||
now := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)
|
||||
store := conditions.NewInMemory()
|
||||
told := &conditions.Told{}
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: told,
|
||||
Now: func() time.Time { return now }})
|
||||
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
||||
|
||||
look := func(at time.Time, held map[string]int, aging map[string][]link.Aging) {
|
||||
f := calm(at)
|
||||
f.deadLetters, f.deadLettersAging = held, aging
|
||||
w.see(t.Context(), f)
|
||||
}
|
||||
mail := "MAIL.ace_mailer"
|
||||
look(now, map[string]int{mail: 3, "EVENTS.media_sonarr": 1}, map[string][]link.Aging{mail: {
|
||||
{ID: 1, AgesOut: now.Add(time.Hour)}, {ID: 2, AgesOut: now.Add(2 * time.Hour)}, {ID: 3, AgesOut: now.Add(3 * time.Hour)}}})
|
||||
if open, _ := k.Open(t.Context()); len(open) != 2 {
|
||||
t.Fatalf("open: %+v", open)
|
||||
}
|
||||
// One dropped by a person before its age: not aged out.
|
||||
look(now.Add(30*time.Minute), map[string]int{mail: 2, "EVENTS.media_sonarr": 1}, map[string][]link.Aging{mail: {
|
||||
{ID: 2, AgesOut: now.Add(2 * time.Hour)}, {ID: 3, AgesOut: now.Add(3 * time.Hour)}}})
|
||||
look(now.Add(150*time.Minute), map[string]int{mail: 1}, map[string][]link.Aging{mail: {{ID: 3, AgesOut: now.Add(3 * time.Hour)}}})
|
||||
open, _ := k.Open(t.Context())
|
||||
if len(open) != 1 || open[0].Key != conditions.Key(conditions.ScopeBus, mail, link.AdvisoryMaxDeliveries) {
|
||||
t.Fatalf("after one aged out: %+v", open)
|
||||
}
|
||||
look(now.Add(4*time.Hour), map[string]int{}, map[string][]link.Aging{})
|
||||
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||
t.Fatalf("after all aged out: %+v", open)
|
||||
}
|
||||
k.Close(context.Background()) // everything said is said once it closes
|
||||
var mailCleared, sonarrCleared string
|
||||
for _, e := range told.Said() {
|
||||
if e.Change != conditions.ChangeCleared {
|
||||
continue
|
||||
}
|
||||
switch e.Condition.Key {
|
||||
case conditions.Key(conditions.ScopeBus, mail, link.AdvisoryMaxDeliveries):
|
||||
mailCleared = e.Condition.Resolved
|
||||
case conditions.Key(conditions.ScopeBus, "EVENTS.media_sonarr", link.AdvisoryMaxDeliveries):
|
||||
sonarrCleared = e.Condition.Resolved
|
||||
}
|
||||
}
|
||||
if mailCleared != "Resolved: 2 messages of the stream MAIL aged out undelivered" {
|
||||
t.Fatalf("MAIL's condition cleared with %q", mailCleared)
|
||||
}
|
||||
if sonarrCleared != "Resolved: the messages it gave up on were delivered again or dropped" {
|
||||
t.Fatalf("an EVENTS consumer's condition cleared with %q", sonarrCleared)
|
||||
}
|
||||
if why, ok := conditions.Plain(mailCleared); !ok {
|
||||
t.Fatalf("not plain: %s", why)
|
||||
}
|
||||
}
|
||||
@@ -952,7 +952,7 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
if *group == "" && len(members) == 1 {
|
||||
// One head, checked again as the forge's announcement would have it (a recheck): the controller's
|
||||
// own path for a pull request, run because the delivery's owner asked.
|
||||
if err := sayingOnTheBus(ctx, inv, func() error { return (following{open}).PullUpdated(ctx, members[0].pull()) }); err != nil {
|
||||
if err := sayingOnTheBus(ctx, inv, func() error { return (following{open: open, oneShot: true}).PullUpdated(ctx, members[0].pull()) }); err != nil {
|
||||
return err
|
||||
}
|
||||
return answer(map[string]any{"rechecked": members[0].ID})
|
||||
@@ -1051,7 +1051,9 @@ func appliedFrom(ctx context.Context, inv *inventory.Inventory) inventory.Applie
|
||||
fmt.Fprintf(os.Stderr, "the report of %s after %s could not be read: %v\n", node, walkSent.Format(time.RFC3339), err)
|
||||
return inventory.AppliedReport{}, false
|
||||
}
|
||||
return r, ok
|
||||
// And, for each module the send handed to the provisioner, its client's report on that build (novox/hq ADR
|
||||
// 0362): applied only once both said it, at the later; failed if the client failed.
|
||||
return inventory.ProvisionedSend(node, send, r, ok, provLookup(ctx))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -54,6 +55,10 @@ type deskGive struct {
|
||||
trusted func(module string) (bool, error)
|
||||
// ask asks one machine's node-launcher.secret and answers its result, or the holder's refusal.
|
||||
ask func(machine string, args map[string]any) (json.RawMessage, error)
|
||||
// promptFor is the one line the secret's declaration asks for it as (novox/hq issue 789): read from the
|
||||
// controller's own catalogue, never from the call or the module's runtime. Nil, or an empty answer, asks by
|
||||
// name alone, as the desk does today.
|
||||
promptFor func(module, name string) (string, error)
|
||||
// accept seals the value as `secret accept` does, and says whether it lives until the module's start.
|
||||
accept func(value string) (untilStart bool, err error)
|
||||
// record writes the act in the hand-act log.
|
||||
@@ -67,6 +72,9 @@ type deskGive struct {
|
||||
// nil keeps no record (a test that does not look). end closes it with how it ended.
|
||||
open func(node, module, name, desk string) (int64, error)
|
||||
end func(id int64, outcome string) error
|
||||
// vault, when set, is the path of a module whose secrets are filled the new way (vault_secrets.go): the vault
|
||||
// records the ask, holds its bounds and opens what is typed; open, end and accept are not used.
|
||||
vault vaultAsker
|
||||
}
|
||||
|
||||
// askedByName is the caller as the prompt names it: the first clause of what the bus said, in the characters
|
||||
@@ -133,6 +141,9 @@ func (d deskGive) give(node, module, name, desk string) (string, error) {
|
||||
"bus, where an agent may answer first (novox/hq ADR 0259 §10)", module, name, node, module, name)
|
||||
}
|
||||
}
|
||||
if d.vault != nil {
|
||||
return d.giveToVault(node, module, name, desk)
|
||||
}
|
||||
// The bounds, read and kept before anybody is asked to type (novox/hq ADR 0277): one open prompt per secret,
|
||||
// few an hour. How the ask ends is recorded whatever happens below.
|
||||
outcome := "failed"
|
||||
@@ -153,15 +164,28 @@ func (d deskGive) give(node, module, name, desk string) (string, error) {
|
||||
}
|
||||
// By name, never by words: the holder writes the prompt from these, and says the controller asks, which
|
||||
// the bus alone makes true (broker.ControllerOnly). Who asked is the bus's word on the caller, cut to a
|
||||
// name's characters — never an argument of the call.
|
||||
raw, err := d.ask(desk, map[string]any{
|
||||
// name's characters — never an argument of the call. `what` is the catalogue's own words for the secret
|
||||
// (novox/hq issue 789), never a word the caller chose; omitted when the module declares none, so an old
|
||||
// desk and an old module behave as today.
|
||||
args := map[string]any{
|
||||
"module": module,
|
||||
"secret": name,
|
||||
"node": node,
|
||||
"asked_by": askedByName(d.askedBy),
|
||||
"seal_to": public,
|
||||
"timeout_seconds": deskPromptWithin,
|
||||
})
|
||||
}
|
||||
if d.promptFor != nil {
|
||||
what, err := d.promptFor(module, name)
|
||||
if err != nil {
|
||||
outcome = "refused"
|
||||
return "", fmt.Errorf("nobody was asked to type anything: %w", err)
|
||||
}
|
||||
if what != "" {
|
||||
args["what"] = what
|
||||
}
|
||||
}
|
||||
raw, err := d.ask(desk, args)
|
||||
if err != nil {
|
||||
outcome = "refused"
|
||||
return "", fmt.Errorf("the desk on %s could not be asked: %w", desk, err)
|
||||
@@ -236,8 +260,9 @@ func askAtDesk(ctx context.Context, node, module, name, desk string) error {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
askedBy := link.Caller()
|
||||
d := deskGive{
|
||||
askedBy: link.Caller(),
|
||||
askedBy: askedBy,
|
||||
open: func(node, module, name, desk string) (int64, error) {
|
||||
return open.inventory.OpenSecretAsk(ctx, node, module, name, askedByName(link.Caller()), desk)
|
||||
},
|
||||
@@ -247,7 +272,8 @@ func askAtDesk(ctx context.Context, node, module, name, desk string) error {
|
||||
_, err := open.inventory.NodeByName(ctx, machine)
|
||||
return err
|
||||
},
|
||||
trusted: func(module string) (bool, error) { return open.inventory.RunsAsItsOwnAccount(ctx, module) },
|
||||
trusted: func(module string) (bool, error) { return open.inventory.RunsAsItsOwnAccount(ctx, module) },
|
||||
promptFor: func(module, name string) (string, error) { return open.inventory.OwnSecretPrompt(ctx, module, name) },
|
||||
ask: func(machine string, args map[string]any) (json.RawMessage, error) {
|
||||
var result json.RawMessage
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
@@ -273,7 +299,18 @@ func askAtDesk(ctx context.Context, node, module, name, desk string) error {
|
||||
return err
|
||||
})
|
||||
},
|
||||
announce: func(node, module, name, how string) error { return announceSecretGiven(ctx, node, module, name, how) },
|
||||
announce: func(node, module, name, how string) error {
|
||||
return announceSecretGiven(ctx, node, module, name, how, askedBy)
|
||||
},
|
||||
}
|
||||
// A module whose secrets are filled the new way is asked for by the vault (ADR 0349 clause 9); every other
|
||||
// keeps today's path (clause 17).
|
||||
ours, err := filledTheNewWay(module)
|
||||
if err != nil {
|
||||
return fmt.Errorf("nothing was asked: %w", err)
|
||||
}
|
||||
if ours {
|
||||
d.vault = theVault(ctx)
|
||||
}
|
||||
words, err := d.give(node, module, name, desk)
|
||||
if err != nil {
|
||||
@@ -324,10 +361,85 @@ func secretNameWords(name string) string {
|
||||
return strings.Join(strings.FieldsFunc(name, func(r rune) bool { return r == '-' || r == '_' || r == '.' }), " ")
|
||||
}
|
||||
|
||||
// announceSecretGiven raises it on this controller's keeper.
|
||||
func announceSecretGiven(ctx context.Context, node, module, name, how string) error {
|
||||
return withKeeper(ctx, func(k *conditions.Keeper) error {
|
||||
_, err := k.Observe(ctx, secretGivenObservation(node, module, name, how, time.Now()))
|
||||
return err
|
||||
// tellWithin is how long the operator channel's `tell` is given to answer: well under a human's patience,
|
||||
// and short because nothing in the call waits on the operator — it asks nothing (novox/hq ADR 0380).
|
||||
const tellWithin = 10 * time.Second
|
||||
|
||||
// askOperatorChannel asks the operator channel's seat (the holder issue 797 gives it) its `tell` verb,
|
||||
// through the controller's grant (novox/hq ADR 0380 §8), over this controller's own bus connection; a seam
|
||||
// a test replaces whole. The holder's own refusal is an error naming it; an absent holder or a refused
|
||||
// grant answer as link.ErrNothingServes or link.ErrGrantRefused (checked with errors.Is), so the caller can
|
||||
// tell the two apart from any other failure.
|
||||
var askOperatorChannel = func(ctx context.Context, args map[string]any) (json.RawMessage, error) {
|
||||
var result json.RawMessage
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
answer, err := link.AskMeshSeatTool(ctx, conn, "operator-channel", "tell", args, tellWithin)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return fmt.Errorf("operator-channel.tell refused: %s", answer.Error)
|
||||
}
|
||||
result = answer.Result
|
||||
return nil
|
||||
})
|
||||
return result, err
|
||||
}
|
||||
|
||||
// announceSecretGiven tells the operator, once, that a module's own secret was given (novox/hq ADR 0380):
|
||||
// one plain message through the operator channel's `tell`, naming the secret, the module, the machine,
|
||||
// where and when it was given and who asked — asking nothing, and opening no condition. **The controller's
|
||||
// change waits for that verb at run time, not at merge** (clause 9): while nothing on the bus yet serves it,
|
||||
// or the controller's own grant is refused, the secret-given condition is raised exactly as before ADR 0380
|
||||
// — any other failure (the holder itself refusing, or answering something unreadable) is the announcement's
|
||||
// own failure, said to the caller as it always was.
|
||||
func announceSecretGiven(ctx context.Context, node, module, name, how, askedBy string) error {
|
||||
at := time.Now()
|
||||
body := map[string]any{
|
||||
"headline": fmt.Sprintf("A password was entered for %s's %s on %s", module, secretNameWords(name), node),
|
||||
"subject": node,
|
||||
"lines": []string{
|
||||
fmt.Sprintf("%s at %s, asked by %s.", how, at.Local().Format("15:04"), askedByName(askedBy)),
|
||||
"If that was not you, give it again yourself at the controller's terminal.",
|
||||
},
|
||||
}
|
||||
_, told := askOperatorChannel(ctx, body)
|
||||
switch {
|
||||
case told == nil:
|
||||
clearSecretGivenConditions(ctx)
|
||||
return nil
|
||||
case errors.Is(told, link.ErrNothingServes), errors.Is(told, link.ErrGrantRefused):
|
||||
log.Printf("secret-given: %s's %s on %s could not be told through the operator channel (%v); the "+
|
||||
"secret-given condition is raised as it was before novox/hq ADR 0380 (its clause 9)", module, name,
|
||||
node, told)
|
||||
return withKeeper(ctx, func(k *conditions.Keeper) error {
|
||||
_, err := k.Observe(ctx, secretGivenObservation(node, module, name, how, at))
|
||||
return err
|
||||
})
|
||||
default:
|
||||
return told
|
||||
}
|
||||
}
|
||||
|
||||
// clearSecretGivenConditions retires every open secret-given condition once a plain message has replaced
|
||||
// it (novox/hq ADR 0380 §10): called after the first time a secret-given message is told successfully. No
|
||||
// resolved line is sent — the condition is simply gone — and why is kept in the keeper's own history.
|
||||
func clearSecretGivenConditions(ctx context.Context) {
|
||||
if err := withKeeper(ctx, func(k *conditions.Keeper) error {
|
||||
open, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, c := range open {
|
||||
if c.Kind != kindSecretGiven {
|
||||
continue
|
||||
}
|
||||
if _, err := k.ClearSaying(ctx, c.Key, "retired by novox/hq ADR 0380", ""); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
log.Printf("secret-given: the open secret-given conditions, retired by ADR 0380, could not be cleared: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -456,6 +456,52 @@ func TestThePromptNamesWhoAskedFromTheBussWordAlone(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The desk's prompt names which account of a family is asked for, from the catalogue's own words (novox/hq
|
||||
// issue 789): a family declaring `prompt` with `{member}` sends `what` with the member's suffix filled in.
|
||||
func TestThePromptNamesWhichAccountOfAFamilyIsAsked(t *testing.T) {
|
||||
d, _, _, asked := aDesk(t, sealedTo(t, typed))
|
||||
d.declares = func(module, name string) error {
|
||||
if module != "mesh-mail" || name != "mail-credential-gmail" {
|
||||
return errors.New(module + " does not declare " + name + " as an own secret")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
d.promptFor = func(module, name string) (string, error) {
|
||||
if module == "mesh-mail" && name == "mail-credential-gmail" {
|
||||
return "the password of the mail account gmail", nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
if _, err := d.give("anchor", "mesh-mail", "mail-credential-gmail", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := (*asked)[0]["what"]; got != "the password of the mail account gmail" {
|
||||
t.Errorf(`"what" = %v`, got)
|
||||
}
|
||||
}
|
||||
|
||||
// Without a declared prompt, no `what` is sent: an old module, or one that declares none, is asked by name alone,
|
||||
// as the desk does today (novox/hq issue 789).
|
||||
func TestWithNoDeclaredPromptNoWhatIsSent(t *testing.T) {
|
||||
d, _, _, asked := aDesk(t, sealedTo(t, typed))
|
||||
d.promptFor = func(module, name string) (string, error) { return "", nil }
|
||||
if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, there := (*asked)[0]["what"]; there {
|
||||
t.Errorf(`"what" was sent with no declared prompt: %v`, (*asked)[0])
|
||||
}
|
||||
// And with no promptFor wired at all (an older caller of deskGive), same thing: `give` still works, asking
|
||||
// nothing more than it does today.
|
||||
d2, _, _, asked2 := aDesk(t, sealedTo(t, typed))
|
||||
if _, err := d2.give("anchor", "telegram", "telegram-token", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, there := (*asked2)[0]["what"]; there {
|
||||
t.Errorf(`"what" was sent with promptFor nil: %v`, (*asked2)[0])
|
||||
}
|
||||
}
|
||||
|
||||
// An ask for a secret is bounded before anybody is asked to type (ADR 0277): the record refuses it, nothing is
|
||||
// asked; and how every ask ends is recorded.
|
||||
func TestASecretAskIsBoundedAndItsEndRecorded(t *testing.T) {
|
||||
@@ -510,3 +556,172 @@ func TestASecretAskIsNeverASecretRead(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// askOperatorChannelFake replaces askOperatorChannel for the test and records every call it was given.
|
||||
func askOperatorChannelFake(t *testing.T, answer func(args map[string]any) (json.RawMessage, error)) *[]map[string]any {
|
||||
t.Helper()
|
||||
var calls []map[string]any
|
||||
was := askOperatorChannel
|
||||
askOperatorChannel = func(_ context.Context, args map[string]any) (json.RawMessage, error) {
|
||||
calls = append(calls, args)
|
||||
return answer(args)
|
||||
}
|
||||
t.Cleanup(func() { askOperatorChannel = was })
|
||||
return &calls
|
||||
}
|
||||
|
||||
// announceWithRealTell wires a deskGive's announce to the real announceSecretGiven, as askAtDesk does,
|
||||
// rather than the fakes the tests above use to look at what `give` does around it.
|
||||
func announceWithRealTell(ctx context.Context, askedBy string) func(node, module, name, how string) error {
|
||||
return func(node, module, name, how string) error {
|
||||
return announceSecretGiven(ctx, node, module, name, how, askedBy)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0380 clauses 1-2: a secret given at the desk is told to the operator once, in a plain
|
||||
// message through the operator channel's `tell`, naming the secret, the module, where and when and who
|
||||
// asked, and opens no condition — the kind the review of 2026-10-09 (M4) raised is retired.
|
||||
func TestASecretGivenAtTheDeskIsToldOnceAndOpensNoCondition(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
k, _ := withConditionsInMemory(t)
|
||||
calls := askOperatorChannelFake(t, func(map[string]any) (json.RawMessage, error) { return nil, nil })
|
||||
d, _, _, _ := aDesk(t, sealedTo(t, typed))
|
||||
d.announce = announceWithRealTell(ctx, "mail-seats-835@novox")
|
||||
if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(*calls) != 1 {
|
||||
t.Fatalf("told the operator %d times, want exactly 1: %v", len(*calls), *calls)
|
||||
}
|
||||
call := (*calls)[0]
|
||||
if call["headline"] != "A password was entered for telegram's telegram token on anchor" {
|
||||
t.Errorf("headline %v", call["headline"])
|
||||
}
|
||||
if call["subject"] != "anchor" {
|
||||
t.Errorf("subject %v", call["subject"])
|
||||
}
|
||||
lines, _ := call["lines"].([]string)
|
||||
if len(lines) != 2 || !strings.Contains(lines[0], "at the desk on laptop at") ||
|
||||
!strings.Contains(lines[0], "asked by mail-seats-835@novox") ||
|
||||
lines[1] != "If that was not you, give it again yourself at the controller's terminal." {
|
||||
t.Errorf("lines %v", lines)
|
||||
}
|
||||
open, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(open) != 0 {
|
||||
t.Errorf("a condition was raised though the operator was told: %+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0380 §9: until the operator channel's `tell` is delivered — nothing yet serves it, or the
|
||||
// controller's own grant is refused — the secret-given condition is raised exactly as before.
|
||||
func TestASecretGivenAtTheDeskRaisesTheConditionWhileNothingServesTell(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for _, failure := range []error{link.ErrNothingServes, link.ErrGrantRefused} {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
askOperatorChannelFake(t, func(map[string]any) (json.RawMessage, error) { return nil, failure })
|
||||
d, _, _, _ := aDesk(t, sealedTo(t, typed))
|
||||
d.announce = announceWithRealTell(ctx, "mesh-mail-835, novox")
|
||||
if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(open) != 1 || open[0].Kind != kindSecretGiven {
|
||||
t.Fatalf("%v: the condition was not raised as before: %+v", failure, open)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0380 §10: once the operator channel is told successfully, every secret-given condition
|
||||
// still open from before this change is retired, without a resolved line of its own.
|
||||
func TestASuccessfulTellRetiresEveryOpenSecretGivenCondition(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
k, _ := withConditionsInMemory(t)
|
||||
for _, module := range []string{"telegram", "mesh-mail"} {
|
||||
if _, err := k.Observe(ctx, secretGivenObservation("anchor", module, "a-secret", "at the desk on laptop",
|
||||
time.Now())); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
askOperatorChannelFake(t, func(map[string]any) (json.RawMessage, error) { return nil, nil })
|
||||
if err := announceSecretGiven(ctx, "anchor", "telegram", "telegram-token", "at the desk on laptop", "me"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(open) != 0 {
|
||||
t.Fatalf("the open secret-given conditions were not retired: %+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0380 clause 7: a trusted party's secret given at the controller's terminal is announced
|
||||
// before it is kept, and not kept when the announcement fails. It still fails when telling the operator
|
||||
// AND raising the fallback condition both fail — nothing silently keeps a trusted party's value unheard.
|
||||
func TestATrustedPartysSecretAtTheTerminalIsNotKeptWhenTellAndTheConditionBothFail(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
askOperatorChannelFake(t, func(map[string]any) (json.RawMessage, error) { return nil, link.ErrNothingServes })
|
||||
// conditionsFrom is left nil and there is no bus to dial here, so the fallback condition cannot be
|
||||
// raised either: both of ADR 0380's ways to tell the operator fail.
|
||||
kept := false
|
||||
if _, _, err := keepGiven(true,
|
||||
func() error {
|
||||
return announceSecretGiven(ctx, "anchor", "telegram", "telegram-token", "at the controller's terminal", "me")
|
||||
},
|
||||
func() (bool, error) { kept = true; return false, nil }); err == nil {
|
||||
t.Error("a trusted party's secret was kept though telling the operator and raising the condition both failed")
|
||||
}
|
||||
if kept {
|
||||
t.Error("the value was kept despite the failure")
|
||||
}
|
||||
}
|
||||
|
||||
// Review round 2 of mesh-controller (feat/mail-seats, head 5f24a47d): ADR 0380 §9's fallback — raising the
|
||||
// secret-given condition — answers only link.ErrNothingServes and link.ErrGrantRefused. Any other failure of
|
||||
// the operator channel's `tell` (the holder itself refusing, or answering something unreadable) is the
|
||||
// announcement's own failure, surfaced to the caller, and opens no condition.
|
||||
func TestAnOperatorChannelFailureOtherThanNothingServesOrGrantRefusedSurfacesAndRaisesNoCondition(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
k, _ := withConditionsInMemory(t)
|
||||
refused := errors.New("operator-channel.tell refused: too many lines")
|
||||
askOperatorChannelFake(t, func(map[string]any) (json.RawMessage, error) { return nil, refused })
|
||||
if err := announceSecretGiven(ctx, "anchor", "telegram", "telegram-token", "at the desk on laptop", "me"); err != refused {
|
||||
t.Fatalf("announceSecretGiven = %v, want the refusal itself surfaced", err)
|
||||
}
|
||||
open, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(open) != 0 {
|
||||
t.Errorf("a condition was raised though the failure was neither ErrNothingServes nor ErrGrantRefused: %+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
// Review round 2 of mesh-controller: a secret accepted at the controller's terminal writes a hand act naming
|
||||
// who asked, where and when it was given — and never the value (novox/hq ADR 0380 §6).
|
||||
func TestASecretAcceptedAtTheTerminalWritesAHandActNamingWhoWhereAndNeverTheValue(t *testing.T) {
|
||||
act := terminalGiveHandAct("anchor", "telegram", "telegram-token", "mail-seats-835@novox")
|
||||
if act.Verb != "secret accept" || act.Cause != "given-at-the-terminal" {
|
||||
t.Fatalf("act %+v", act)
|
||||
}
|
||||
if !strings.Contains(act.Why, "at the controller's terminal") {
|
||||
t.Errorf("the act does not name where: %q", act.Why)
|
||||
}
|
||||
if !strings.Contains(act.Why, "asked by mail-seats-835@novox") {
|
||||
t.Errorf("the act does not name who: %q", act.Why)
|
||||
}
|
||||
if strings.Contains(act.Why, typed) {
|
||||
t.Error("the act carries the value")
|
||||
}
|
||||
// When is RecordHandAct's own job (it stamps At with time.Now when the act leaves none, and By with the
|
||||
// caller when the act leaves that empty too): this act leaves both for it to fill.
|
||||
if act.By != "" || !act.At.IsZero() {
|
||||
t.Errorf("the act sets its own who/when instead of leaving them to RecordHandAct: %+v", act)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -146,6 +146,11 @@ var probeRegistry = []probe{
|
||||
{ID: probeFillID, Asserts: "every module's bucket and log holds less than three quarters of its cap; one " +
|
||||
"said filling is cleared once it holds less than seven tenths", From: "ADR 0297, issue 501",
|
||||
Kind: kindBucketOrLogFilling, Raises: []string{kindBucketOrLogUnread}, Phase: 1, run: probeFill},
|
||||
// A grant of a restricted stream that composes nothing (novox/hq ADR 0376 §2): the module is carried by the
|
||||
// machine's tool runner, which never hears a restricted stream, so it hears nothing it was granted.
|
||||
{ID: probeWithheldID, Asserts: "no grant of a restricted stream names a module that composes nothing for it: every " +
|
||||
"module granted one runs as an account of its own", From: "ADR 0376 §2", Kind: kindGrantComposesNothing,
|
||||
Phase: 1, run: probeWithheldHearing},
|
||||
{ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals",
|
||||
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs},
|
||||
// The core's health definitions (novox/hq to-be 45 §8, ADR 0236): what a core component's new build is
|
||||
|
||||
@@ -348,11 +348,11 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot
|
||||
if layer.From == catalogue.MeshWideLayer {
|
||||
if !meshWide[module] {
|
||||
meshWide[module] = true
|
||||
f.Settings = append(f.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Values)})
|
||||
f.Settings = append(f.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Whole())})
|
||||
}
|
||||
continue
|
||||
}
|
||||
m.Settings = append(m.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Values)})
|
||||
m.Settings = append(m.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Whole())})
|
||||
}
|
||||
}
|
||||
m.Declaration = declarationFacts(ctx, open, n.Name, gens, gensErr, scrub)
|
||||
|
||||
@@ -22,6 +22,7 @@ func foundDirectory(module string, since time.Time) inventory.ResourceHealth {
|
||||
}
|
||||
|
||||
func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
now := time.Now()
|
||||
sent := now.Add(-time.Minute)
|
||||
f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now,
|
||||
@@ -48,6 +49,7 @@ func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) {
|
||||
// The whole walk: a module whose directory was used as found long before still gets its fix to every machine,
|
||||
// its pass kept and the wait said; a directory this very send found holds it and puts it back.
|
||||
func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
found time.Duration // when the directory was found, against now
|
||||
@@ -116,6 +118,7 @@ func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) {
|
||||
|
||||
// The condition says the wait in its own kind: the operator's, never urgent, and cleared once handed over.
|
||||
func TestADirectoryUsedAsFoundIsItsOwnCondition(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
k, _ := withConditionsInMemory(t)
|
||||
ctx := t.Context()
|
||||
rs := map[string][]inventory.ResourceHealth{"notes": {foundDirectory("notes", time.Now().Add(-time.Hour))}}
|
||||
|
||||
@@ -235,6 +235,10 @@ type gateFacts struct {
|
||||
sent map[string]inventory.SentDeclaration
|
||||
sentBuilds map[string]map[string]string
|
||||
commits map[string]string
|
||||
// provisioned answers each machine's provisioner client's newest report of a module (novox/hq ADR 0349 §15):
|
||||
// a module the gate's send handed to the provisioner is healthy only once it reported that build placed. Nil
|
||||
// reads as nothing said.
|
||||
provisioned inventory.ProvLookup
|
||||
}
|
||||
|
||||
// heldReading is the provider a module's findings are held under, as "<module> on <machine>", and, when that
|
||||
@@ -291,7 +295,7 @@ func (f gateFacts) supersededOn(module, machine string) (string, bool) {
|
||||
var gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
inv := open.inventory
|
||||
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
|
||||
rolledBack: witnessed.all()}
|
||||
rolledBack: witnessed.all(), provisioned: provLookup(ctx)}
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return f, err
|
||||
@@ -368,6 +372,12 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string,
|
||||
case r.Outcome != inventory.OutcomeApplied:
|
||||
return healthNotYet, fmt.Sprintf("%s reported %q", machine, r.Outcome)
|
||||
}
|
||||
// **And its files placed** (novox/hq ADR 0349 §15): a module the send handed to the provisioner is judged only
|
||||
// once that machine's client reported the build's templates placed, so a module whose files are not yet written
|
||||
// is never judged; a failure there is the build's on that machine.
|
||||
if h, why := judgeProvisioned(module, machine, r, f); h != healthGood {
|
||||
return h, why
|
||||
}
|
||||
// **No new condition about it**: about the machine itself, or naming the module on that machine,
|
||||
// raised since the judging began. The gate's own are not evidence about the build. A fault that was
|
||||
// there at the send and reopened since is not new; one that had cleared before the send and came back
|
||||
@@ -449,6 +459,32 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string,
|
||||
return healthGood, ""
|
||||
}
|
||||
|
||||
// judgeProvisioned is whether a module's files are placed on a machine, for a module the gate's send handed to the
|
||||
// provisioner there: healthy for any other.
|
||||
func judgeProvisioned(module, machine string, r inventory.Reported, f gateFacts) (health, string) {
|
||||
sent, kept := f.sent[machine]
|
||||
digest := sent.Provisioned[module]
|
||||
if !kept || digest == "" {
|
||||
return healthGood, ""
|
||||
}
|
||||
at := time.Time{}
|
||||
if r.At != nil {
|
||||
at = *r.At
|
||||
}
|
||||
one := inventory.SentDeclaration{Digest: sent.Digest, At: sent.At, Provisioned: map[string]string{module: digest}}
|
||||
got, ok := inventory.ProvisionedSend(machine, one, inventory.AppliedReport{At: at,
|
||||
Outcome: inventory.OutcomeApplied}, true, f.provisioned)
|
||||
switch {
|
||||
case !ok:
|
||||
return healthNotYet, fmt.Sprintf("the provisioner on %s has not reported %s's files placed for the new "+
|
||||
"build", machine, module)
|
||||
case got.Outcome != inventory.OutcomeApplied:
|
||||
return healthBroken, fmt.Sprintf("the provisioner on %s did not place %s's files: %s", machine, module,
|
||||
got.Outcome)
|
||||
}
|
||||
return healthGood, ""
|
||||
}
|
||||
|
||||
// machineWord is what one judging found wrong with a machine itself, apart from its modules: facts is
|
||||
// the judging's facts without those conditions, on what each names among the modules the send moved,
|
||||
// and whole what holds the machine back as a whole.
|
||||
@@ -1443,6 +1479,12 @@ func probeGates(ctx context.Context, d *doctor) ([]conditions.Observation, error
|
||||
}
|
||||
// A release held after a failed one, while builds still wait for a gate (ADR 0236).
|
||||
out = append(out, backlogObservation()...)
|
||||
// And a rest that went silent (novox/hq ADR 0353 §4).
|
||||
silent, err := silentObservation(ctx, d.open.inventory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, silent...)
|
||||
return sortedFound(dedupeObservations(out)), nil
|
||||
}
|
||||
|
||||
@@ -1532,6 +1574,12 @@ func gateLine(g *inventory.PlanGate) string {
|
||||
case inventory.GateFailed:
|
||||
what = "FAILED"
|
||||
}
|
||||
switch {
|
||||
case g.Verdict == "" && g.Pending:
|
||||
what = "kept, not yet sent" // novox/hq ADR 0353 §8
|
||||
case g.Verdict == "" && g.Silent:
|
||||
what = "silent on its send, left" // ADR 0353 §4
|
||||
}
|
||||
if g.Verdict == "" && g.Waiting() {
|
||||
what = "waits for a busy container on " + g.Busy.Machine
|
||||
}
|
||||
|
||||
@@ -57,6 +57,7 @@ func healthyContainer(module string) inventory.ResourceHealth {
|
||||
// tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass,
|
||||
// counted over the gate's spacing — before it says its own. The broken one alone is put back.
|
||||
func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
@@ -99,6 +100,7 @@ const broker_runtime = "node-tools"
|
||||
// **A pass is counted only the gate's spacing after the one before** (review (c)): a send judged every tick
|
||||
// while a module beside it is not yet healthy counts the healthy one once.
|
||||
func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
withGateBounds(t, time.Hour, time.Hour, time.Hour)
|
||||
@@ -123,6 +125,7 @@ func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) {
|
||||
|
||||
// **A carried build's pass carries its own wait, never another's** (review (e)).
|
||||
func TestACarriedPassCarriesOnlyItsOwnWait(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
@@ -147,6 +150,7 @@ func TestACarriedPassCarriesOnlyItsOwnWait(t *testing.T) {
|
||||
// carried its own module, healthy, and a build waiting on that machine that never became healthy. At the
|
||||
// bound the waiting one is put back and marked; the plan's own module keeps its pass, so no walk waits on it.
|
||||
func TestThePlansOwnModuleKeepsItsPassWhenItsSendFails(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := g.open.inventory
|
||||
@@ -227,6 +231,7 @@ func TestThePlansOwnModuleKeepsItsPassWhenItsSendFails(t *testing.T) {
|
||||
// **A provider waiting for a login says who waits on it** (review (g)): its consumer, failing a check that
|
||||
// needs it, is held under it, and the provider's relogin-needed condition lists it and is urgent.
|
||||
func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
@@ -289,6 +294,7 @@ func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) {
|
||||
// tools are marked and put back in the very judging that found them broken — their registered build is the
|
||||
// one before, and the laptop is sent it — while `app` goes on being judged, recovers, and keeps its own pass.
|
||||
func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
@@ -431,6 +437,7 @@ func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testi
|
||||
// **A move from a build not known excuses no wait** (issue 318 review): a plan's own module whose previous
|
||||
// build was not recorded, or whose previous manifest is not kept, cannot be shown to have added the group.
|
||||
func TestAMoveFromAnUnknownBuildExcusesNoWait(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
|
||||
@@ -36,6 +36,10 @@ func aGateMesh(t *testing.T) *gateMesh {
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
g := &gateMesh{open: open, health: map[string]health{}}
|
||||
// The gate's own tests judge a first machine, as walks a controller before novox/hq ADR 0360 started still are.
|
||||
wasGated := gateFirstMachine
|
||||
gateFirstMachine = true
|
||||
t.Cleanup(func() { gateFirstMachine = wasGated })
|
||||
g.keeper, _ = withConditionsInMemory(t)
|
||||
g.told = &conditions.Told{}
|
||||
was := doctorFrom
|
||||
@@ -271,6 +275,34 @@ func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A build rolls everywhere with no hand and no gate (novox/hq ADR 0360 §1): both machines in one send, no health
|
||||
// judged, and the walk done once both reported on the send. Before, anchor was judged first, then laptop sent.
|
||||
func TestABuildRollsToEveryMachineAtOnceWithNoGate(t *testing.T) {
|
||||
g := aGateMesh(t)
|
||||
gateFirstMachine = false
|
||||
ctx := t.Context()
|
||||
judged := false
|
||||
wasGather := gatherGateFacts
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
judged = true
|
||||
return wasGather(ctx, open, component)
|
||||
}
|
||||
t.Cleanup(func() { gatherGateFacts = wasGather })
|
||||
for i := 0; i < 6; i++ {
|
||||
advancePlans(ctx, g.open)
|
||||
}
|
||||
p := g.plan(t)
|
||||
if !reflect.DeepEqual(g.sent, [][]string{{"anchor", "laptop"}}) {
|
||||
t.Fatalf("sent %v, not both machines in one send", g.sent)
|
||||
}
|
||||
if s := p.Modules["app"]; p.State != inventory.PlanDone || s.Gate != nil || s.FirstAt != nil || judged {
|
||||
t.Fatalf("the plan is %s (%s), app %+v, judged %v", p.State, p.Note, s, judged)
|
||||
}
|
||||
if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 {
|
||||
t.Fatalf("a rolled build left a condition: %+v %v", obs, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A leaf module passes its gate on one reading, as soon as its first machine is healthy, without waiting out the
|
||||
// core's settle time (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it, issue 524).
|
||||
func TestALeafModulePassesOnOneHealthyReading(t *testing.T) {
|
||||
|
||||
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
@@ -60,12 +61,30 @@ type sentRecord struct {
|
||||
toldGeneration int64
|
||||
sender string
|
||||
modules []string
|
||||
// plans is each module the send handed to the provisioner, its plan, as JSON (design 57 P6); nil for none.
|
||||
plans []byte
|
||||
}
|
||||
|
||||
// sentRecordOf is a composed send's record: its sender is the caller this process acts for.
|
||||
func sentRecordOf(ctx context.Context, d sendable) sentRecord {
|
||||
return sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom,
|
||||
r := sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom,
|
||||
toldGeneration: d.Generation, sender: senderOf(callerOf(ctx)), modules: d.modules}
|
||||
if len(d.plans) > 0 {
|
||||
kept := map[string]keptPlan{}
|
||||
for module, p := range d.plans {
|
||||
kept[module] = keptPlan{Plan: p, RequiredBy: d.requiredBy[module]}
|
||||
}
|
||||
// A map of plain strings and slices: it marshals.
|
||||
r.plans, _ = json.Marshal(kept)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// keptPlan is a handed module's plan as a send keeps it (migration 0100): the plan, and for a module only pulled in
|
||||
// by another, the module that requires it (issue 765, inventory.SentProvisioningPlans).
|
||||
type keptPlan struct {
|
||||
catalogue.Plan
|
||||
RequiredBy string `json:"required-by,omitempty"`
|
||||
}
|
||||
|
||||
// callerOf is who asked for what this process does: the seat call's caller when ctx belongs to one, the
|
||||
|
||||
@@ -8,7 +8,8 @@ import (
|
||||
)
|
||||
|
||||
// A rotation asked through the seat carries why to the command, which records it in the hand-act
|
||||
// log (novox/hq ADR 0228); why with a provision is refused as passed over, not dropped.
|
||||
// log (novox/hq ADR 0228); and with a provision too, since the vault asks why of every rotation of a
|
||||
// credential it holds (issue 767).
|
||||
func TestARotationThroughTheSeatCarriesWhy(t *testing.T) {
|
||||
argv, err := argvFor("rotate", map[string]any{"node": "anchor", "module": "letta",
|
||||
"secret": "server-password", "why": "leaked into logs", "cause": "leaked"})
|
||||
@@ -19,8 +20,10 @@ func TestARotationThroughTheSeatCarriesWhy(t *testing.T) {
|
||||
if err != nil || strings.Join(argv, " ") != "secret rotate anchor letta server-password" {
|
||||
t.Fatalf("without why: %v %v", argv, err)
|
||||
}
|
||||
if argv, err := argvFor("rotate", map[string]any{"provision": "postgres-database", "why": "leaked"}); err == nil {
|
||||
t.Fatalf("why beside a provision was passed over: %v", argv)
|
||||
argv, err = argvFor("rotate", map[string]any{"provision": "postgres-database", "consumer": "ace", "why": "leaked",
|
||||
"cause": "leaked"})
|
||||
if err != nil || strings.Join(argv, " ") != "rotate postgres-database --consumer ace --why leaked --cause leaked" {
|
||||
t.Fatalf("why beside a provision: %v %v", argv, err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -105,6 +105,12 @@ var handActVerbs = []handActVerb{
|
||||
// (ADR 0257).
|
||||
{Verb: "mirrors", Decision: "which copies in the store no record names are the mesh's is a person's " +
|
||||
"word (ADR 0257)"},
|
||||
// A move of data a release runs once after its delivery (design 57): a person's step of the release, never a
|
||||
// repair.
|
||||
{Verb: "move-placement", Decision: "a release's move of data runs once after its delivery, on a person's " +
|
||||
"word (design 57, issue 736)"},
|
||||
{Verb: "move-config", Decision: "a release's move of data runs once after its delivery, on a person's " +
|
||||
"word (design 57, issue 664)"},
|
||||
{Verb: "bus upgrade", Decision: "the bus is never rolled by the mesh: replacing it is a planned step a " +
|
||||
"person starts (ADR 0236)"},
|
||||
{Verb: "upgrade release-backlog", Decision: "after a release plan failed, the next opens only when a " +
|
||||
@@ -118,6 +124,9 @@ var handActVerbs = []handActVerb{
|
||||
{Verb: "secret accept", Decision: "a value an outside party issued is given by a person, at their desk"},
|
||||
{Verb: "secret rotate", Decision: "a value a person judged disclosed is replaced on their word",
|
||||
DecidedFor: []string{causeLeakedInLogs}},
|
||||
// A provider's credential the vault holds, rotated there on a person's word (issue 767): the same judgement.
|
||||
{Verb: "rotate", Decision: "a value a person judged disclosed is replaced on their word",
|
||||
DecidedFor: []string{causeLeakedInLogs}},
|
||||
}
|
||||
|
||||
// personsDecision is whether an act in the log is a person's decision by design, by the verb that
|
||||
|
||||
@@ -642,7 +642,8 @@ func planStale(ctx context.Context, inv *inventory.Inventory, p inventory.Plan)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if u.RollOut {
|
||||
// A recorded module whose move goes out on a machine is still the walk's to send (novox/hq ADR 0377).
|
||||
if u.RollOut || len(wentOut(s.Recorded)) > 0 {
|
||||
return "", "", nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// A grant of a restricted stream that composes nothing (novox/hq ADR 0376 §2).
|
||||
//
|
||||
// A restricted stream is heard only by a module that runs as an account of its own (`runs-as`, ADR 0259): the
|
||||
// tool runner's bus account is the one every agent session on its machine reaches the bus as, so the controller
|
||||
// composes no consumer, grant or bind word for a module it carries (broker.ComposedHears). Where a grant still
|
||||
// names such a module, the module hears nothing it was granted, and that is said — in `doctor` and in the
|
||||
// controller's log on every raise and send — naming the module, the stream and the remedy. Never silent.
|
||||
|
||||
const (
|
||||
// probeWithheldID is the self-check's probe of it, and kindGrantComposesNothing the condition it raises.
|
||||
probeWithheldID = "D-heard"
|
||||
kindGrantComposesNothing = "grant-composes-nothing"
|
||||
)
|
||||
|
||||
// probeWithheldHearing says every granted hearing of a restricted stream composed for nothing.
|
||||
func probeWithheldHearing(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
records, err := d.open.inventory.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return withheldObservations(broker.WithheldOf(records)), nil
|
||||
}
|
||||
|
||||
// withheldObservations is each withheld hearing as a condition of its module on its machine.
|
||||
func withheldObservations(withheld []broker.Withheld) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, w := range withheld {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeModule,
|
||||
ID: w.Module + "." + strings.ToLower(w.Stream) + "." + w.Node,
|
||||
Kind: kindGrantComposesNothing, Machine: w.Node, Severity: conditions.Warning,
|
||||
Summary: w.Said(), Said: fmt.Sprintf("%s on %s granted %s; composed nothing", w.Module, w.Node, w.Stream)})
|
||||
}
|
||||
return sortedFound(out)
|
||||
}
|
||||
|
||||
// sayWithheld puts every withheld hearing in the controller's log, once per raise or send that composes it.
|
||||
func sayWithheld(records broker.Records, indent string) {
|
||||
for _, w := range broker.WithheldOf(records) {
|
||||
fmt.Printf("%sA GRANT COMPOSES NOTHING: %s\n", indent, w.Said())
|
||||
}
|
||||
}
|
||||
|
||||
// withheldWords are the plain words of a withheld hearing: the module, the machine and what to do.
|
||||
func withheldWords(o conditions.Observation) words {
|
||||
module, m := "a module", machineOr(o, "a machine")
|
||||
if o.Scope == conditions.ScopeModule && idPart(o, 0) != "" {
|
||||
module = idPart(o, 0)
|
||||
}
|
||||
return words{Headline: clip(conditions.Capital(fmt.Sprintf("%s on %s hears nothing it was granted", module, m)), 60),
|
||||
Needs: fmt.Sprintf("give %s an account of its own in its manifest; the details say which stream.", module),
|
||||
Explanation: fmt.Sprintf("You granted %s part of a stream that only modules running under an account of "+
|
||||
"their own may hear. It runs inside the shared tool runner on %s, which every assistant session there "+
|
||||
"also uses, so the mesh gives it none of that stream.", module, m),
|
||||
Resolved: fmt.Sprintf("%s on %s hears what it was granted, or is no longer granted it", module, m)}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// A grant naming a module the tool runner carries composes nothing, and doctor says so in plain words, naming the
|
||||
// module, the stream and the remedy (novox/hq ADR 0376 §2).
|
||||
func TestAGrantThatComposesNothingIsSaid(t *testing.T) {
|
||||
records := broker.Records{Nodes: []string{"ace"}, Assigned: map[string][]broker.Declared{"ace": {
|
||||
{Module: broker.RuntimeModule},
|
||||
{Module: "reader", Hears: []broker.Heard{{Stream: "MAIL", Emitter: "mesh-mail", Restricted: true,
|
||||
Filters: []string{"mesh.stream.mesh-mail.event.mail.mesh.>"}}}},
|
||||
{Module: "mailer", RunsAs: "mailer", Hears: []broker.Heard{{Stream: "MAIL", Emitter: "mesh-mail", Restricted: true,
|
||||
Filters: []string{"mesh.stream.mesh-mail.event.mail.mesh.>"}}}},
|
||||
}}}
|
||||
found := withheldObservations(broker.WithheldOf(records))
|
||||
if len(found) != 1 {
|
||||
t.Fatalf("said %+v", found)
|
||||
}
|
||||
o := found[0]
|
||||
if o.Kind != kindGrantComposesNothing || o.Machine != "ace" || !strings.Contains(o.Summary, "reader on ace") ||
|
||||
!strings.Contains(o.Summary, "MAIL") || !strings.Contains(o.Summary, "runs-as") {
|
||||
t.Fatalf("%+v", o)
|
||||
}
|
||||
w := plainWordings[kindGrantComposesNothing](o)
|
||||
if why, ok := conditions.PlainWords(w, "ace"); !ok {
|
||||
t.Fatalf("not plain: %s (%+v)", why, w)
|
||||
}
|
||||
registered := false
|
||||
for _, p := range probeRegistry {
|
||||
if p.ID == probeWithheldID && p.Kind == kindGrantComposesNothing && p.run != nil {
|
||||
registered = true
|
||||
}
|
||||
}
|
||||
if !registered {
|
||||
t.Fatal("doctor does not run the probe of a grant that composes nothing")
|
||||
}
|
||||
}
|
||||
@@ -146,12 +146,28 @@ func heldMachines(ctx context.Context, open *stores, names []string) (map[string
|
||||
same[m] = current[m].Commit
|
||||
}
|
||||
}
|
||||
why := heldBack(node, modules, same, known, current, plans)
|
||||
// A move under a module's own `record` that recreates nothing and moves no data goes as a module that rolls
|
||||
// out goes (novox/hq ADR 0377): a walk's to send, not a person's.
|
||||
here, copied := current, false
|
||||
for m, was := range same {
|
||||
if now, held := current[m]; held && now.OwnRecord() && !f.identical(m, was, now.Commit) && f.rolls(m, was) {
|
||||
if !copied {
|
||||
copied = true
|
||||
here = make(map[string]inventory.CurrentBuild, len(current))
|
||||
for k, v := range current {
|
||||
here[k] = v
|
||||
}
|
||||
}
|
||||
now.RollOut = true
|
||||
here[m] = now
|
||||
}
|
||||
}
|
||||
why := heldBack(node, modules, same, known, here, plans)
|
||||
// And a build no gate has seen is not carried by a send that does not judge it (ADR 0236).
|
||||
for _, mv := range f.moves(node, modules, same, known, false) {
|
||||
if planStillToSend(plans, mv.Module, node) == "" {
|
||||
why = append(why, fmt.Sprintf("%s would move from %s to %s, which has passed no gate yet — a release "+
|
||||
"plan sends it, one machine at a time, judged", mv.Module, buildName(mv.From), buildName(mv.To)))
|
||||
"plan sends it, one machine first and then the rest at once, judged", mv.Module, buildName(mv.From), buildName(mv.To)))
|
||||
}
|
||||
}
|
||||
if len(why) > 0 {
|
||||
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
// another send reaches anchor, unreported; the gate still passes. And a send that moves a judged module
|
||||
// to another build supersedes the judging: no verdict, nothing put back.
|
||||
func TestReplay352(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn)
|
||||
t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging)
|
||||
@@ -67,6 +68,7 @@ func testANewerSendDoesNotUnreportTheGatesOwn(t *testing.T) {
|
||||
|
||||
// A plan's own first send waits while a release judges the same module on that machine with another build.
|
||||
func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
@@ -82,6 +84,7 @@ func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
|
||||
|
||||
// A merge plan's judging is superseded the same way: another send moved its module on the first machine.
|
||||
func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
@@ -140,6 +143,7 @@ func testASendThatMovesTheModuleSupersedesTheJudging(t *testing.T) {
|
||||
// A report is on the gate's own send: the declaration itself, or one sequenced after it; a gate kept
|
||||
// without its send reads the report against the send made last, as before. Pure.
|
||||
func TestAReportIsHeldAgainstTheGatesOwnSend(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
sent := inventory.SentDeclaration{Digest: "d-490", Sequence: 490}
|
||||
for _, c := range []struct {
|
||||
r inventory.Reported
|
||||
@@ -185,6 +189,7 @@ func TestAReportIsHeldAgainstTheGatesOwnSend(t *testing.T) {
|
||||
// A gate judges only the build the machine was last sent: last sent another build of the module, the
|
||||
// judging is superseded, whatever the machine reports. Pure.
|
||||
func TestAGateJudgesOnlyTheBuildTheMachineWasLastSent(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
at := time.Now()
|
||||
f := gateFacts{now: at, reports: map[string]inventory.Reported{"anchor": {Node: "anchor", Outcome: inventory.OutcomeApplied,
|
||||
At: &at, Current: true}}, engines: map[string]string{}, served: map[string]served{}, rolledBack: map[string][]lease.Rollback{},
|
||||
@@ -212,6 +217,7 @@ func TestAGateJudgesOnlyTheBuildTheMachineWasLastSent(t *testing.T) {
|
||||
// A move of another build of a module to a machine where a release or a plan is judging that module
|
||||
// waits for that judging; the same build to that machine is already there. Pure.
|
||||
func TestAWalkWaitsForAJudgingOfTheSameModuleOnThatMachine(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
at := time.Now()
|
||||
release := inventory.Plan{ID: "release-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{
|
||||
Gate: &inventory.PlanGate{Machines: []string{"novox"}, Carried: []inventory.CarriedMove{
|
||||
@@ -249,6 +255,7 @@ func TestAWalkWaitsForAJudgingOfTheSameModuleOnThatMachine(t *testing.T) {
|
||||
// The controller is never put back to a build that reaches less of the store's schema than the store
|
||||
// has, or to one that never said what it reaches: the current build is kept, and the condition says so.
|
||||
func TestTheControllerIsNotPutBackToABuildOlderThanTheStoresSchema(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
@@ -345,6 +352,7 @@ func TestTheControllerIsNotPutBackToABuildOlderThanTheStoresSchema(t *testing.T)
|
||||
|
||||
// The lease's holder names the build the declaration told it it is, and the version stamp only without one.
|
||||
func TestTheHolderNamesTheBuildTheDeclarationToldIt(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
t.Setenv(RunningBuildVar, " ad62528c47c7 ")
|
||||
if h := holderOf("x"); h.Build != "ad62528c47c7" {
|
||||
t.Fatalf("the holder's build is %q", h.Build)
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A tier whose module is no move — made from the source every machine running it runs, so nothing is
|
||||
// sent — is not waited on for a report after its marking (novox/hq issue 648). Its machines answered the
|
||||
// send that last carried that build before the walk got there, and nothing will make them report again:
|
||||
// the walk asks the next tier in the same step, rather than "waiting for app on anchor, laptop" for ever.
|
||||
func TestANoMoveTierWaitsForNoNewReport(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := g.open.inventory
|
||||
|
||||
// app holds the build seat, so a module of the next tier is built by it; since novox/hq ADR 0360 §3 that gates
|
||||
// nothing.
|
||||
for _, m := range []catalogue.Manifest{
|
||||
{Module: "app", Version: "c2", Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}},
|
||||
{Module: "web", Version: "w1"},
|
||||
} {
|
||||
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
|
||||
Path: "modules/" + m.Module, BuiltFrom: m.Version, Head: m.Version, Asked: time.Now().Add(-time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// What anchor and laptop were last sent (c1) was made from the source the plan's build of c2 is.
|
||||
for _, b := range []inventory.Build{
|
||||
{ID: "build-1s", Module: "app", Commit: "c1", SourceFingerprint: "src1:app",
|
||||
Asked: time.Now().Add(-30 * time.Second), At: time.Now().Add(-30 * time.Second)},
|
||||
{ID: "build-2s", Module: "app", Commit: "c2", SourceFingerprint: "src1:app", Asked: time.Now(), At: time.Now()},
|
||||
} {
|
||||
b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// Each machine answered that send, applied, before the walk's tier was built.
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.RecordDoing(ctx, nodeID(t, g.open, n), inventory.Doing{Node: n,
|
||||
Outcome: inventory.OutcomeApplied, Declared: "d-" + n + "-c1", Applied: 1,
|
||||
At: time.Now().Add(-10 * time.Second)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
built := time.Now().UTC()
|
||||
p := g.plan(t)
|
||||
p.Tiers = [][]string{{"app"}, {"web"}}
|
||||
p.Modules["app"].Build, p.Modules["app"].BuiltAt = "build-2s", &built
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
advancePlans(ctx, g.open)
|
||||
|
||||
if len(g.sent) != 0 {
|
||||
t.Fatalf("a no-move build was sent: %v", g.sent)
|
||||
}
|
||||
p = g.plan(t)
|
||||
if s := p.Modules["app"]; s == nil || !strings.HasPrefix(s.Why, "no move") {
|
||||
t.Fatalf("the build was not marked no move: %+v", s)
|
||||
}
|
||||
if p.Tier < 1 || strings.Contains(p.Note, "waiting for app") {
|
||||
t.Fatalf("the walk waits on a no-move tier for a report nothing will cause: tier %d, %s %q", p.Tier, p.State, p.Note)
|
||||
}
|
||||
|
||||
// A machine that has not answered its last send is not waited on either: a built-by edge gates no tier
|
||||
// (novox/hq ADR 0360 §3). Before, the walk waited for app on laptop.
|
||||
p.Tier, p.State, p.Note = 0, inventory.PlanRolling, ""
|
||||
if err := inv.RecordSent(ctx, nodeID(t, g.open, "laptop"), "d-laptop-later", map[string]string{"app": "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, g.open)
|
||||
p = g.plan(t)
|
||||
if p.Tier < 1 || strings.Contains(p.Note, "waiting for app") {
|
||||
t.Fatalf("the walk waits on a built-by edge: tier %d, %q", p.Tier, p.Note)
|
||||
}
|
||||
}
|
||||
@@ -43,6 +43,32 @@ const issuedAsk = 20 * time.Second
|
||||
// issuedProvider is one assignment of a module whose offer says its software issues the provision's credential.
|
||||
type issuedProvider struct {
|
||||
Node, Module, Tool string
|
||||
// Holders are the provision's holders from this assignment's machine, named to the vault, whose tool may answer
|
||||
// a value for each (issue 749).
|
||||
Holders []issuedHolder
|
||||
}
|
||||
|
||||
// issuedHolder is one holder of a provision from a provider: a consuming module on one machine (issue 749).
|
||||
type issuedHolder struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
}
|
||||
|
||||
func (h issuedHolder) String() string { return h.Module + " on " + h.Node }
|
||||
|
||||
// issuedHoldersFrom are the provision's holders from provider, each consuming module on each machine once.
|
||||
func issuedHoldersFrom(holders []inventory.Holder, provider string) []issuedHolder {
|
||||
seen := map[issuedHolder]bool{}
|
||||
var out []issuedHolder
|
||||
for _, h := range holders {
|
||||
one := issuedHolder{Node: h.Consumer, Module: h.ConsumerModule}
|
||||
if h.Provider != provider || seen[one] {
|
||||
continue
|
||||
}
|
||||
seen[one] = true
|
||||
out = append(out, one)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// issuedTaker is the line's work, with what it reaches handed in: the store, the bus and the sends.
|
||||
@@ -53,6 +79,10 @@ type issuedTaker struct {
|
||||
keep func(provision, provider, value string) (inventory.IssuedKept, error)
|
||||
record func(act link.HandAct) error
|
||||
send func(machines []string) error
|
||||
// ofTheVault says whether a provider module's secrets are filled the new way; for one that is, the value goes
|
||||
// into the vault (vault, vault_secrets.go) and nothing is kept, sealed or sent here. Nil is none.
|
||||
ofTheVault vaultSwitch
|
||||
vault vaultAsker
|
||||
}
|
||||
|
||||
// take asks every provider of the provision for its software's value and keeps each; it answers what it did, in
|
||||
@@ -90,6 +120,15 @@ func (t issuedTaker) take(provision string) (string, error) {
|
||||
|
||||
// takeFrom is one provider: asked, opened, kept, recorded and sent when it changed.
|
||||
func (t issuedTaker) takeFrom(provision string, p issuedProvider) (string, error) {
|
||||
if t.ofTheVault != nil {
|
||||
ours, err := t.ofTheVault(p.Module)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if ours {
|
||||
return t.takeIntoVault(provision, p)
|
||||
}
|
||||
}
|
||||
public, private, err := secrets.Keypair()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("no key could be made to take the value: %w", err)
|
||||
@@ -174,6 +213,16 @@ func issuedProvidersOf(ctx context.Context, inv *inventory.Inventory, provision
|
||||
if err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
out, err := issuedProvidersFrom(shelf, holders, func(node string) ([]string, error) {
|
||||
return inv.Assigned(ctx, node)
|
||||
}, provision)
|
||||
return out, true, err
|
||||
}
|
||||
|
||||
// issuedProvidersFrom are the assignments to ask, from the provision's holders: on every machine providing it, the
|
||||
// assigned modules whose offer says their software issues it, each with that machine's holders (issue 749).
|
||||
func issuedProvidersFrom(shelf map[string]catalogue.Manifest, holders []inventory.Holder,
|
||||
assignedOn func(node string) ([]string, error), provision string) ([]issuedProvider, error) {
|
||||
nodes := map[string]bool{}
|
||||
for _, h := range holders {
|
||||
nodes[h.Provider] = true
|
||||
@@ -185,13 +234,17 @@ func issuedProvidersOf(ctx context.Context, inv *inventory.Inventory, provision
|
||||
sort.Strings(names)
|
||||
var out []issuedProvider
|
||||
for _, node := range names {
|
||||
assigned, err := inv.Assigned(ctx, node)
|
||||
assigned, err := assignedOn(node)
|
||||
if err != nil {
|
||||
return nil, true, err
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, issuedOn(shelf, node, assigned, provision)...)
|
||||
on := issuedOn(shelf, node, assigned, provision)
|
||||
for i := range on {
|
||||
on[i].Holders = issuedHoldersFrom(holders, node)
|
||||
}
|
||||
out = append(out, on...)
|
||||
}
|
||||
return out, true, nil
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// offersIssued is whether any module offers the provision with a credential its software issues.
|
||||
@@ -262,7 +315,9 @@ func secretIssued(ctx context.Context, args []string) error {
|
||||
return err
|
||||
})
|
||||
},
|
||||
send: func(machines []string) error { return sendTo(ctx, open, machines) },
|
||||
send: func(machines []string) error { return sendTo(ctx, open, machines) },
|
||||
ofTheVault: filledTheNewWay,
|
||||
vault: theVault(ctx),
|
||||
}
|
||||
words, err := t.take(provision)
|
||||
if words != "" {
|
||||
|
||||
@@ -193,7 +193,7 @@ func TestTheProvidersAskedAreTheOffersThatSaySo(t *testing.T) {
|
||||
"jellyfin": {Module: "jellyfin", Provides: []catalogue.Offer{{Name: "plex-api"}}},
|
||||
}
|
||||
got := issuedOn(shelf, "ace", []string{"jellyfin", "plex", "kometa"}, "plex-api")
|
||||
if len(got) != 1 || got[0] != (issuedProvider{Node: "ace", Module: "plex", Tool: "plex_issued_credential"}) {
|
||||
if len(got) != 1 || got[0].Node != "ace" || got[0].Module != "plex" || got[0].Tool != "plex_issued_credential" {
|
||||
t.Fatalf("the providers asked: %+v", got)
|
||||
}
|
||||
if err := issuedRefusesRotation(shelf, "plex-api"); err == nil || !strings.Contains(err.Error(), "secret issued plex-api") {
|
||||
@@ -216,3 +216,84 @@ func TestAnIssuedValueIsADecisionInTheHandActLog(t *testing.T) {
|
||||
}
|
||||
t.Fatal("secret issued is not in the hand-act table")
|
||||
}
|
||||
|
||||
// **A need's credential is the vault's for its holder alone** (issue 757) exactly when its providing module is marked
|
||||
// moved, so `secret issued` goes to the vault naming each holder, and its software issues the provision's credential.
|
||||
func TestANeedIsIssuedToItsHolderOnlyWhereTheProviderIsMovedAndItsSoftwareIssuesIt(t *testing.T) {
|
||||
forge := func(moved bool, issued bool) catalogue.Manifest {
|
||||
offer := catalogue.Offer{Name: "gitea-read"}
|
||||
if issued {
|
||||
offer.Credential = &catalogue.OfferCredential{IssuedBy: catalogue.IssuedBySoftware, Tool: "gitea_agent_token"}
|
||||
}
|
||||
return catalogue.Manifest{Module: "gitea", Moved: moved, Provides: []catalogue.Offer{offer}}
|
||||
}
|
||||
shared := func(moved bool) catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "gitea", Moved: moved, OwnSecrets: catalogue.OwnSecrets{"key": {Path: "/k"}},
|
||||
Provides: []catalogue.Offer{{Name: "gitea-read", Credential: &catalogue.OfferCredential{Own: "key"}}}}
|
||||
}
|
||||
given := func() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "gitea", Moved: true, Provides: []catalogue.Offer{{Name: "gitea-read",
|
||||
Credential: &catalogue.OfferCredential{IssuedBy: catalogue.IssuedOutside}}}}
|
||||
}
|
||||
need := catalogue.Needed{Name: "gitea-read", From: "novox", Module: "gitea", For: "claude-code"}
|
||||
for why, c := range map[string]struct {
|
||||
shelf map[string]catalogue.Manifest
|
||||
need catalogue.Needed
|
||||
want bool
|
||||
}{
|
||||
"moved and issued by its software": {map[string]catalogue.Manifest{"gitea": forge(true, true)}, need, true},
|
||||
"not moved": {map[string]catalogue.Manifest{"gitea": forge(false, true)}, need, false},
|
||||
"a login for each consumer": {map[string]catalogue.Manifest{"gitea": forge(true, false)}, need, true},
|
||||
"a login for each, not moved": {map[string]catalogue.Manifest{"gitea": forge(false, false)}, need, false},
|
||||
"a credential every holder shares": {map[string]catalogue.Manifest{"gitea": shared(true)}, need, false},
|
||||
"a credential a person gives": {map[string]catalogue.Manifest{"gitea": given()}, need, false},
|
||||
"a provider not on the shelf": {map[string]catalogue.Manifest{}, need, false},
|
||||
"answered by a record": {map[string]catalogue.Manifest{"gitea": forge(true, true)},
|
||||
catalogue.Needed{Name: "gitea-read", ByRecord: true, Module: "gitea", For: "claude-code"}, false},
|
||||
} {
|
||||
if got := issuedToHolder(c.shelf, c.need); got != c.want {
|
||||
t.Errorf("%s: issued to its holder is %v, want %v", why, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **The vault is a moved provider for its consumers' credentials** (issue 793): its pair rows are the vault's since
|
||||
// V3, so a need of `secret` is issued to its holder and answered by a moved provider, whether or not the module is
|
||||
// marked moved; another module is not.
|
||||
func TestTheModuleHoldingTheVaultSeatCountsAsMovedForItsConsumers(t *testing.T) {
|
||||
vault := catalogue.Manifest{Module: "mesh-vault", Claims: []catalogue.Claim{{Name: "mesh-vault"}},
|
||||
Provides: []catalogue.Offer{{Name: "secret"}}}
|
||||
other := catalogue.Manifest{Module: "mesh-vault", Provides: []catalogue.Offer{{Name: "secret"}}}
|
||||
need := catalogue.Needed{Name: "secret", Local: "admin", From: "novox", Module: "mesh-vault", For: "gitea"}
|
||||
if !issuedToHolder(map[string]catalogue.Manifest{"mesh-vault": vault}, need) ||
|
||||
!providerMoved(map[string]catalogue.Manifest{"mesh-vault": vault}, need) {
|
||||
t.Error("the vault's consumer is not answered by a moved provider")
|
||||
}
|
||||
if issuedToHolder(map[string]catalogue.Manifest{"mesh-vault": other}, need) ||
|
||||
providerMoved(map[string]catalogue.Manifest{"mesh-vault": other}, need) {
|
||||
t.Error("a module not holding the vault seat counts as moved")
|
||||
}
|
||||
}
|
||||
|
||||
// **A provision answered by the helper makes its provider a moved one for that provision alone** (issue 792): the
|
||||
// provides entry says `"answered-by": "helper"`, the module need not be marked moved, and no credential is the vault's
|
||||
// for it. Another provision of the module, or a module without the entry, does not count.
|
||||
func TestAnOfferAnsweredByTheHelperCountsAsMovedForThatProvisionOnly(t *testing.T) {
|
||||
proxy := catalogue.Manifest{Module: "route-proxy", Provides: []catalogue.Offer{
|
||||
{Name: "route", AnsweredBy: catalogue.AnsweredByHelper}, {Name: "other"}}}
|
||||
plain := catalogue.Manifest{Module: "route-proxy", Provides: []catalogue.Offer{{Name: "route"}}}
|
||||
shelf := func(m catalogue.Manifest) map[string]catalogue.Manifest {
|
||||
return map[string]catalogue.Manifest{m.Module: m}
|
||||
}
|
||||
route := catalogue.Needed{Name: "route", From: "novox", Module: "route-proxy", For: "gitea"}
|
||||
other := catalogue.Needed{Name: "other", From: "novox", Module: "route-proxy", For: "gitea"}
|
||||
if !providerMoved(shelf(proxy), route) {
|
||||
t.Error("a provision answered by the helper is not answered by a moved provider")
|
||||
}
|
||||
if providerMoved(shelf(proxy), other) || providerMoved(shelf(plain), route) {
|
||||
t.Error("a provision not answered by the helper counts as moved")
|
||||
}
|
||||
if issuedToHolder(shelf(proxy), route) {
|
||||
t.Error("the flag made a credential the vault's")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A covered account is handed to the vault when it is minted (novox/hq ADR 0374, issue 787).
|
||||
//
|
||||
// **Covered** are the bus accounts `module issue` mints for a core module (catalogue.CoreCredentialModules) or for a
|
||||
// module marked moved; no other account is. Until the vault makes bus accounts (ADR 0113 clause 6, issue 774) the
|
||||
// controller mints such an account and writes its hash to the bus's user list, as before, and then hands the whole
|
||||
// value at once to the vault's `accept` with origin `issued`: sealed here to the key the vault made for that call, so
|
||||
// the vault alone opens it (ADR 0349 clause 8). The vault keeps it as the module's own secret on that machine and
|
||||
// raises `secret-changed`; a re-issue is the same hand with a new value, which replaces the old in place. An account
|
||||
// not covered is minted and kept as it was and handed to nobody.
|
||||
|
||||
// vaultAskerForIssue is the vault as `module issue` asks it. A test replaces it.
|
||||
var vaultAskerForIssue = func(ctx context.Context) vaultAsker { return theVault(ctx) }
|
||||
|
||||
// accountIsCovered says whether the bus accounts minted for m are the vault's to hold (ADR 0374 clause 1).
|
||||
func accountIsCovered(m catalogue.Manifest) bool {
|
||||
return catalogue.CarriesVaultSealed(m.Module) || m.Moved
|
||||
}
|
||||
|
||||
// vaultInstalled says whether the vault is assigned to any machine. Until it is, as in genesis (ADR 0113 §15 to §18,
|
||||
// mesh-host's bootstrap, which issues the bus's and the provisioner's accounts before the vault step), there is no
|
||||
// vault to hand to: the value is kept in the controller's store as before, and the vault takes it afterwards by the
|
||||
// move (move-in takes a value of the controller's store with origin made, accepted or issued, as it is). Only a
|
||||
// vault that is installed and does not answer fails the issue.
|
||||
func vaultInstalled(ctx context.Context, inv *inventory.Inventory) (bool, error) {
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Manifest.Module == "mesh-vault" && len(e.On) > 0 {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// handIssuedToVault hands the value just minted for module's own secret name on node to the vault, when the module's
|
||||
// accounts are covered, and does nothing otherwise. Loud when the vault does not take it: the bus's user list
|
||||
// already holds the new hash, so a vault that kept the old value would serve a password that no longer logs in.
|
||||
func handIssuedToVault(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node, name, held string) error {
|
||||
if !accountIsCovered(m) {
|
||||
return nil
|
||||
}
|
||||
if installed, err := vaultInstalled(ctx, inv); err != nil {
|
||||
return fmt.Errorf("whether the vault is installed cannot be read, so %s of %s was not handed to it: %w", name,
|
||||
m.Module, err)
|
||||
} else if !installed {
|
||||
return nil
|
||||
}
|
||||
target := ownSecretArgs(node, m.Module, name)
|
||||
target["origin"] = "issued"
|
||||
if _, err := acceptIntoVault(vaultAskerForIssue(ctx), target, held, "module issue"); err != nil {
|
||||
return fmt.Errorf("the credential is minted and kept here, and the bus's user list holds its hash, but the "+
|
||||
"vault did not take %s of %s on %s: %w (`module issue` again hands a new one)", name, m.Module, node, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// Every other test mints without a vault to hand to: a stand-in that takes anything, replaced by the tests below.
|
||||
func init() {
|
||||
vaultAskerForIssue = func(context.Context) vaultAsker { return newStandInVault(nil).ask }
|
||||
}
|
||||
|
||||
// A covered account is handed to the vault when it is minted (novox/hq ADR 0374, issue 787): the whole value, sealed
|
||||
// to the key the vault made for the call, with origin issued. A module that is neither core nor moved hands nothing.
|
||||
func TestModuleIssueHandsACoveredAccountToTheVaultAndOnlyThat(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
theVaultIsInstalled(t, open)
|
||||
inv := open.inventory
|
||||
known := broker.Broker{Fingerprint: "ab"}
|
||||
|
||||
core := aProvisioner()
|
||||
core.Module = "nats"
|
||||
moved := aBot("moved-bot", true)
|
||||
moved.OwnSecrets["broker"] = catalogue.OwnSecret{Path: "/var/lib/moved-bot/broker"}
|
||||
plain := aBot("kept-bot", false)
|
||||
plain.OwnSecrets["broker"] = catalogue.OwnSecret{Path: "/var/lib/kept-bot/broker"}
|
||||
|
||||
vault := newStandInVault(t)
|
||||
vaultAskerForIssue = func(context.Context) vaultAsker { return vault.ask }
|
||||
t.Cleanup(func() {
|
||||
vaultAskerForIssue = func(context.Context) vaultAsker { return newStandInVault(nil).ask }
|
||||
})
|
||||
|
||||
for _, m := range []catalogue.Manifest{core, moved, plain} {
|
||||
register(t, open, m)
|
||||
if _, err := assign(ctx, open, "laptop", m.Module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
issue := func(m catalogue.Manifest, password string) {
|
||||
t.Helper()
|
||||
if err := issueWith(ctx, inv, m, "laptop", "", known, "10.0.0.1:4222", "module."+m.Module+".laptop", password); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
issue(core, "first-core-password")
|
||||
issue(moved, "first-moved-password")
|
||||
issue(plain, "kept-password")
|
||||
if err := issueProvisionerClient(ctx, inv, core, "laptop", known, "10.0.0.1:4222"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
said := vault.said()
|
||||
for _, want := range []string{`"module":"nats"`, `"module":"moved-bot"`, `"name":"broker"`,
|
||||
`"name":"` + broker.ProvisionerSecret + `"`, `"node":"laptop"`} {
|
||||
if !strings.Contains(said, want) {
|
||||
t.Errorf("the vault was told %s, without %s", said, want)
|
||||
}
|
||||
}
|
||||
if n := strings.Count(said, `"origin":"issued"`); n != 3 {
|
||||
t.Errorf("%d accounts were handed with origin issued, want 3 (the core's two and the moved module's): %s", n, said)
|
||||
}
|
||||
if strings.Contains(said, "kept-bot") {
|
||||
t.Errorf("an account that is neither core nor moved was handed to the vault: %s", said)
|
||||
}
|
||||
if strings.Contains(said, "first-core-password") || strings.Contains(said, "first-moved-password") {
|
||||
t.Errorf("a value crossed the bus in the clear: %s", said)
|
||||
}
|
||||
|
||||
// A re-issue hands the new value, which the vault opens: the whole {url, fingerprint, node, user, password}.
|
||||
issue(moved, "second-moved-password")
|
||||
if got := vault.taken["accept"]; !strings.Contains(got, "second-moved-password") ||
|
||||
!strings.Contains(got, `"user":"module.moved-bot.laptop"`) || !strings.Contains(got, `"node":"laptop"`) ||
|
||||
!strings.Contains(got, `"url":"nats://10.0.0.1:4222"`) || !strings.Contains(got, `"fingerprint":"ab"`) {
|
||||
t.Errorf("the vault was handed %q after a re-issue, not the new account whole", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A vault that does not take the value fails the issue loudly: the bus's user list holds the new hash already.
|
||||
func TestModuleIssueFailsWhenTheVaultDoesNotTakeACoveredAccount(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
moved := aBot("moved-bot", true)
|
||||
moved.OwnSecrets["broker"] = catalogue.OwnSecret{Path: "/var/lib/moved-bot/broker"}
|
||||
theVaultIsInstalled(t, open)
|
||||
register(t, open, moved)
|
||||
if _, err := assign(ctx, open, "laptop", moved.Module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
vault := newStandInVault(t)
|
||||
vault.refuse = errors.New("the vault is not there")
|
||||
vaultAskerForIssue = func(context.Context) vaultAsker { return vault.ask }
|
||||
t.Cleanup(func() {
|
||||
vaultAskerForIssue = func(context.Context) vaultAsker { return newStandInVault(nil).ask }
|
||||
})
|
||||
err := issueWith(ctx, open.inventory, moved, "laptop", "", broker.Broker{Fingerprint: "ab"}, "10.0.0.1:4222",
|
||||
"module.moved-bot.laptop", "pw")
|
||||
if err == nil || !strings.Contains(err.Error(), "vault did not take") {
|
||||
t.Fatalf("a vault that took nothing was reported as %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func theVaultIsInstalled(t *testing.T, open *stores) {
|
||||
t.Helper()
|
||||
vault := catalogue.Manifest{Module: "mesh-vault", Version: "1"}
|
||||
register(t, open, vault)
|
||||
if _, err := assign(t.Context(), open, "laptop", vault.Module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// In genesis the bus's and the provisioner's accounts are issued before the vault is installed (ADR 0113 §15 to §18):
|
||||
// there is no vault to hand to, so nothing is asked and nothing fails, and the value stays in the controller's store
|
||||
// as before, for the move to carry into the vault. Once the vault is installed, one that does not answer fails.
|
||||
func TestModuleIssueBeforeTheVaultIsInstalledKeepsTheValueAndHandsNothing(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
known := broker.Broker{Fingerprint: "ab"}
|
||||
core := aProvisioner()
|
||||
core.Module = "nats"
|
||||
register(t, open, core)
|
||||
if _, err := assign(ctx, open, "laptop", core.Module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
vault := newStandInVault(t)
|
||||
vault.refuse = errors.New("there is no vault yet")
|
||||
vaultAskerForIssue = func(context.Context) vaultAsker { return vault.ask }
|
||||
t.Cleanup(func() {
|
||||
vaultAskerForIssue = func(context.Context) vaultAsker { return newStandInVault(nil).ask }
|
||||
})
|
||||
|
||||
if err := issueWith(ctx, inv, core, "laptop", "", known, "10.0.0.1:4222", "module.nats.laptop", "pw"); err != nil {
|
||||
t.Fatalf("genesis's issue of the bus's account failed with no vault installed: %v", err)
|
||||
}
|
||||
if err := issueProvisionerClient(ctx, inv, core, "laptop", known, "10.0.0.1:4222"); err != nil {
|
||||
t.Fatalf("genesis's issue of the client's account failed with no vault installed: %v", err)
|
||||
}
|
||||
if said := vault.said(); said != "" {
|
||||
t.Errorf("the vault was asked before it was installed: %s", said)
|
||||
}
|
||||
if _, held, err := inv.ModuleSecretIfIssued(ctx, "laptop", "nats", "broker"); err != nil || !held {
|
||||
t.Errorf("the value is not kept in the controller's store (held %v, %v)", held, err)
|
||||
}
|
||||
|
||||
theVaultIsInstalled(t, open)
|
||||
if err := issueWith(ctx, inv, core, "laptop", "", known, "10.0.0.1:4222", "module.nats.laptop", "pw2"); err == nil {
|
||||
t.Error("a vault that is installed and does not answer did not fail the issue")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0360 §4, §5: a walk ends on its machines' reports — every machine sent has reported on its send,
|
||||
// or is silent past ApplySilentAfter — and names a failed or refused apply; it never waits for a health reading.
|
||||
func TestAWalkEndsOnItsMachinesReports(t *testing.T) {
|
||||
now := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC)
|
||||
sent := now.Add(-time.Minute)
|
||||
long := now.Add(-inventory.ApplySilentAfter - time.Minute)
|
||||
at := now.Add(-10 * time.Second)
|
||||
p := inventory.Plan{Modules: map[string]*inventory.PlanModule{
|
||||
"app": {Rest: map[string]inventory.SentDeclaration{
|
||||
"ace": {Digest: "d-ace", At: sent},
|
||||
"novox": {Digest: "d-novox", At: sent},
|
||||
"shanks": {Digest: "d-shanks", At: sent},
|
||||
"laptop": {Digest: "d-laptop", At: long},
|
||||
}},
|
||||
"proxy": {Rest: map[string]inventory.SentDeclaration{"ace": {Digest: "d-ace", At: sent}}},
|
||||
}}
|
||||
reports := []inventory.Reported{
|
||||
{Node: "ace", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-ace"},
|
||||
{Node: "novox", Outcome: inventory.OutcomeFailed, At: &at, Declared: "d-novox"},
|
||||
{Node: "shanks", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-before"},
|
||||
}
|
||||
waiting, failed, silent := walkReports(p, reports, now, nil)
|
||||
if !slices.Equal(waiting, []string{"shanks"}) || !slices.Equal(failed, []string{"novox"}) ||
|
||||
!slices.Equal(silent, []string{"laptop"}) {
|
||||
t.Fatalf("waiting %v, failed %v, silent %v", waiting, failed, silent)
|
||||
}
|
||||
reports[2].Declared = "d-shanks"
|
||||
if waiting, _, _ := walkReports(p, reports, now, nil); len(waiting) != 0 {
|
||||
t.Fatalf("the walk still waits once every machine reported: %v", waiting)
|
||||
}
|
||||
}
|
||||
|
||||
// withTheGate turns the first-machine gate on for a test of its own path: kept for the walks a controller before
|
||||
// novox/hq ADR 0360 started.
|
||||
func withTheGate(t *testing.T) {
|
||||
t.Helper()
|
||||
was := gateFirstMachine
|
||||
gateFirstMachine = true
|
||||
t.Cleanup(func() { gateFirstMachine = was })
|
||||
}
|
||||
|
||||
// Every machine heard from is sent at once and passed on its applied report, with no judging (novox/hq ADR 0360 §7);
|
||||
// before, the first machine was judged before the rest were sent (ADR 0353 §1).
|
||||
func TestAReleaseSendsEveryMachineAtOnce(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklogOnThree(t)
|
||||
ctx := t.Context()
|
||||
said := events(t)
|
||||
for i := 0; i < 12; i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
p := b.release(t)
|
||||
if p.State != inventory.PlanDone || len(p.Release.Done) != 3 {
|
||||
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
for _, n := range []string{"desk", "laptop", "anchor"} {
|
||||
if !slices.Contains(*said, "send "+n) {
|
||||
t.Fatalf("%s was not sent: %v", n, *said)
|
||||
}
|
||||
}
|
||||
if slices.Contains(*said, "judge") {
|
||||
t.Fatalf("a machine was judged by its health: %v", *said)
|
||||
}
|
||||
if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine that reports its apply failed is named and fails the release, with nothing put back, and every other
|
||||
// machine goes on to its own report (novox/hq ADR 0360 §5, §7); before, it was put back (ADR 0353 §3).
|
||||
func TestAMachineThatFailsItsApplyIsNamedAndNothingIsPutBack(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklogOnThree(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
b.broken["desk"] = true // desk fails what it is sent
|
||||
advancePlans(ctx, b.open)
|
||||
for i := 0; i < 12 && b.release(t).Open(); i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
p := b.release(t)
|
||||
if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) {
|
||||
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
if !slices.Contains(p.Release.Done, "laptop") || !slices.Contains(p.Release.Done, "anchor") {
|
||||
t.Fatalf("the others have no verdict of their own: %+v", p.Release)
|
||||
}
|
||||
if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c2" {
|
||||
t.Fatalf("desk was put back: %v", sent)
|
||||
}
|
||||
}
|
||||
|
||||
// A release's send kept before it was made, and made, is judged by the controller that takes over without being
|
||||
// made again (ADR 0353 §8), with every machine sent at once (ADR 0360 §7).
|
||||
func TestAReleaseSendKeptAndMadeIsNotMadeAgainAtOnce(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
p, err := releaseBacklog(ctx, b.open, "")
|
||||
if err != nil || p == nil {
|
||||
t.Fatalf("no release opened: %v", err)
|
||||
}
|
||||
if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gates["anchor"] == nil || !p.Release.Gates["anchor"].Pending {
|
||||
t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err)
|
||||
}
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The send is made, and the controller is replaced before it keeps it.
|
||||
if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, b.open)
|
||||
if n := len(slices.DeleteFunc(slices.Clone(b.sent), func(s []string) bool { return !slices.Equal(s, []string{"anchor"}) })); n != 1 {
|
||||
t.Fatalf("anchor was sent again: %v", b.sent)
|
||||
}
|
||||
g := b.release(t).Release.Gates["anchor"]
|
||||
if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 {
|
||||
t.Fatalf("anchor is not judged on the send made: %+v", g)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put
|
||||
// back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4); its step also sent anchor, so the release is done, not failed (ADR 0360 §7).
|
||||
func TestAMachineSilentPastItsBoundIsLeftAndSaidAtOnce(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
reporting := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing
|
||||
b.sent = append(b.sent, names)
|
||||
current, _ := open.inventory.CurrentBuilds(ctx)
|
||||
return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep",
|
||||
map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit})
|
||||
}
|
||||
return reporting(ctx, open, names)
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = reporting })
|
||||
for i := 0; i < 8; i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
p := b.release(t)
|
||||
g := p.Release.Gates["laptop"]
|
||||
if !p.Open() || g == nil || g.Verdict != "" || g.Silent {
|
||||
t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g)
|
||||
}
|
||||
since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute)
|
||||
g.Since = &since
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, b.open)
|
||||
p = b.release(t)
|
||||
// anchor applied in the same step, so the release is done, laptop named silent (novox/hq ADR 0360 §7).
|
||||
if p.State != inventory.PlanDone || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 {
|
||||
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed {
|
||||
t.Fatal("a silent machine's build was marked failed")
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" {
|
||||
t.Fatalf("laptop was put back: %v", sent)
|
||||
}
|
||||
obs, err := silentObservation(ctx, inv)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) {
|
||||
t.Fatalf("the silence is not said: %+v", obs)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 840 (novox/hq ADR 0362 §1-5): a walk's end counts a machine's send of a provisioned module applied only once
|
||||
// the engine and the machine's provisioner client both said so.
|
||||
func TestAWalkEndWaitsForTheProvisioner(t *testing.T) {
|
||||
now := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)
|
||||
at := now.Add(-10 * time.Second)
|
||||
plan := func(sentAt time.Time) inventory.Plan {
|
||||
return inventory.Plan{Modules: map[string]*inventory.PlanModule{"app": {Rest: map[string]inventory.SentDeclaration{
|
||||
"ace": {Digest: "d-ace", At: sentAt, Provisioned: map[string]string{"app": "t1"}},
|
||||
"g14": {Digest: "d-g14", At: sentAt, Provisioned: map[string]string{"app": "t1"}},
|
||||
}}}}
|
||||
}
|
||||
reports := []inventory.Reported{
|
||||
{Node: "ace", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-ace"},
|
||||
{Node: "g14", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-g14"},
|
||||
}
|
||||
says := map[string]inventory.ProvReport{}
|
||||
prov := func(node, module string) (inventory.ProvReport, bool) { r, ok := says[node+"/"+module]; return r, ok }
|
||||
applied := inventory.ProvReport{Status: inventory.ProvApplied, Templates: "t1", At: at}
|
||||
|
||||
// (a) the engine applied, the client has not said: the end waits on both.
|
||||
waiting, failed, silent := walkReports(plan(now.Add(-time.Minute)), reports, now, prov)
|
||||
if !slices.Equal(waiting, []string{"ace", "g14"}) || len(failed)+len(silent) != 0 {
|
||||
t.Fatalf("a: waiting %v, failed %v, silent %v", waiting, failed, silent)
|
||||
}
|
||||
// ace's client applied, g14's failed on that build (b): ace done, g14 a failed apply naming the reason.
|
||||
says["ace/app"] = applied
|
||||
says["g14/app"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t1", Reason: "no value", At: at}
|
||||
waiting, failed, silent = walkReports(plan(now.Add(-time.Minute)), reports, now, prov)
|
||||
if len(waiting) != 0 || len(silent) != 0 || len(failed) != 1 || !strings.HasPrefix(failed[0], "g14 (") ||
|
||||
!strings.Contains(failed[0], "no value") {
|
||||
t.Fatalf("b: waiting %v, failed %v, silent %v", waiting, failed, silent)
|
||||
}
|
||||
// (c) no client report past ApplySilentAfter: named silent, never applied.
|
||||
delete(says, "g14/app")
|
||||
waiting, failed, silent = walkReports(plan(now.Add(-inventory.ApplySilentAfter-time.Minute)), reports, now, prov)
|
||||
if len(waiting) != 0 || len(failed) != 0 || !slices.Equal(silent, []string{"g14"}) {
|
||||
t.Fatalf("c: waiting %v, failed %v, silent %v", waiting, failed, silent)
|
||||
}
|
||||
// both said it: nothing waits.
|
||||
says["g14/app"] = applied
|
||||
if waiting, failed, silent = walkReports(plan(now.Add(-time.Minute)), reports, now, prov); len(waiting)+len(failed)+len(silent) != 0 {
|
||||
t.Fatalf("both said: waiting %v, failed %v, silent %v", waiting, failed, silent)
|
||||
}
|
||||
// (d) a send handing nothing to the provisioner is the engine's report alone, as before.
|
||||
bare := plan(now.Add(-time.Minute))
|
||||
bare.Modules["app"].Rest = map[string]inventory.SentDeclaration{"ace": {Digest: "d-ace", At: now.Add(-time.Minute)}}
|
||||
if waiting, failed, silent = walkReports(bare, reports, now, func(string, string) (inventory.ProvReport, bool) {
|
||||
return inventory.ProvReport{}, false
|
||||
}); len(waiting)+len(failed)+len(silent) != 0 {
|
||||
t.Fatalf("d: waiting %v, failed %v, silent %v", waiting, failed, silent)
|
||||
}
|
||||
}
|
||||
@@ -106,6 +106,8 @@ func run() error {
|
||||
return killCommand(ctx, args[1:])
|
||||
case "forget-checks":
|
||||
return forgetChecksCommand(ctx, args[1:])
|
||||
case "try":
|
||||
return tryCommand(ctx, args[1:])
|
||||
case "pause", "resume":
|
||||
return pauseCommand(ctx, args[0], args[1:])
|
||||
case "collection":
|
||||
@@ -119,6 +121,10 @@ func run() error {
|
||||
return imagesCommand(ctx, args[1:])
|
||||
case "mirrors":
|
||||
return mirrorsCommand(ctx, args[1:])
|
||||
case "move-placement":
|
||||
return movePlacementCommand(ctx, args[1:])
|
||||
case "move-config":
|
||||
return moveConfigCommand(ctx, args[1:])
|
||||
case "plans":
|
||||
return plansCommand(ctx, args[1:])
|
||||
case "delivery":
|
||||
@@ -273,6 +279,10 @@ func usage() {
|
||||
artifacts [--json] [--repository <r>] [--collected] every artifact the mesh made: kept and why, eligible, collected (ADR 0251)
|
||||
collect [--json] [--most <n>] what the store's sweep would let go of, now; nothing is done
|
||||
collect --confirm --why <text> [--most <n>] run the sweep now: hold every kept archive, let go of the eligible
|
||||
move-placement [--json] which settings layers still hold placement keys (endpoints, expose, reach, ports); nothing is moved
|
||||
move-placement --confirm --why <text> move them beside the assignment, once, after the delivery (issue 736)
|
||||
move-config [--json] every config value the settings layers hold, and where in mesh-config it goes; nothing is set
|
||||
move-config --confirm --why <text> set them in mesh-config, read them back, and read config from there (issue 664)
|
||||
images <node> [--json] every container image the machine's declaration names, now and as last sent
|
||||
seats [--json] every seat this mesh defines, what it delivers, and who holds it
|
||||
seat rename <from> <to> rename a seat; its former name still resolves (ADR 0122)
|
||||
@@ -329,7 +339,7 @@ func usage() {
|
||||
licence add|list|use|key model access, under the name a person calls it
|
||||
licence manager <name> <node> the node that holds a refreshable licence's refresh token
|
||||
licence refresh <name> mint a new access token and seal it to every holder
|
||||
rotate <provision> [--consumer <n>] [--module <m>] a new credential for every holder, both ends at once
|
||||
rotate <provision> [--consumer <n>] [--module <m>] [--why <text>] a new credential for every holder, both ends at once (--why where the vault holds it)
|
||||
ask <module> <tool> [json] call one of a module's tools over the broker, and print its answer
|
||||
pin <node> <provision> <from-node> <module>
|
||||
which provider this one gets a provision from: the module, and its node
|
||||
@@ -381,6 +391,13 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
||||
// **A merge check builds nothing** (novox/hq to-be 45 §9): its verdict is said, and nothing of it is
|
||||
// recorded or registered.
|
||||
if result.Check != nil || result.Checked != nil {
|
||||
if isTry(result) {
|
||||
// A branch tried before its pull request (novox/hq issue 686): its asker reads the outcome from the
|
||||
// build's log. Never said as `checked`, so the forge's holder sets no status.
|
||||
fmt.Printf("%s: a try of %s at %.8s, read with `try --id`: said to nobody, nothing taken in\n", result.ID,
|
||||
result.Repository, result.Ref)
|
||||
return nil
|
||||
}
|
||||
checked(ctx, result)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -550,8 +550,17 @@ func shelfWithChange(base map[string]catalogue.Manifest, sources map[string]inve
|
||||
out[k] = m
|
||||
}
|
||||
var failures []string
|
||||
// The list of predefined keys at the tree's root (novox/hq ADR 0348 §7): what the build of this tree judges each
|
||||
// manifest against and registration publishes, so what the gate judges each against too. A list that is
|
||||
// wrong is refused by name, and every manifest is then judged against none.
|
||||
predefined, err := catalogue.ReadPredefined(tree)
|
||||
if err != nil {
|
||||
failures = append(failures, fmt.Sprintf("%s: the controller judging this (%s) refuses it — %s",
|
||||
catalogue.PredefinedFile, version, oneLine(err.Error())))
|
||||
predefined = nil
|
||||
}
|
||||
found := map[string]string{} // module → its directory in the tree
|
||||
err := filepath.WalkDir(tree, func(path string, d fs.DirEntry, err error) error {
|
||||
err = filepath.WalkDir(tree, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -575,7 +584,12 @@ func shelfWithChange(base map[string]catalogue.Manifest, sources map[string]inve
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
m, err := catalogue.ParseManifestWith(raw, predefined)
|
||||
if err == nil {
|
||||
// Each template's text from the tree beside the manifest, as the build reads it back from the archive it
|
||||
// packs (novox/hq ADR 0349 §2): so a change to a template alone is a change to its module.
|
||||
m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(path))
|
||||
}
|
||||
if err != nil {
|
||||
if inScope(dir) {
|
||||
failures = append(failures, fmt.Sprintf("%s: the controller judging this (%s) refuses it — %s",
|
||||
@@ -605,7 +619,7 @@ func shelfWithChange(base map[string]catalogue.Manifest, sources map[string]inve
|
||||
// As registration reads a built manifest: strictly, again (takeIn).
|
||||
var raw []byte
|
||||
if raw, err = json.Marshal(resolved); err == nil {
|
||||
resolved, err = catalogue.ParseManifest(raw)
|
||||
resolved, err = catalogue.ParseManifestWith(raw, predefined)
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
@@ -1083,6 +1097,19 @@ func raiseFromFacts(ctx context.Context, open *stores, f snapshot.Facts, shelf m
|
||||
continue
|
||||
}
|
||||
}
|
||||
// And the provisioner client's account, the same way (novox/hq ADR 0349 clause 11, design 57 P3).
|
||||
if a.Provider == "" && a.Name == broker.ProvisionerSecret {
|
||||
if pm, held := shelf[a.Module]; !held || !runsTheProvisionerClient(pm) {
|
||||
continue
|
||||
}
|
||||
if _, err := inv.MintBusPassword(ctx, inventory.BusUser{Kind: inventory.BusProvisioner, Node: m.Name,
|
||||
Module: a.Module, Username: broker.Principal{Kind: broker.KindProvisioner, Node: m.Name,
|
||||
Module: a.Module}.Username()}); err != nil {
|
||||
notes = append(notes, fmt.Sprintf("%s's provisioner client account for %s is not kept: %s",
|
||||
m.Described(), a.Module, oneLine(err.Error())))
|
||||
continue
|
||||
}
|
||||
}
|
||||
var err error
|
||||
if a.Provider == "" {
|
||||
err = inv.AcceptSecretForModule(ctx, m.Name, a.Module, a.Name, standIn)
|
||||
|
||||
@@ -33,7 +33,21 @@ import (
|
||||
func aMesh(t *testing.T) *stores {
|
||||
t.Helper()
|
||||
inventory.ForTest(t) // raises the store, migrates it, and points the environment at it
|
||||
licences.ForTest(t) // planning reaches this one too, by name and never by connection
|
||||
return aMeshOnTheStore(t)
|
||||
}
|
||||
|
||||
// aMeshBefore is aMesh on a store migrated only below migration `number`, with the function that runs the
|
||||
// rest: for a test of what a migration that moves data does to what the mesh composes (novox/hq issue 651).
|
||||
func aMeshBefore(t *testing.T, number int) (*stores, func()) {
|
||||
t.Helper()
|
||||
_, migrate := inventory.ForTestBefore(t, number)
|
||||
return aMeshOnTheStore(t), migrate
|
||||
}
|
||||
|
||||
// aMeshOnTheStore is the mesh aMesh makes, on the store the environment points at.
|
||||
func aMeshOnTheStore(t *testing.T) *stores {
|
||||
t.Helper()
|
||||
licences.ForTest(t) // planning reaches this one too, by name and never by connection
|
||||
|
||||
open, err := openStores(t.Context())
|
||||
if err != nil {
|
||||
|
||||
@@ -7,7 +7,10 @@ import (
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
@@ -123,8 +126,82 @@ func answerMeshCLI(inv *inventory.Inventory) link.CLIHandler {
|
||||
// cliJournal says one line in the controller's journal (its standard output); a variable so a test can read it.
|
||||
var cliJournal = func(line string) { fmt.Println(line) }
|
||||
|
||||
// runForMeshCLI runs a judged line and answers what it said. Every line is said in the journal first: its call,
|
||||
// who asked on which node, its command word only, and how it runs (review of ADR 0272).
|
||||
// onlyReads says a line only reads, judged by the generic `command` verb's reading forms (novox/hq ADR 0266) — the
|
||||
// same allow list that decides what an ordinary call may run, never a second list kept here — judged on the line's
|
||||
// every word, so `plans go`, `conditions silence` and `settings set` are not reads while `plans`, `conditions` and
|
||||
// `settings show` are. A line those forms do not name is not judged a read, and is said (novox/hq issue 762).
|
||||
func onlyReads(asked link.CLIAsked, v cliVerdict) bool {
|
||||
if v.refused != "" || len(asked.Line) == 0 || len(asked.Stdin) > 0 || cliServers[asked.Line[0]] {
|
||||
return false
|
||||
}
|
||||
return commandReads(asked.Line) == nil && terminalOnly(asked.Line) == nil
|
||||
}
|
||||
|
||||
// readCalls counts the read calls said in no line, per caller, until the next hourly line says them.
|
||||
type readCalls struct {
|
||||
mu sync.Mutex
|
||||
counts map[string]int
|
||||
}
|
||||
|
||||
// cliReads are this controller's read calls not yet said.
|
||||
var cliReads = &readCalls{}
|
||||
|
||||
func (r *readCalls) add(caller string) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.counts == nil {
|
||||
r.counts = map[string]int{}
|
||||
}
|
||||
r.counts[caller]++
|
||||
}
|
||||
|
||||
// said is the line that says the read calls counted since the last, and starts counting again; empty when there
|
||||
// were none, so an idle hour says nothing.
|
||||
func (r *readCalls) said(over time.Duration) string {
|
||||
r.mu.Lock()
|
||||
counts := r.counts
|
||||
r.counts = nil
|
||||
r.mu.Unlock()
|
||||
if len(counts) == 0 {
|
||||
return ""
|
||||
}
|
||||
callers := make([]string, 0, len(counts))
|
||||
for c := range counts {
|
||||
callers = append(callers, c)
|
||||
}
|
||||
sort.Strings(callers)
|
||||
parts := make([]string, len(callers))
|
||||
for i, c := range callers {
|
||||
parts[i] = fmt.Sprintf("%s %d", c, counts[c])
|
||||
}
|
||||
return fmt.Sprintf("mesh-cli read calls in the last %s, said in no line of their own: %s", over,
|
||||
strings.Join(parts, ", "))
|
||||
}
|
||||
|
||||
// readCallsEvery is how often the count of read calls is said.
|
||||
const readCallsEvery = time.Hour
|
||||
|
||||
// sayReadCalls says, every readCallsEvery until ctx ends, how many read calls each caller made — when any did — so
|
||||
// a runaway poller stays visible while its calls write no line each (novox/hq issue 762).
|
||||
func sayReadCalls(ctx context.Context) {
|
||||
tick := time.NewTicker(readCallsEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
if line := cliReads.said(readCallsEvery); line != "" {
|
||||
cliJournal(line)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// runForMeshCLI runs a judged line and answers what it said. Every line that changes something, is refused or fails
|
||||
// is said in the journal: its call, who asked on which node, its command word only, and how it runs (review of ADR
|
||||
// 0272). A line that only reads and succeeds is counted instead, and the count said hourly (novox/hq issue 762): a
|
||||
// page polling `plans` every few seconds would otherwise write thousands of lines a day.
|
||||
func runForMeshCLI(ctx context.Context, node string, asked link.CLIAsked, v cliVerdict) link.CLIAnswer {
|
||||
how := "as an ordinary call"
|
||||
switch {
|
||||
@@ -137,7 +214,23 @@ func runForMeshCLI(ctx context.Context, node string, asked link.CLIAsked, v cliV
|
||||
if call == "" {
|
||||
call = "(no call)"
|
||||
}
|
||||
cliJournal(fmt.Sprintf("mesh-cli %s: %s on %s asked %q, %s", call, asked.Account, node, asked.Line[0], how))
|
||||
line := fmt.Sprintf("mesh-cli %s: %s on %s asked %q, %s", call, asked.Account, node, asked.Line[0], how)
|
||||
if !onlyReads(asked, v) {
|
||||
cliJournal(line)
|
||||
return runJudgedLine(ctx, node, asked, v)
|
||||
}
|
||||
answer := runJudgedLine(ctx, node, asked, v)
|
||||
if answer.Exit != 0 || answer.Refused != "" {
|
||||
// A read that was refused or failed is said, as a change is.
|
||||
cliJournal(fmt.Sprintf("%s, and it failed (exit %d)", line, answer.Exit))
|
||||
} else {
|
||||
cliReads.add(fmt.Sprintf("%s on %s", asked.Account, node))
|
||||
}
|
||||
return answer
|
||||
}
|
||||
|
||||
// runJudgedLine runs a line runForMeshCLI has said or counted.
|
||||
func runJudgedLine(ctx context.Context, node string, asked link.CLIAsked, v cliVerdict) link.CLIAnswer {
|
||||
if v.refused != "" {
|
||||
return link.CLIRefusal(v.refused)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// echoFails makes the echoing test binary end with a failure, as a read that failed would.
|
||||
const echoFails = "MESH_TEST_ECHO_FAILS"
|
||||
|
||||
// journalled runs lines through runForMeshCLI and answers what the journal said, with a fresh count of read calls.
|
||||
func journalled(t *testing.T) *[]string {
|
||||
t.Helper()
|
||||
t.Setenv(echoEnvironment, "1")
|
||||
said := &[]string{}
|
||||
was, wasReads := cliJournal, cliReads
|
||||
cliJournal = func(line string) { *said = append(*said, line) }
|
||||
cliReads = &readCalls{}
|
||||
t.Cleanup(func() { cliJournal, cliReads = was, wasReads })
|
||||
return said
|
||||
}
|
||||
|
||||
// **A call that only reads writes no journal line; one that changes something does** (novox/hq issue 762).
|
||||
func TestAReadCallWritesNoJournalLine(t *testing.T) {
|
||||
said := journalled(t)
|
||||
ctx := link.WithCallID(context.Background(), "call-1")
|
||||
ordinary := cliVerdict{why: "not the terminal"}
|
||||
for _, line := range [][]string{{"plans"}, {"status"}, {"conditions"}, {"node", "list"}, {"builds"}, {"queue"},
|
||||
{"settings", "show", "zsh"}, {"plans", "--json"}} {
|
||||
if a := runForMeshCLI(ctx, "g14", cliAsked("jochen", 1000, line...), ordinary); a.Exit != 0 || a.Refused != "" {
|
||||
t.Fatalf("%q did not run: %+v", line, a)
|
||||
}
|
||||
}
|
||||
runForMeshCLI(ctx, "control", cliAsked("jochen", 1000, "plans"), cliVerdict{terminal: true, why: "the terminal"})
|
||||
if len(*said) != 0 {
|
||||
t.Fatalf("read calls wrote journal lines: %q", *said)
|
||||
}
|
||||
runForMeshCLI(ctx, "control", cliAsked("jochen", 1000, "assign", "g14", "zsh"), cliVerdict{terminal: true, why: "the terminal"})
|
||||
if len(*said) != 1 || !strings.Contains((*said)[0], `asked "assign"`) {
|
||||
t.Fatalf("a call that changes something was not said: %q", *said)
|
||||
}
|
||||
}
|
||||
|
||||
// **A refused or failed read keeps its line** (novox/hq issue 762).
|
||||
func TestAFailedOrRefusedReadIsSaid(t *testing.T) {
|
||||
said := journalled(t)
|
||||
runForMeshCLI(context.Background(), "g14", cliAsked("root", 0, "plans"), cliVerdict{refused: "root is not answered"})
|
||||
t.Setenv(echoFails, "1")
|
||||
a := runForMeshCLI(context.Background(), "g14", cliAsked("jochen", 1000, "plans"), cliVerdict{why: "not the terminal"})
|
||||
if a.Exit == 0 {
|
||||
t.Fatalf("the read did not fail: %+v", a)
|
||||
}
|
||||
if len(*said) != 2 || !strings.Contains((*said)[0], "refused") || !strings.Contains((*said)[1], "failed (exit 3)") {
|
||||
t.Fatalf("the journal said %q", *said)
|
||||
}
|
||||
if line := cliReads.said(readCallsEvery); line != "" {
|
||||
t.Fatalf("a failed read was counted as a read: %q", line)
|
||||
}
|
||||
}
|
||||
|
||||
// **A verb is classed by what the call does, not its name** (novox/hq issue 762): the same command word reads with
|
||||
// some arguments and writes with others, and a line no reading form names is said.
|
||||
func TestAVerbIsClassedByItsArguments(t *testing.T) {
|
||||
ordinary := cliVerdict{why: "not the terminal"}
|
||||
for _, c := range []struct {
|
||||
line []string
|
||||
reads bool
|
||||
}{
|
||||
{[]string{"settings", "show", "zsh"}, true},
|
||||
{[]string{"settings", "preferences"}, true},
|
||||
{[]string{"settings", "set", "zsh", `{"a":1}`}, false},
|
||||
{[]string{"settings", "clear", "zsh"}, false},
|
||||
{[]string{"plans"}, true},
|
||||
{[]string{"plans", "p-1"}, true},
|
||||
{[]string{"plans", "go", "p-1"}, false},
|
||||
{[]string{"plans", "--json", "stop", "p-1"}, false},
|
||||
{[]string{"plans", "close", "p-1"}, false},
|
||||
{[]string{"plans", "retry", "p-1"}, false},
|
||||
{[]string{"conditions"}, true},
|
||||
{[]string{"conditions", "show", "x"}, true},
|
||||
{[]string{"conditions", "silence", "x"}, false},
|
||||
{[]string{"doctor", "run"}, false},
|
||||
{[]string{"node", "account", "g14", "x"}, false},
|
||||
{[]string{"assign", "g14", "zsh"}, false},
|
||||
{[]string{"a-command-added-later"}, false},
|
||||
{[]string{"serve"}, false},
|
||||
} {
|
||||
if got := onlyReads(cliAsked("jochen", 1000, c.line...), ordinary); got != c.reads {
|
||||
t.Errorf("%q judged a read: %v, want %v", c.line, got, c.reads)
|
||||
}
|
||||
}
|
||||
if onlyReads(cliAsked("jochen", 1000, "plans"), cliVerdict{refused: "no"}) {
|
||||
t.Error("a refused line judged a read")
|
||||
}
|
||||
stdin := cliAsked("jochen", 1000, "status")
|
||||
stdin.Stdin = []byte("x")
|
||||
if onlyReads(stdin, cliVerdict{terminal: true}) {
|
||||
t.Error("a line carrying standard input judged a read")
|
||||
}
|
||||
}
|
||||
|
||||
// **Once an hour the read calls are counted per caller, only if there were any** (novox/hq issue 762).
|
||||
func TestTheReadCallsAreCountedPerCaller(t *testing.T) {
|
||||
said := journalled(t)
|
||||
ordinary := cliVerdict{why: "not the terminal"}
|
||||
for range 3 {
|
||||
runForMeshCLI(context.Background(), "g14", cliAsked("jochen", 1000, "plans"), ordinary)
|
||||
}
|
||||
runForMeshCLI(context.Background(), "ace", cliAsked("jochen", 1000, "status"), ordinary)
|
||||
if len(*said) != 0 {
|
||||
t.Fatalf("read calls wrote lines: %q", *said)
|
||||
}
|
||||
line := cliReads.said(readCallsEvery)
|
||||
if !strings.Contains(line, "jochen on ace 1, jochen on g14 3") || !strings.Contains(line, "1h0m0s") {
|
||||
t.Fatalf("the hourly line said %q", line)
|
||||
}
|
||||
if again := cliReads.said(readCallsEvery); again != "" {
|
||||
t.Fatalf("an hour with no read calls said %q", again)
|
||||
}
|
||||
}
|
||||
@@ -86,7 +86,7 @@ func owedToTheRecord(ctx context.Context, announced merges, m link.SourceMoved,
|
||||
|
||||
// catchingUpOnMerges reads back the forge's announcements on a timer, until the context ends.
|
||||
func catchingUpOnMerges(ctx context.Context, open *stores, announced merges) {
|
||||
f := following{open}
|
||||
f := following{open: open}
|
||||
catalogued := func(ctx context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
|
||||
entries, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
|
||||
@@ -240,6 +240,7 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
continue
|
||||
}
|
||||
o := moduleUnhealthyObservation(m, node, unhealthy[m])
|
||||
declaredSeverity(&o, hold, m)
|
||||
if hold != nil {
|
||||
if by, held := hold.heldWith(node, m, unhealthy[m]); held {
|
||||
// Held under the provider's condition: nothing of its own, and the provider's says it waits. The
|
||||
@@ -347,6 +348,7 @@ func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p cata
|
||||
} else {
|
||||
_, rest := besideAWait(p.Module, rs)
|
||||
o = moduleUnhealthyObservation(p.Module, p.Node, rest)
|
||||
declaredSeverity(&o, hold, p.Module)
|
||||
if said, waits := personWait(p.Module, p.Node, rest); waits {
|
||||
o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rest)
|
||||
}
|
||||
@@ -481,6 +483,19 @@ func sayWaitingOn(o *conditions.Observation, waiters []string) {
|
||||
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
|
||||
}
|
||||
|
||||
// declaredSeverity raises a module's unhealthy condition urgent when its manifest declares `unhealthy` urgent
|
||||
// (novox/hq ADR 0355 §1). Only that condition changes (ADR 0355 §4): a wait for a new login, a module that needs
|
||||
// the operator and a directory used as found keep their severities, and without the field, or without a manifest
|
||||
// to read it from, it stays the warning it was.
|
||||
func declaredSeverity(o *conditions.Observation, hold *holding, module string) {
|
||||
if o.Kind != kindModuleUnhealthy || hold == nil {
|
||||
return
|
||||
}
|
||||
if m, ok := hold.manifestOf(module); ok && m.UnhealthyIsUrgent() {
|
||||
o.Severity = conditions.Urgent
|
||||
}
|
||||
}
|
||||
|
||||
// moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module,
|
||||
// the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence.
|
||||
func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -127,6 +128,10 @@ func moduleCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// A manifest handed over by hand takes each template's text from the tree beside it (novox/hq ADR 0349 §2).
|
||||
if m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(positionals[0])); err != nil {
|
||||
return err
|
||||
}
|
||||
from, err := whereItComesFrom(*repo, *ref, *commit, *path, *self)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -441,6 +446,10 @@ func settingsCommand(ctx context.Context, args []string) error {
|
||||
if len(positionals) == 2 {
|
||||
values = positionals[1]
|
||||
}
|
||||
// Once its config values are mesh-config's, they are proposed there (issue 664).
|
||||
if forwarded, err := forwardPropose(ctx, inv, positionals[0], *node, values, *clear); forwarded || err != nil {
|
||||
return err
|
||||
}
|
||||
return proposeCommand(ctx, positionals[0], *node, values, *clear, *replace)
|
||||
case "proposals":
|
||||
if len(positionals) > 1 || *node != "" || *clear || *replace || *history {
|
||||
@@ -461,6 +470,15 @@ func settingsCommand(ctx context.Context, args []string) error {
|
||||
where = *node
|
||||
}
|
||||
|
||||
// Once the move of config has run, a module in the config form is read and set through mesh-config (issue 664).
|
||||
if (args[0] == "set" && len(positionals) == 2) || ((args[0] == "show" && !*history) || args[0] == "clear") &&
|
||||
len(positionals) == 1 {
|
||||
if forwarded, err := forwardSettings(ctx, inv, args[0], positionals[0], *node, positionals, *replace); forwarded ||
|
||||
err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
switch args[0] {
|
||||
case "set":
|
||||
if len(positionals) != 2 {
|
||||
@@ -537,6 +555,10 @@ func settingsCommand(ctx context.Context, args []string) error {
|
||||
if p.SetBy != "" {
|
||||
by = "; " + p.SetBy
|
||||
}
|
||||
if p.Why != "" {
|
||||
// A layer whose placement moved out of it (migration 0097): still the layer, as it stands.
|
||||
by += "; " + p.Why
|
||||
}
|
||||
fmt.Printf("%s on %s, until %s (%s%s):\n %s\n", positionals[0], where,
|
||||
p.ReplacedAt.Local().Format("2006-01-02 15:04:05"), p.ReplacedBy, by, shown)
|
||||
}
|
||||
@@ -940,7 +962,55 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password)
|
||||
if err := issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password); err != nil {
|
||||
return err
|
||||
}
|
||||
if !runsTheProvisionerClient(m) {
|
||||
return nil
|
||||
}
|
||||
return issueProvisionerClient(ctx, inv, m, node, known, reachable)
|
||||
}
|
||||
|
||||
// runsTheProvisionerClient says a module runs its machine's provisioner client, which reaches the bus on an
|
||||
// account of its own (novox/hq ADR 0349 clause 11, design 57 P3): it claims the seat and declares the own secret
|
||||
// that account is delivered in. The same test the bus's user list makes (inventory's declaredFor).
|
||||
func runsTheProvisionerClient(m catalogue.Manifest) bool {
|
||||
_, reads := m.OwnSecrets[broker.ProvisionerSecret]
|
||||
return reads && m.ClaimsSeat(broker.NodeProvisionerSeat)
|
||||
}
|
||||
|
||||
// issueProvisionerClient gives a machine's provisioner client its account: minted under the client's own name,
|
||||
// never the module's or the tool runner's, and sealed to the machine as the module's own secret
|
||||
// broker.ProvisionerSecret, which the client's service reads (novox/hq ADR 0349 clause 11, design 57 P3). No
|
||||
// claims and no consumer: the client holds no seat on this account and is delivered nothing but its answers.
|
||||
func issueProvisionerClient(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node string,
|
||||
known broker.Broker, reachable string) error {
|
||||
user := broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: m.Module}.Username()
|
||||
password, err := inv.MintBusPassword(ctx, inventory.BusUser{
|
||||
Username: user, Kind: inventory.BusProvisioner, Node: node, Module: m.Module,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
held, err := json.Marshal(struct {
|
||||
URL string `json:"url"`
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
Node string `json:"node"`
|
||||
User string `json:"user"`
|
||||
Password string `json:"password"`
|
||||
}{URL: "nats://" + reachable, Fingerprint: known.Fingerprint, Node: node, User: user, Password: password})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.AcceptSecretForModule(ctx, node, m.Module, broker.ProvisionerSecret, string(held)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := handIssuedToVault(ctx, inv, m, node, broker.ProvisionerSecret, string(held)); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("bus user %s minted for %s's provisioner client, scoped to %s's keys, its reports and the vault\n",
|
||||
user, m.Module, node)
|
||||
return nil
|
||||
}
|
||||
|
||||
// busKindOf is what a module's bus user is recorded as: the node's tool runtime where the module is
|
||||
@@ -985,6 +1055,9 @@ func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manife
|
||||
if err := inv.AcceptSecretForModule(ctx, node, m.Module, "broker", string(held)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := handIssuedToVault(ctx, inv, m, node, "broker", string(held)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// And how it hears what it consumes. Derived from its declaration, and only when it declared
|
||||
// something: a module that consumes nothing needs no consumer, and creating one would be a
|
||||
|
||||
@@ -30,6 +30,7 @@ func userUnit(module, account string) inventory.ResourceHealth {
|
||||
// Only what depends on the new login alone is a wait; anything else unhealthy of the module is judged as
|
||||
// before, so that a fault is never excused.
|
||||
func TestOnlyWhatDependsOnTheNewLoginIsAWait(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
notInGroup := relogin("lights", "operator")
|
||||
notInGroup.Reason = "not in the group lights: the apply has not put operator there; its outcome says why"
|
||||
systemUnit := userUnit("lights", "operator")
|
||||
@@ -74,6 +75,7 @@ func TestOnlyWhatDependsOnTheNewLoginIsAWait(t *testing.T) {
|
||||
// The gate reads a wait for a person as its own reading, never a fault, and passes with it; the same module
|
||||
// with a container down beside it is not yet healthy, as before.
|
||||
func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
now := time.Now()
|
||||
since := now.Add(-time.Minute)
|
||||
account := relogin("lights", "operator")
|
||||
@@ -102,6 +104,7 @@ func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) {
|
||||
// The module's condition says the wait in one sentence for a person, and clears on the first statement that
|
||||
// no longer says it — said as the module's own fault when its unit still fails after the new login.
|
||||
func TestTheReloginConditionSaysTheWaitAndClearsAfterTheLogin(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
k, _ := withConditionsInMemory(t)
|
||||
@@ -158,6 +161,7 @@ func TestTheReloginConditionSaysTheWaitAndClearsAfterTheLogin(t *testing.T) {
|
||||
// A module that is healthy on its own keeps its pass when another module of the same send fails at the
|
||||
// bound: it is neither put back nor left without a verdict for every other walk to wait on.
|
||||
func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
@@ -207,6 +211,7 @@ func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) {
|
||||
// What the operator reads of the wait (ADR 0253, ADR 0254): it needs them, so it is never quiet, and it offers
|
||||
// no button, since nothing but their own new login can do it.
|
||||
func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
o := reloginObservation("openrazer", "g14", "relogin needed on g14: …", "operator", []inventory.ResourceHealth{
|
||||
relogin("openrazer", "operator"), userUnit("openrazer", "operator")})
|
||||
plainExample(t, o, "openrazer waits for a new login on g14",
|
||||
@@ -238,6 +243,7 @@ func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) {
|
||||
// When a module not working turns into one waiting for a new login, and back, the clearing line says what it
|
||||
// became, never that it works again (issue 318 review).
|
||||
func TestAConditionThatBecameTheOtherKindSaysSoWhenItClears(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
store := conditions.NewInMemory()
|
||||
@@ -286,6 +292,7 @@ func TestAConditionThatBecameTheOtherKindSaysSoWhenItClears(t *testing.T) {
|
||||
|
||||
// Which moves add an account group, read from the builds' manifests (issue 318 review).
|
||||
func TestOnlyAMoveThatAddsAnAccountGroupCanBringAWait(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
user := func(groups ...any) catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "lights", Resources: []map[string]any{{"id": "operator", "type": "user",
|
||||
"name": "operator", "groups": groups}}}
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// movePlacementAnswer is what the move of placement says it did, or would do (issue 736).
|
||||
type movePlacementAnswer struct {
|
||||
// DryRun is true when nothing was moved: asked without --confirm.
|
||||
DryRun bool `json:"dry-run"`
|
||||
// Moved is each layer that held a placement key among its settings, and which keys.
|
||||
Moved []inventory.MovedLayer `json:"moved"`
|
||||
}
|
||||
|
||||
// movePlacementCommand is `move-placement`: the move of placement out of the settings layers (novox/hq ADR 0348
|
||||
// clause 11, issue 736), one of the moves of data a release runs once after its delivery (design 57).
|
||||
func movePlacementCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("move-placement", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "answer as JSON")
|
||||
confirm := set.Bool("confirm", false, "move the keys, rather than only say which layers hold them")
|
||||
f := addHandActFlags(set)
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 0 {
|
||||
return errors.New("move-placement [--json] [--confirm --why <text>]")
|
||||
}
|
||||
if !*confirm && f.given() {
|
||||
return errors.New("move-placement takes --why only with --confirm: without it nothing is moved. Nothing was done")
|
||||
}
|
||||
if *confirm {
|
||||
// A move a person asks for says why, before anything is asked of the store.
|
||||
if err := f.require("move-placement"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
if *confirm {
|
||||
f.record(ctx, "move-placement", nil)
|
||||
}
|
||||
answer, err := runMovePlacement(ctx, open.inventory, *confirm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
return encoder.Encode(answer)
|
||||
}
|
||||
fmt.Print(describeMovePlacement(answer))
|
||||
return nil
|
||||
}
|
||||
|
||||
// runMovePlacement is the move against a store: a dry run says what it would move.
|
||||
func runMovePlacement(ctx context.Context, inv *inventory.Inventory, apply bool) (movePlacementAnswer, error) {
|
||||
moved, err := inv.MovePlacement(ctx, apply)
|
||||
if err != nil {
|
||||
return movePlacementAnswer{}, err
|
||||
}
|
||||
return movePlacementAnswer{DryRun: !apply, Moved: moved}, nil
|
||||
}
|
||||
|
||||
// describeMovePlacement says the answer as a person reads it.
|
||||
func describeMovePlacement(a movePlacementAnswer) string {
|
||||
if len(a.Moved) == 0 {
|
||||
return "no settings layer holds a placement key: nothing to move\n"
|
||||
}
|
||||
var b strings.Builder
|
||||
verb := "moved"
|
||||
if a.DryRun {
|
||||
verb = "would move"
|
||||
}
|
||||
for _, m := range a.Moved {
|
||||
where := m.Node
|
||||
if where == "" {
|
||||
where = "the whole mesh"
|
||||
}
|
||||
fmt.Fprintf(&b, "%s on %s: %s %s\n", m.Module, where, verb, strings.Join(m.Keys, ", "))
|
||||
}
|
||||
if a.DryRun {
|
||||
fmt.Fprintf(&b, "%d layer(s); nothing was moved — with --confirm --why <text> they are\n", len(a.Moved))
|
||||
} else {
|
||||
fmt.Fprintf(&b, "%d layer(s) moved beside their assignments, each kept whole in its history first\n", len(a.Moved))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// **Placement leaves the settings layers and no machine is told anything new** (novox/hq ADR 0348 clause
|
||||
// 11, issue 651). A mesh is seeded as the store holds it before migration 0097 — `places`, `accesses` and a
|
||||
// kept network among the settings, at the mesh's level and a machine's, one layer of placement only — then
|
||||
// migrated, and every machine's declaration is composed. What each machine is told of the seeded modules is
|
||||
// byte for byte what the controller before this change composed from the same seed: the golden files were
|
||||
// written by the integration branch's controller at bfc79c6 running this same test (MESH_WRITE_GOLDEN=1),
|
||||
// where there was no migration 0097 to run.
|
||||
func TestADeclarationIsTheSameAfterPlacementMoves(t *testing.T) {
|
||||
open, migrate := aMeshBefore(t, 97)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
image := "registry.invalid:5000/library@sha256:" + strings.Repeat("a", 64)
|
||||
register(t, open, catalogue.Manifest{Module: "library", Version: "1",
|
||||
Accesses: []catalogue.Access{{ID: "series"}, {ID: "downloads", Mode: catalogue.AccessRead}},
|
||||
Resources: []map[string]any{
|
||||
{"id": "config", "type": "directory", "mode": "0750"},
|
||||
{"id": "server", "type": "container", "image": image,
|
||||
"volumes": []any{"${dir:config}:/config", "${access:series}:/series", "${access:downloads}:/downloads:ro"}},
|
||||
// Today's form: a mergeable file naming no keys takes every key of the layer (ADR 0326 §3).
|
||||
{"id": "runtime-config", "type": "file", "path": "${dir:config}/runtime.json", "mode": "0640",
|
||||
"content": `{"log":"info"}`, "merge": "json"},
|
||||
}})
|
||||
register(t, open, catalogue.Manifest{Module: "found", Version: "1",
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "image": image}}})
|
||||
if err := inv.SetAdopted(ctx, "anchor", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, l := range []struct{ node, module, values string }{
|
||||
{"", "library", `{"accesses": {"downloads": "/storage/downloads"}, "log": "warn"}`},
|
||||
{"anchor", "library", `{"places": {"config": {"path": "/var/lib/library/config", "owner": "1001:2000"}},
|
||||
"accesses": {"series": "/storage/media/series"}, "workers": 4}`},
|
||||
{"laptop", "library", `{"accesses": {"series": "/home/media/series"}}`},
|
||||
{"anchor", "found", `{"networks": {"server": ["predecessor_default"]}}`},
|
||||
} {
|
||||
if l.node == "" {
|
||||
inv.ExecForTest(t, `insert into settings (node, module, values) values (null, $1, $2)`, l.module, l.values)
|
||||
continue
|
||||
}
|
||||
inv.ExecForTest(t, `insert into settings (node, module, values) select id, $2, $3 from node where name = $1`,
|
||||
l.node, l.module, l.values)
|
||||
}
|
||||
|
||||
migrate()
|
||||
// Assigned once the store is migrated: an assignment reads the layers, as this controller holds them.
|
||||
for _, a := range [][]string{{"anchor", "library", "found"}, {"laptop", "library"}} {
|
||||
if _, err := assign(ctx, open, a[0], a[1:]...); err != nil {
|
||||
t.Fatalf("assign %v: %v", a, err)
|
||||
}
|
||||
}
|
||||
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
plan, settings, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Allocating)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
composed, err := plan.Compose(with)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(composed.LeftOut) > 0 {
|
||||
t.Fatalf("%s leaves out %v", node, composed.LeftOut)
|
||||
}
|
||||
var told []map[string]any
|
||||
for _, r := range composed.Resources {
|
||||
if owner := composed.Owner[r["id"].(string)]; owner == "library" || owner == "found" {
|
||||
told = append(told, r)
|
||||
}
|
||||
}
|
||||
sort.Slice(told, func(i, j int) bool { return told[i]["id"].(string) < told[j]["id"].(string) })
|
||||
got, err := json.MarshalIndent(told, "", " ")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
golden := filepath.Join("testdata", "placement-moved", node+".json")
|
||||
if os.Getenv("MESH_WRITE_GOLDEN") != "" {
|
||||
if err := os.MkdirAll(filepath.Dir(golden), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(golden, append(got, '\n'), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
want, err := os.ReadFile(golden)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(want) != string(got)+"\n" {
|
||||
t.Errorf("%s is told something else after placement moved:\n%s\nwas:\n%s", node, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// wordedMesh is every live layer that says `endpoints`, `expose`, `reach` or `ports` (issue 736) — 36, read with
|
||||
// the settings verb on 2026-10-14, ace's as anchor's and novox's as laptop's, one stand-in for an address — split as
|
||||
// the store holds them once migration 0097 has run: the four words still among the settings, `places` and
|
||||
// `accesses` already beside them. With each module shaped as its definition in the catalogue (mesh-catalog's, and
|
||||
// the media catalogue's) is where the four words reach: its listens, what its containers publish and on which
|
||||
// network, its route, its directories and accesses, and its mergeable files — every key of the layer, or the keys
|
||||
// they name. The shapes are the modules'; the files' text and the images are the test's.
|
||||
type wordedMesh struct {
|
||||
Modules []json.RawMessage `json:"modules"`
|
||||
Layers []struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Settings map[string]any `json:"settings"`
|
||||
Placement map[string]any `json:"placement"`
|
||||
} `json:"layers"`
|
||||
}
|
||||
|
||||
func theWordedMesh(t *testing.T) wordedMesh {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "placement-words-moved", "mesh.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var w wordedMesh
|
||||
if err := json.Unmarshal(raw, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// aMeshWithTheWordsInItsLayers is aMesh with every module of the worded mesh registered, a route provider on both
|
||||
// machines, and its layers held as the store holds them before the four words move.
|
||||
func aMeshWithTheWordsInItsLayers(t *testing.T) (*stores, wordedMesh) {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
w := theWordedMesh(t)
|
||||
image := "registry.invalid:5000/library@sha256:" + strings.Repeat("a", 64)
|
||||
register(t, open, catalogue.Manifest{Module: "proxy", Version: "1", Provides: catalogue.FromAnywhere("route"),
|
||||
Receives: map[string]string{"route": "/var/lib/proxy/routes.json"}})
|
||||
for _, text := range w.Modules {
|
||||
m, err := catalogue.ParseManifest([]byte(strings.ReplaceAll(string(text), "IMAGE", image)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, m)
|
||||
}
|
||||
on := map[string][]string{"anchor": {"proxy"}, "laptop": {"proxy"}}
|
||||
for _, l := range w.Layers {
|
||||
settings, err := json.Marshal(l.Settings)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inv.ExecForTest(t, `insert into settings (node, module, values) select id, $2, $3 from node where name = $1`,
|
||||
l.Node, l.Module, string(settings))
|
||||
if l.Placement != nil {
|
||||
placement, err := json.Marshal(l.Placement)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inv.ExecForTest(t, `insert into placement (node, module, values) select id, $2, $3 from node where name = $1`,
|
||||
l.Node, l.Module, string(placement))
|
||||
}
|
||||
on[l.Node] = append(on[l.Node], l.Module)
|
||||
}
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
if err := inv.SetPublicDomain(ctx, node, node+".example.test"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, node, on[node]...); err != nil {
|
||||
t.Fatalf("assign %s %v: %v", node, on[node], err)
|
||||
}
|
||||
}
|
||||
return open, w
|
||||
}
|
||||
|
||||
// toldOfTheWords is what a machine is told of the modules whose layers say the four words, and of the route
|
||||
// provider that serves their names: those modules' resources and the filter's rules, as JSON a person can diff.
|
||||
func toldOfTheWords(t *testing.T, open *stores, node string) string {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, settings, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Allocating)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
composed, err := plan.Compose(with)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(composed.LeftOut) > 0 {
|
||||
t.Fatalf("%s leaves out %v", node, composed.LeftOut)
|
||||
}
|
||||
rules, err := plan.Rules(with)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ours := map[string]bool{"proxy": true}
|
||||
for _, l := range theWordedMesh(t).Layers {
|
||||
ours[l.Module] = true
|
||||
}
|
||||
var told []map[string]any
|
||||
for _, r := range composed.Resources {
|
||||
if ours[composed.Owner[r["id"].(string)]] {
|
||||
told = append(told, r)
|
||||
}
|
||||
}
|
||||
sort.Slice(told, func(i, j int) bool { return told[i]["id"].(string) < told[j]["id"].(string) })
|
||||
got, err := json.MarshalIndent(map[string]any{"resources": told, "rules": rules}, "", " ")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(got) + "\n"
|
||||
}
|
||||
|
||||
// sameAsTheGolden holds what a machine is told to the golden file the controller before issue 736 wrote from the
|
||||
// same seed (MESH_WRITE_GOLDEN=1 writes it instead).
|
||||
func sameAsTheGolden(t *testing.T, open *stores, when string) {
|
||||
t.Helper()
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
got := toldOfTheWords(t, open, node)
|
||||
golden := filepath.Join("testdata", "placement-words-moved", node+".json")
|
||||
if os.Getenv("MESH_WRITE_GOLDEN") != "" {
|
||||
if err := os.MkdirAll(filepath.Dir(golden), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(golden, []byte(got), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
want, err := os.ReadFile(golden)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(want) != got {
|
||||
t.Errorf("%s: %s is told something else:\n%s\nwas:\n%s", when, node, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **The four words leave the settings layers and no machine is told anything new** (novox/hq ADR 0348 clause 11,
|
||||
// issue 736). Every live layer that says `endpoints`, `expose`, `reach` or `ports` is seeded as the store holds it
|
||||
// after delivery, on a module shaped as its definition is, and every machine's declaration is composed. What each
|
||||
// is told is byte for byte what the controller before this change composed from the same seed.
|
||||
func TestADeclarationIsTheSameWhileTheWordsAreStillInTheLayers(t *testing.T) {
|
||||
open, _ := aMeshWithTheWordsInItsLayers(t)
|
||||
sameAsTheGolden(t, open, "before the move")
|
||||
}
|
||||
|
||||
// **And after the move, still nothing new** (issue 736). The move of placement, asked without confirming, says
|
||||
// every layer that holds one of the four words and moves nothing; confirmed, it moves them all, no settings layer
|
||||
// holds a placement key after, and each machine is told byte for byte what it was told before the move — and what
|
||||
// the controller before this change told it. Asked again, it moves nothing.
|
||||
func TestADeclarationIsTheSameAfterTheMoveOfPlacement(t *testing.T) {
|
||||
open, w := aMeshWithTheWordsInItsLayers(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asked, err := runMovePlacement(ctx, inv, false)
|
||||
if err != nil || !asked.DryRun || len(asked.Moved) != len(w.Layers) {
|
||||
t.Fatalf("the move, not confirmed, says %+v (%v); %d layers hold the words", asked, err, len(w.Layers))
|
||||
}
|
||||
if held := inv.QueryForTest(t, `select module from settings where values ?| $1`, catalogue.PlacementKeys()); len(held) != len(w.Layers) {
|
||||
t.Fatalf("the move, not confirmed, moved something: %d layers still hold the words", len(held))
|
||||
}
|
||||
sameAsTheGolden(t, open, "after the move was asked without confirming")
|
||||
|
||||
moved, err := runMovePlacement(ctx, inv, true)
|
||||
if err != nil || moved.DryRun || !reflect.DeepEqual(moved.Moved, asked.Moved) {
|
||||
t.Fatalf("the move says %+v (%v), it said it would move %+v", moved, err, asked.Moved)
|
||||
}
|
||||
if held := inv.QueryForTest(t, `select module from settings where values ?| $1`, catalogue.PlacementKeys()); len(held) != 0 {
|
||||
t.Fatalf("a settings layer still holds a placement key: %v", held)
|
||||
}
|
||||
sameAsTheGolden(t, open, "after the move")
|
||||
|
||||
if again, err := runMovePlacement(ctx, inv, true); err != nil || len(again.Moved) != 0 {
|
||||
t.Fatalf("asked again, the move moved %+v (%v)", again, err)
|
||||
}
|
||||
if said := describeMovePlacement(movePlacementAnswer{}); !strings.Contains(said, "nothing to move") {
|
||||
t.Fatalf("an empty move says %q", said)
|
||||
}
|
||||
}
|
||||
|
||||
// The verb's command line: a dry run unless confirmed, and a confirmed move says why, recorded as a person's step
|
||||
// of the release, never a repair (issue 736).
|
||||
func TestTheMoveOfPlacementIsADryRunUnlessConfirmed(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
args map[string]any
|
||||
want string
|
||||
}{
|
||||
{map[string]any{}, "move-placement --json"},
|
||||
{map[string]any{"confirm": "false"}, "move-placement --json"},
|
||||
{map[string]any{"confirm": "true", "why": "release 57 is delivered"}, "move-placement --json --confirm --why release 57 is delivered"},
|
||||
} {
|
||||
argv, err := argvFor("move-placement", c.args)
|
||||
if err != nil || strings.Join(argv, " ") != c.want {
|
||||
t.Errorf("%v: %q %v, want %q", c.args, argv, err, c.want)
|
||||
}
|
||||
}
|
||||
for _, args := range []map[string]any{
|
||||
{"confirm": "true"}, // a move without a why
|
||||
{"why": "tidy"}, // a why for a dry run, recorded nowhere
|
||||
{"confirm": "yes", "why": "x"}, // a switch is true or false
|
||||
{"node": "anchor"},
|
||||
} {
|
||||
if argv, err := argvFor("move-placement", args); err == nil {
|
||||
t.Errorf("%v was composed as %q", args, argv)
|
||||
}
|
||||
}
|
||||
if repairingCommand([]string{"move-placement", "--json", "--confirm", "--why", "x"}) != "move-placement" {
|
||||
t.Error("a confirmed move through the generic verb would go unrecorded")
|
||||
}
|
||||
if repairingCommand([]string{"move-placement", "--json"}) != "" {
|
||||
t.Error("a dry run was taken for an act by hand")
|
||||
}
|
||||
if !personsDecision(link.HandAct{Verb: "move-placement"}) {
|
||||
t.Error("a move of the release would count toward a healer the mesh lacks")
|
||||
}
|
||||
}
|
||||
@@ -117,6 +117,8 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
||||
"reaches it. It keeps running what it has.", m),
|
||||
Resolved: m + " can get new instructions again"}
|
||||
}),
|
||||
// A grant of a restricted stream that composes nothing (novox/hq ADR 0376 §2).
|
||||
kindGrantComposesNothing: worded(withheldWords),
|
||||
kindAgentCanBecomeRoot: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "Sessions on " + m + " could become root",
|
||||
@@ -184,6 +186,15 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
||||
}),
|
||||
// The controller or the node-engine kept at the build a machine was last sent, over an older record (novox/hq
|
||||
// issue 201).
|
||||
kindVaultSilent: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "The vault did not give " + m + " the core's credentials",
|
||||
Needs: "check the vault answers (vault_status); the next send asks it again.",
|
||||
Explanation: fmt.Sprintf("The bus's and the provisioner's credentials reach %s sealed by the vault, and "+
|
||||
"the vault did not answer the last send for them. %s keeps the files it has; a core module not "+
|
||||
"there yet is not placed. The mesh never seals them itself instead.", m, m),
|
||||
Resolved: "Resolved: the vault gave " + m + " the core's credentials"}
|
||||
}),
|
||||
kindCoreDowngrade: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
component := idPart(o, 0)
|
||||
@@ -456,6 +467,9 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
||||
"release-held": worded(func(o conditions.Observation) words {
|
||||
return releaseHeldWords(nil, o.Also)
|
||||
}),
|
||||
"release-silent": worded(func(o conditions.Observation) words {
|
||||
return releaseSilentWords(o.Also)
|
||||
}),
|
||||
"facts-stale": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Merge checks use outdated facts",
|
||||
Explanation: "The facts a merge check judges a change against are out of date; they are gathered again " +
|
||||
@@ -617,6 +631,18 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
||||
Explanation: "The bus reports a listener too slow to keep up, so messages to it are late.",
|
||||
Resolved: "The listener keeps up again"}
|
||||
}),
|
||||
kindBusFaultsUnread: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Slow clients of the bus cannot be checked",
|
||||
Explanation: "The controller could not read what the bus said about its clients' connections, so a " +
|
||||
"client the bus cut off would go unnoticed. It tries again at every check.",
|
||||
Resolved: "Resolved: the bus's clients are checked again"}
|
||||
}),
|
||||
"connection-closed": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The bus cut a client off",
|
||||
Explanation: "The bus closed a client's connection because it broke the bus's rules, so what it was " +
|
||||
"sending was lost.",
|
||||
Resolved: "Resolved: the client keeps to the bus's rules"}
|
||||
}),
|
||||
kindBusReconnects: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A client keeps losing the bus",
|
||||
Explanation: "One of the mesh's clients lost its connection to the bus again and again in the last hour. " +
|
||||
@@ -841,6 +867,19 @@ func releaseHeldWords(modules, machines []string) words {
|
||||
Resolved: "Resolved: the held updates are released"}
|
||||
}
|
||||
|
||||
// releaseSilentWords are the plain words of machines that said nothing on an update they were sent (ADR 0353 §4).
|
||||
func releaseSilentWords(machines []string) words {
|
||||
where := "Some machines"
|
||||
if len(machines) > 0 {
|
||||
where = conditions.Capital(namesWords(machines, 4))
|
||||
}
|
||||
return words{Headline: "Machines did not answer an update",
|
||||
Needs: "check that they are awake and on the mesh; the next walk sends them what waits.",
|
||||
Explanation: where + " were sent module updates and said nothing about them within fifteen minutes, so " +
|
||||
"they were left as they were. What they run is not known to be the update.",
|
||||
Resolved: "Resolved: the next walk reached them"}
|
||||
}
|
||||
|
||||
// reloginNeeds is what an account waiting for its groups needs (ADR 0252).
|
||||
func reloginNeeds(node string) string {
|
||||
return fmt.Sprintf("log out of %s completely and log in again, or restart it.", node)
|
||||
|
||||
+128
-4
@@ -150,6 +150,10 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
|
||||
// password a provider is told to create is the one its consumer was given — and sealed to
|
||||
// this node before it was ever written down, so nothing between here and there can read it.
|
||||
for i, n := range resolved.Needs {
|
||||
// **Issued for this holder alone and kept by the vault** (issue 757): where this consumer is handed to the
|
||||
// provisioner, its machine's client asks the vault for it, and the declaration carries no value.
|
||||
resolved.Needs[i].IssuedToHolder = issuedToHolder(shelf, n)
|
||||
resolved.Needs[i].ProviderMoved = providerMoved(shelf, n)
|
||||
if n.ByRecord {
|
||||
// Answered by something the mesh holds, so there is no pair-wise secret between two
|
||||
// machines. Its key was supplied by a person and sealed to this node then; the mesh
|
||||
@@ -476,9 +480,23 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
||||
out := sendable{Resources: composed.Resources, Adoption: adoption,
|
||||
Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
|
||||
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld,
|
||||
unbound: with.Unbound, foreseen: composed.Foreseen, unplaced: composed.Unplaced,
|
||||
modules: namedModules(plan, composed.LeftOut),
|
||||
unbound: with.Unbound, foreseen: composed.Foreseen, unplaced: composed.Unplaced, handedOnly: composed.HandedOnly,
|
||||
modules: namedModules(plan, composed.LeftOut),
|
||||
Filled: composed.Filled, plans: composed.Plans,
|
||||
localNetwork: catalogue.LocalNetworkPorts(rules, with.LocalNetworks)}
|
||||
if len(composed.Filled) > 0 {
|
||||
out.Provisioner = with.Provisioner
|
||||
}
|
||||
// A handed module only pulled in by another keeps what requires it, for its value of its own in the assignments
|
||||
// mirror (issue 765).
|
||||
for module := range composed.Plans {
|
||||
if by := plan.RequiredBy(module); by != "" {
|
||||
if out.requiredBy == nil {
|
||||
out.requiredBy = map[string]string{}
|
||||
}
|
||||
out.requiredBy[module] = by
|
||||
}
|
||||
}
|
||||
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
|
||||
// 0221). Read only on the send path: a question about what would be sent records nothing.
|
||||
if choosing == Allocating {
|
||||
@@ -591,6 +609,10 @@ func reportLeftOut(node string, declared sendable) {
|
||||
for _, u := range declared.unplaced {
|
||||
fmt.Printf("%s: %s\n", node, u)
|
||||
}
|
||||
// And each file only the provisioner's client fills, left out because the module is not handed here.
|
||||
for _, u := range declared.handedOnly {
|
||||
fmt.Printf("%s: %s\n", node, u)
|
||||
}
|
||||
}
|
||||
|
||||
// busCredentialIssued refuses an own secret called `broker` whose bus account nobody issued.
|
||||
@@ -601,10 +623,17 @@ func reportLeftOut(node string, declared sendable) {
|
||||
// cannot read and report the machine applied. Refused by name, with the verb — on the send, and on
|
||||
// a question asked ahead of it (Foreseeing), so that one is never told the push will make it.
|
||||
func busCredentialIssued(ctx context.Context, inv *inventory.Inventory, node, module, name string) error {
|
||||
if name != "broker" {
|
||||
var user string
|
||||
switch name {
|
||||
case "broker":
|
||||
user = broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username()
|
||||
case broker.ProvisionerSecret:
|
||||
// And the machine's provisioner client's account, issued with its module (novox/hq ADR 0349 clause 11,
|
||||
// design 57 P3): the same refusal, for the same reason.
|
||||
user = broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: module}.Username()
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username()
|
||||
if _, minted, err := inv.BusUserHash(ctx, user); err != nil {
|
||||
return err
|
||||
} else if !minted {
|
||||
@@ -728,7 +757,18 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
// per node, so a module running on three machines has three.
|
||||
needed := map[string]map[string]string{}
|
||||
foreseen := map[string]map[string]bool{}
|
||||
// **A module handed to the provisioner here has its secrets from the vault** (novox/hq ADR 0349 §7, §11,
|
||||
// design 57 V4): its machine's client asks for each, so the controller makes, reads and seals none of them for
|
||||
// this machine. Every other module's are made and sealed as before.
|
||||
hands, err := engineTakesHanded(ctx, inv, node)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
handing := catalogue.Rendering{Hands: hands, Provisioner: provisionerAccount(plan)}
|
||||
for _, m := range plan.Modules {
|
||||
if handing.Handing(m) {
|
||||
continue
|
||||
}
|
||||
// **A secret family is never made** (novox/hq ADR 0283): each member a person gave on this machine is
|
||||
// placed, and one not given is nothing — the module says it waits for it.
|
||||
for _, family := range m.OwnSecrets.Families() {
|
||||
@@ -822,6 +862,22 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
needed[m.Module][name] = sealed
|
||||
}
|
||||
}
|
||||
// **The core's own credentials as the vault sealed them** (novox/hq ADR 0349 §16, issue 755), for an engine that
|
||||
// opens the vault's box (core_box.go): the declaration carries the box, never the seal made above. Below that
|
||||
// contract nothing here is asked and the composition is as it was.
|
||||
var core coreBoxes
|
||||
opensBox, err := engineOpensVaultBox(ctx, inv, node)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
if opensBox {
|
||||
core, err = coreBoxesFor(ctx, inv, node, plan.Modules, func(m catalogue.Manifest) bool {
|
||||
return handing.Handing(m)
|
||||
}, choosing)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
}
|
||||
// And a certificate for this machine's name inside the mesh, when anything on it asks. Issued
|
||||
// rather than stored: the node's key does not change, so signing again produces an equally
|
||||
// valid certificate and there is nothing to keep in step.
|
||||
@@ -1016,8 +1072,11 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
ReadsHealth: readsHealth,
|
||||
JudgesRoot: judgesRoot,
|
||||
ReadsBusy: readsBusy,
|
||||
Hands: hands,
|
||||
Provisioner: provisionerAccount(plan),
|
||||
BusMembership: memberships[node],
|
||||
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports,
|
||||
VaultSealed: core.sealed, NotPlaced: core.notPlaced,
|
||||
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
||||
Machines: machines, Zones: zones, Holders: replicas,
|
||||
Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
|
||||
@@ -1059,6 +1118,35 @@ func engineReadsBusy(ctx context.Context, inv *inventory.Inventory, node string)
|
||||
return had && stated.Contract >= link.BusyContract, nil
|
||||
}
|
||||
|
||||
// engineTakesHanded says whether a machine's node-engine writes a filled file on the provisioner client's word alone
|
||||
// (link.HandedContract, design 57 P7), by its own newest statement: only there is a moved module handed to the
|
||||
// provisioner (design 57 P6). Anywhere else a filled file would have two writers, and the module is sent as before.
|
||||
func engineTakesHanded(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) {
|
||||
stated, had, err := inv.HealthOf(ctx, node)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return had && stated.Contract >= link.HandedContract, nil
|
||||
}
|
||||
|
||||
// provisionerAccount is the account the provisioner's client runs as on this machine (novox/hq ADR 0349 §4): the user
|
||||
// the process of the module running it runs as, empty where no such module is on the machine.
|
||||
func provisionerAccount(plan catalogue.Resolution) string {
|
||||
for _, m := range plan.Modules {
|
||||
if !runsTheProvisionerClient(m) {
|
||||
continue
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] == "process" {
|
||||
if user, _ := r["user"].(string); user != "" && user != "root" {
|
||||
return user
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// busyNotSent is, for a node-engine that does not read a container's busy, each container whose module declares
|
||||
// one: not sent it, so replaced at once while it may hold work (novox/hq ADR 0335). Nothing for one that reads it.
|
||||
func busyNotSent(modules []catalogue.Manifest, reads bool) []string {
|
||||
@@ -1846,6 +1934,42 @@ func issuedOutsideBy(shelf map[string]catalogue.Manifest, n catalogue.Needed) (s
|
||||
return catalogue.OffersIssuedOutside(shelf, n.Name)
|
||||
}
|
||||
|
||||
// issuedToHolder is whether the vault keeps a need's credential for its holder alone (issues 749, 757 and 779): its
|
||||
// providing module is marked moved, so the vault holds the credential in a row of its own for each holder
|
||||
// (filledTheNewWay), and its offer says either that the provider's software issues the provision's credential, or
|
||||
// that the provider gives each consumer a login of its own (form pair, ADR 0158 §11). A credential every holder
|
||||
// shares, or one a person gives, is neither.
|
||||
func issuedToHolder(shelf map[string]catalogue.Manifest, n catalogue.Needed) bool {
|
||||
if n.ByRecord || n.Module == "" {
|
||||
return false
|
||||
}
|
||||
provider, known := shelf[n.Module]
|
||||
if !known || !movedProvider(provider) {
|
||||
return false
|
||||
}
|
||||
if _, issued := provider.IssuedCredentialOf(n.Name); issued {
|
||||
return true
|
||||
}
|
||||
return provider.CredentialFormOf(n.Name) == catalogue.CredentialFormPair
|
||||
}
|
||||
|
||||
// providerMoved is whether a need is answered by a module marked moved (issue 788), or by one whose offer of the
|
||||
// provision says the provider helper answers it (issue 792).
|
||||
func providerMoved(shelf map[string]catalogue.Manifest, n catalogue.Needed) bool {
|
||||
if n.ByRecord || n.Module == "" {
|
||||
return false
|
||||
}
|
||||
provider, known := shelf[n.Module]
|
||||
return known && (movedProvider(provider) || provider.AnsweredByHelperFor(n.Name))
|
||||
}
|
||||
|
||||
// movedProvider is whether a provider's credentials are the vault's: the module is marked moved, or it holds the
|
||||
// mesh-vault seat, whose pair rows have been in the vault since V3 (issue 793), so its consumers' credentials
|
||||
// (gitea's internal token and admin password) are filled from those very rows with the values unchanged.
|
||||
func movedProvider(m catalogue.Manifest) bool {
|
||||
return m.Moved || slices.ContainsFunc(m.Claims, func(c catalogue.Claim) bool { return c.Name == "mesh-vault" })
|
||||
}
|
||||
|
||||
func providerModuleOf(resolved catalogue.Resolution, open *stores, ctx context.Context, n catalogue.Needed) string {
|
||||
for _, m := range resolved.Modules {
|
||||
if _, shared := m.SharedCredentialOf(n.Name); shared {
|
||||
|
||||
@@ -372,6 +372,13 @@ func joinAPlan(ctx context.Context, open *stores, module string) (bool, string,
|
||||
if !found {
|
||||
return false, "", nil
|
||||
}
|
||||
// **A walk that waits for its delivery's word asks nothing** (novox/hq ADR 0239, issue 754): a rebuild does not
|
||||
// join it, and its outcome is not the walk's. The walk asks its own tier once it has its word.
|
||||
if p.Waiting() {
|
||||
return false, fmt.Sprintf("not joining %s (%s at %s): it waits for %s's word, so this build is not its "+
|
||||
"outcome; once let go, the walk asks %s itself", p.ID, p.Repository, short(p.Commit), p.Delivery.Awaits,
|
||||
module), nil
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
|
||||
@@ -133,7 +133,7 @@ func TestASharedRepositoryIsPlannedFromTheRecordedBuildSources(t *testing.T) {
|
||||
{"the route proxy's program: the route proxy alone", "mesh-controller",
|
||||
[]string{"examples/route-proxy/main.go"}, "route-proxy", true},
|
||||
{"a package all three build from: all three", "mesh-controller",
|
||||
[]string{"internal/broker/broker.go"}, "mesh-controller | build-agent | route-proxy", false},
|
||||
[]string{"internal/broker/broker.go"}, "mesh-controller,route-proxy | build-agent", false},
|
||||
{"the route proxy's directory in the catalogue: it alone", "mesh-catalog",
|
||||
[]string{"modules/route-proxy/module.json"}, "route-proxy", false},
|
||||
{"the build agent's directory: it alone, nothing it builds", "mesh-catalog",
|
||||
|
||||
@@ -145,8 +145,9 @@ func TestAPlanIsWhatTheChangeTouchedAndWhatIsBuiltOnIt(t *testing.T) {
|
||||
repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a,x"},
|
||||
{what: "built-by never widens", edges: []inventory.Edge{dep("x", builtBy, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x"}, want: "a"},
|
||||
{what: "built-by orders", edges: []inventory.Edge{dep("x", builtBy, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a | x"},
|
||||
// Whichever builder runs builds it (novox/hq ADR 0360 §3): built-by orders nothing, as before it ordered.
|
||||
{what: "built-by does not order", edges: []inventory.Edge{dep("x", builtBy, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a,x"},
|
||||
{what: "worker-of never widens", edges: []inventory.Edge{dep("x", workerOf, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x"}, want: "a"},
|
||||
{what: "worker-of orders", edges: []inventory.Edge{dep("x", workerOf, "a")},
|
||||
@@ -296,18 +297,18 @@ func TestASharedRepositoryMovesOnlyWhatItsBuildSourceHolds(t *testing.T) {
|
||||
[]string{"internal/inventory/migrations/0088-a-build-says-its-build-source.sql"}, read,
|
||||
"mesh-controller | build-agent", ""},
|
||||
// A package all three build from: all three; the build seat's holder after the controller whose worker
|
||||
// it binds (worker-of, issue 206), the proxy after the holder that builds it (built-by).
|
||||
// it binds (worker-of, issue 206), the proxy beside the controller: built-by orders nothing (ADR 0360 §3).
|
||||
{"a package all three build from: all three", "mesh-controller",
|
||||
[]string{"internal/broker/broker.go"}, read, "mesh-controller | build-agent | route-proxy", ""},
|
||||
[]string{"internal/broker/broker.go"}, read, "mesh-controller,route-proxy | build-agent", ""},
|
||||
{"a vendored package all three build from: all three", "mesh-controller",
|
||||
[]string{"vendor/github.com/nats-io/nats.go/nats.go"}, read, "mesh-controller | build-agent | route-proxy", ""},
|
||||
[]string{"vendor/github.com/nats-io/nats.go/nats.go"}, read, "mesh-controller,route-proxy | build-agent", ""},
|
||||
{"go.sum: all three", "mesh-controller",
|
||||
[]string{"go.sum"}, read, "mesh-controller | build-agent | route-proxy", ""},
|
||||
[]string{"go.sum"}, read, "mesh-controller,route-proxy | build-agent", ""},
|
||||
{"a file added to the proxy's package: the proxy", "mesh-controller",
|
||||
[]string{"examples/route-proxy/new.go"}, read, "route-proxy", ""},
|
||||
// Before any build source is said: as before.
|
||||
{"no build source said: a README moves all three, as before", "mesh-controller",
|
||||
[]string{"README.md"}, unsaid, "mesh-controller | build-agent | route-proxy", ""},
|
||||
[]string{"README.md"}, unsaid, "mesh-controller,route-proxy | build-agent", ""},
|
||||
// In the catalogue, where they live, each by its own build source.
|
||||
{"the route proxy's recipe: it alone", "mesh-catalog",
|
||||
[]string{"modules/route-proxy/Dockerfile"}, read, "route-proxy", ""},
|
||||
@@ -331,7 +332,7 @@ func TestASharedRepositoryMovesOnlyWhatItsBuildSourceHolds(t *testing.T) {
|
||||
// Files not all said: everything the repository builds, as before.
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main", Commit: "head",
|
||||
Paths: []string{"README.md"}, PathsTruncated: true}
|
||||
if got := tiered(reachOfMerge(m, entries, read, edges).Plan.Tiers); got != "mesh-controller | build-agent | route-proxy" {
|
||||
if got := tiered(reachOfMerge(m, entries, read, edges).Plan.Tiers); got != "mesh-controller,route-proxy | build-agent" {
|
||||
t.Errorf("a merge whose files were not all said: planned %q, wanted all three", got)
|
||||
}
|
||||
}
|
||||
@@ -452,8 +453,9 @@ func TestAPlanIsTheTouchedModulesAndWhatIsReachableAlongTheWideningEdges(t *test
|
||||
kinds := []string{inventory.EdgeStandsOn, inventory.EdgeDeclared, inventory.EdgePackages,
|
||||
inventory.EdgeBuiltBy, inventory.EdgeWorkerOf}
|
||||
widens := map[string]bool{inventory.EdgeStandsOn: true, inventory.EdgeDeclared: true}
|
||||
// Built-by orders nothing since novox/hq ADR 0360 §3.
|
||||
orders := map[string]bool{inventory.EdgeStandsOn: true, inventory.EdgeDeclared: true,
|
||||
inventory.EdgeBuiltBy: true, inventory.EdgeWorkerOf: true}
|
||||
inventory.EdgeWorkerOf: true}
|
||||
// Directory names drawn from one pool, so two repositories hold directories of the same name, and one
|
||||
// is a prefix of another.
|
||||
dirs := []string{"a", "ab", "b", "c", "lib/x", "lib/y", "modules/a", "modules/a/sub"}
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The list of predefined keys the mesh publishes (novox/hq ADR 0348 §7, design 57 package K2, issue 661).
|
||||
//
|
||||
// **The newest trunk build of the catalogue wins.** A build of the catalogue reads predefined.json at the commit
|
||||
// it builds and carries it with the commit's generation (link.PredefinedList). Registration keeps it in the
|
||||
// controller's store when that generation is larger than the one kept (inventory.KeepPredefined), so an older
|
||||
// build's list heard later never replaces a newer one, and every manifest the controller reads again is judged
|
||||
// against the list kept (catalogue.UsePredefined). mesh-config publishes it on the bus from there (design 57
|
||||
// package K3); the controller keeps no config value, only the keys.
|
||||
|
||||
// fromTheCatalogue is whether a build is of the catalogue's repository, whose list alone the mesh publishes: by
|
||||
// its name, in the URL form or the seat form.
|
||||
func fromTheCatalogue(result link.BuildResult) bool {
|
||||
for _, repository := range []string{result.Repository, sourceRepository(result)} {
|
||||
name := strings.TrimSuffix(path.Base(strings.TrimRight(repository, "/")), ".git")
|
||||
if name == catalogue.CatalogueRepository {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func sourceRepository(result link.BuildResult) string {
|
||||
if result.Source == nil {
|
||||
return ""
|
||||
}
|
||||
return result.Source.Repository
|
||||
}
|
||||
|
||||
// carriedList is the list a build of the catalogue carried, read strictly; nil for a build of another
|
||||
// repository, or one that carried none.
|
||||
func carriedList(result link.BuildResult) (catalogue.PredefinedList, error) {
|
||||
if result.Predefined == nil || !fromTheCatalogue(result) {
|
||||
return nil, nil
|
||||
}
|
||||
list, err := catalogue.ParsePredefined(result.Predefined.Keys)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s carried a list of predefined keys that is refused: %w", result.ID, err)
|
||||
}
|
||||
return list, nil
|
||||
}
|
||||
|
||||
// judgedWith is the list a build's manifest is first judged against at registration: the catalogue's own at the
|
||||
// commit built, which its build judged it against, where it carried one; the list kept otherwise.
|
||||
func judgedWith(result link.BuildResult) (catalogue.PredefinedList, error) {
|
||||
list, err := carriedList(result)
|
||||
if err != nil || list != nil {
|
||||
return list, err
|
||||
}
|
||||
return catalogue.PublishedPredefined(), nil
|
||||
}
|
||||
|
||||
// loadPredefined makes the list kept in the store the one this controller judges manifests against. A store
|
||||
// keeping none leaves the list it has.
|
||||
func loadPredefined(ctx context.Context, inv *inventory.Inventory) error {
|
||||
kept, has, err := inv.PredefinedKept(ctx)
|
||||
if err != nil || !has {
|
||||
return err
|
||||
}
|
||||
catalogue.UsePredefined(kept.Keys)
|
||||
return nil
|
||||
}
|
||||
|
||||
// publishPredefined keeps a trunk build's list when it is newer than the one kept, says so, and loads what the
|
||||
// store keeps then — this build's, or a newer one another controller process kept.
|
||||
func publishPredefined(ctx context.Context, inv *inventory.Inventory, result link.BuildResult,
|
||||
list catalogue.PredefinedList) error {
|
||||
kept, err := inv.KeepPredefined(ctx, inventory.PredefinedRecord{Keys: list, Repository: result.Repository,
|
||||
Commit: result.Commit, Generation: result.Predefined.Generation, Build: result.ID})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if kept {
|
||||
fmt.Printf("%s: the list of predefined keys at %s (generation %d) is published: %s\n", result.ID,
|
||||
short(result.Commit), result.Predefined.Generation, strings.Join(list.Keys(), ", "))
|
||||
} else {
|
||||
fmt.Printf("%s: the list of predefined keys at %s (generation %d) is not newer than the one kept, which "+
|
||||
"stands\n", result.ID, short(result.Commit), result.Predefined.Generation)
|
||||
}
|
||||
return loadPredefined(ctx, inv)
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The list of predefined keys (novox/hq ADR 0348 §7, design 57 package K2, issue 661): `module check` and the
|
||||
// merge gate judge each manifest against the list at the root of its tree; registration publishes the newest
|
||||
// trunk build's list of the catalogue and judges every module against the list the mesh publishes.
|
||||
|
||||
const theFirstKeys = `{
|
||||
"font-size": {"type": "number", "description": "the size text is shown in, in points", "minimum": 6, "maximum": 72},
|
||||
"agent-managed-settings": {"type": "object", "description": "the managed settings of every agent session, a JSON object", "trusted": true}
|
||||
}`
|
||||
|
||||
// notifierFallingBackTo is the notifier in the config form, its font size falling back to the key named.
|
||||
func notifierFallingBackTo(key, kind string) string {
|
||||
def := "10"
|
||||
if kind == "string" {
|
||||
def = `"10"`
|
||||
}
|
||||
return `{"module":"notifier","version":"1",
|
||||
"config":{"font-size":{"type":"` + kind + `","description":"the notification's font size, in points",
|
||||
"default":` + def + `,"predefined":"` + key + `"}},
|
||||
"resources":[{"id":"conf","type":"file","path":"/etc/notifier/notifier.conf","mode":"0644",
|
||||
"content":"font = ${config:font-size}\n"}]}`
|
||||
}
|
||||
|
||||
// withoutAPublishedList judges the test against no list the mesh published, and puts the process's back after.
|
||||
func withoutAPublishedList(t *testing.T) {
|
||||
t.Helper()
|
||||
was := catalogue.ReplacePublished(nil)
|
||||
t.Cleanup(func() { catalogue.ReplacePublished(was) })
|
||||
}
|
||||
|
||||
// aRepositoryTree is a repository's checkout holding one module at modules/notifier, and the list when given.
|
||||
func aRepositoryTree(t *testing.T, manifest string, list *string) string {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
if err := os.Mkdir(filepath.Join(root, ".git"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if list != nil {
|
||||
if err := os.WriteFile(filepath.Join(root, catalogue.PredefinedFile), []byte(*list), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return aTemplatedModule(t, filepath.Join(root, "modules", "notifier"), manifest, nil)
|
||||
}
|
||||
|
||||
func TestModuleCheckJudgesAManifestAgainstTheListAtItsRepositoryRoot(t *testing.T) {
|
||||
withoutAPublishedList(t)
|
||||
list := theFirstKeys
|
||||
check := func(path string) (string, error) {
|
||||
var out bytes.Buffer
|
||||
err := moduleCheck([]string{path}, &out)
|
||||
return out.String(), err
|
||||
}
|
||||
if out, err := check(aRepositoryTree(t, notifierFallingBackTo("font-size", "number"), &list)); err != nil ||
|
||||
!strings.Contains(out, "notifier: ok") {
|
||||
t.Fatalf("a module naming font-size was refused: %v\n%s", err, out)
|
||||
}
|
||||
for _, c := range []struct {
|
||||
manifest string
|
||||
list *string
|
||||
says string
|
||||
}{
|
||||
{notifierFallingBackTo("font-scale", "number"), &list, `"font-scale", which is not on the list`},
|
||||
{notifierFallingBackTo("font-size", "string"), &list, "which is a number, and this value is a string"},
|
||||
{notifierFallingBackTo("font-size", "number"), nil, "is not present"},
|
||||
{notifierFallingBackTo("font-size", "number"), ptr(`{"font-size": {"type": "number"}}`), "no description"},
|
||||
} {
|
||||
if out, err := check(aRepositoryTree(t, c.manifest, c.list)); err == nil || !strings.Contains(out, c.says) {
|
||||
t.Errorf("wanted the check refused saying %q, got %v:\n%s", c.says, err, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func ptr(s string) *string { return &s }
|
||||
|
||||
// writeTheList puts the list at the root of a tree the gate judges.
|
||||
func writeTheList(t *testing.T, tree, list string) string {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(filepath.Join(tree, catalogue.PredefinedFile), []byte(list), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return tree
|
||||
}
|
||||
|
||||
func TestTheMergeGateJudgesEachManifestAgainstTheListOfTheTree(t *testing.T) {
|
||||
withoutAPublishedList(t)
|
||||
f, manifests := templatedMesh(t)
|
||||
|
||||
v := gateJudged(t, f, writeTheList(t, aTree(t, withEdit(manifests, "notifier",
|
||||
notifierFallingBackTo("font-size", "number"))), theFirstKeys), "modules/notifier/module.json")
|
||||
if v.Verdict == "fail" || v.Modules["notifier"] != "changed" {
|
||||
t.Fatalf("a module naming font-size on the tree's list failed the gate (%v):\n%s", v.Modules, v.Report())
|
||||
}
|
||||
for _, c := range []struct{ manifest, list, says string }{
|
||||
{notifierFallingBackTo("font-scale", "number"), theFirstKeys, `"font-scale", which is not on the list`},
|
||||
{notifierFallingBackTo("font-size", "string"), theFirstKeys, "which is a number"},
|
||||
{inlineNotifier, `{"font-size": {"type": "number", "description": "points", "unit": "pt"}}`, `unknown field "unit"`},
|
||||
} {
|
||||
v := gateJudged(t, f, writeTheList(t, aTree(t, withEdit(manifests, "notifier", c.manifest)), c.list),
|
||||
"modules/notifier/module.json", catalogue.PredefinedFile)
|
||||
if v.Verdict != "fail" || !strings.Contains(v.Report(), c.says) {
|
||||
t.Errorf("wanted the gate to fail saying %q:\n%s", c.says, v.Report())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// catalogueBuild is a trunk build of the catalogue carrying the list at a generation.
|
||||
func catalogueBuild(id, manifest, list string, generation int) link.BuildResult {
|
||||
r := link.BuildResult{ID: id, Repository: "http://forge.internal:20000/novox/mesh-catalog.git",
|
||||
Path: "modules/notifier", Ref: "main", On: "anchor", Commit: strings.Repeat(id[len(id)-1:], 12),
|
||||
Manifest: json.RawMessage(manifest), Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
||||
if list != "" {
|
||||
r.Predefined = &link.PredefinedList{Keys: json.RawMessage(list), Generation: generation}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func TestRegistrationPublishesTheNewestTrunkListAndJudgesEveryModuleAgainstIt(t *testing.T) {
|
||||
withoutAPublishedList(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
|
||||
// A manifest naming font-size registers, and its build's list is published.
|
||||
if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory,
|
||||
catalogueBuild("b-1", notifierFallingBackTo("font-size", "number"), theFirstKeys, 10))); err != nil {
|
||||
t.Fatalf("a module naming font-size was not registered: %v", err)
|
||||
}
|
||||
kept, has, err := open.inventory.PredefinedKept(ctx)
|
||||
if err != nil || !has || kept.Generation != 10 || kept.Build != "b-1" {
|
||||
t.Fatalf("the list was not published: %+v %v %v", kept, has, err)
|
||||
}
|
||||
if got := catalogue.PublishedPredefined(); len(got) != 2 {
|
||||
t.Fatalf("the controller judges against %v, not the list published", got)
|
||||
}
|
||||
|
||||
// One naming a key not on the list, or of another type, is refused and never enters the catalogue.
|
||||
for i, c := range []struct{ manifest, says string }{
|
||||
{notifierFallingBackTo("font-scale", "number"), "not on the list"},
|
||||
{notifierFallingBackTo("font-size", "string"), "which is a number"},
|
||||
} {
|
||||
id := []string{"b-2", "b-3"}[i]
|
||||
if _, _, err := takeIn(ctx, open.inventory, catalogueBuild(id, c.manifest, theFirstKeys, 10)); err == nil ||
|
||||
!strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("wanted the registration refused saying %q, got %v", c.says, err)
|
||||
}
|
||||
}
|
||||
src, err := open.inventory.SourceOf(ctx, "notifier")
|
||||
if err != nil || src.BuiltFrom != strings.Repeat("1", 12) {
|
||||
t.Fatalf("a refused build replaced the registered one: %+v %v", src, err)
|
||||
}
|
||||
|
||||
// A newer build's list, which drops agent-managed-settings, replaces the published one.
|
||||
smaller := `{"font-size": {"type": "number", "description": "points", "minimum": 6, "maximum": 72}}`
|
||||
if _, _, err := takeIn(ctx, open.inventory, catalogueBuild("b-4",
|
||||
notifierFallingBackTo("font-size", "number"), smaller, 12)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The older build's outcome heard later registers nothing new about the list: the newer stands.
|
||||
if _, _, err := takeIn(ctx, open.inventory, catalogueBuild("b-5",
|
||||
notifierFallingBackTo("font-size", "number"), theFirstKeys, 11)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kept, _, _ = open.inventory.PredefinedKept(ctx)
|
||||
if kept.Build != "b-4" || len(kept.Keys) != 1 || len(catalogue.PublishedPredefined()) != 1 {
|
||||
t.Fatalf("an older build's list heard later replaced the newer one: %+v", kept)
|
||||
}
|
||||
|
||||
// A build of another repository publishes nothing, and its manifest is judged against the published list.
|
||||
other := catalogueBuild("b-6", notifierFallingBackTo("font-size", "number"), theFirstKeys, 99)
|
||||
other.Repository, other.Source.Repository = "http://forge.internal:20000/novox/notifier.git", "novox/notifier"
|
||||
other.Manifest = json.RawMessage(strings.Replace(notifierFallingBackTo("font-size", "number"), `"notifier"`,
|
||||
`"chimes"`, 1))
|
||||
if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, other)); err != nil {
|
||||
t.Fatalf("a module of another repository naming a published key: %v", err)
|
||||
}
|
||||
if kept, _, _ = open.inventory.PredefinedKept(ctx); kept.Build != "b-4" {
|
||||
t.Fatalf("a build of another repository published its list: %+v", kept)
|
||||
}
|
||||
other.ID, other.Manifest = "b-7", json.RawMessage(strings.Replace(notifierFallingBackTo("agent-managed-settings",
|
||||
"number"), `"notifier"`, `"chimes"`, 1))
|
||||
if _, _, err := takeIn(ctx, open.inventory, other); err == nil || !strings.Contains(err.Error(), "not on the list") {
|
||||
t.Fatalf("a key only another repository's list holds was registered: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -929,6 +929,12 @@ func streamDiffers(want broker.Stream, have nats.StreamConfig) string {
|
||||
differs = append(differs, fmt.Sprintf("keeps one of a message id for %s, defined %s", have.Duplicates,
|
||||
time.Duration(want.DuplicatesSeconds)*time.Second))
|
||||
}
|
||||
if want.AllowMsgTTL && !have.AllowMsgTTL {
|
||||
differs = append(differs, "takes no time to live of a message's own, defined to (novox/hq ADR 0376)")
|
||||
}
|
||||
if want.Restricted && have.Metadata[broker.RestrictedMetadata] != "true" {
|
||||
differs = append(differs, "does not say it is restricted, so a message kept from it would not age out with it")
|
||||
}
|
||||
if want.DiscardNew && have.Discard != nats.DiscardNew {
|
||||
differs = append(differs, "drops what it holds when full, defined to refuse what comes next")
|
||||
}
|
||||
|
||||
@@ -54,6 +54,7 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
@@ -906,6 +907,21 @@ func setLayerIn(open *stores) setOnWarrant {
|
||||
return "", fmt.Errorf("the layer of %s on %s changed since the operator was shown the change: it is not set; "+
|
||||
"propose it again", p.Module, p.where())
|
||||
}
|
||||
// A proposal that carries config values, raised before the module's config moved, is not applied after it: its
|
||||
// values in the layer would turn ReadsConfig off and undo the move (issue 830, novox/hq ADR 0348). A proposal
|
||||
// of placement only is set as before: forwardPropose sends it down this path on purpose.
|
||||
carried := p.Values
|
||||
if p.Clear {
|
||||
carried = before
|
||||
}
|
||||
if config, _ := catalogue.SplitPlacement(carried); len(config) > 0 {
|
||||
if reads, err := inv.ReadsConfig(ctx, p.Module); err != nil {
|
||||
return "", err
|
||||
} else if reads {
|
||||
return "", fmt.Errorf("the config of %s moved to mesh-config since the operator was shown the change: it is "+
|
||||
"not set; propose it again (it then goes to mesh-config)", p.Module)
|
||||
}
|
||||
}
|
||||
var changed string
|
||||
if p.Clear {
|
||||
if err := inv.ClearSettingsBy(ctx, p.Node, p.Module, setBy); err != nil {
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// What a machine's provisioner client said of a module's files (novox/hq ADR 0362, ADR 0349 §15, design 57 P6,
|
||||
// issue 672), read where a walk's end and a gate count a machine's send applied: a module the send handed to the
|
||||
// provisioner is applied there only once the client reported `prov.applied` on that build's templates as well.
|
||||
|
||||
// provAppliedOn is the client's newest report of a module on a machine, from the stream that keeps it
|
||||
// (broker.ProvAppliedStream); false when it has said nothing of it. A variable so a test can say the client's words.
|
||||
var provAppliedOn = func(ctx context.Context, node, module string) (inventory.ProvReport, bool, error) {
|
||||
var raw []byte
|
||||
var said bool
|
||||
read := func(js nats.JetStreamContext) error {
|
||||
var err error
|
||||
raw, said, err = broker.LastProvApplied(js, node, module)
|
||||
return err
|
||||
}
|
||||
var err error
|
||||
if d := doctorFrom; d != nil && d.js != nil {
|
||||
err = read(d.js.Context())
|
||||
} else {
|
||||
err = onTheBus(func(conn *nats.Conn) error {
|
||||
js, err := conn.JetStream(nats.Context(ctx))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return read(js)
|
||||
})
|
||||
}
|
||||
if err != nil || !said {
|
||||
return inventory.ProvReport{}, false, err
|
||||
}
|
||||
var r inventory.ProvReport
|
||||
if err := json.Unmarshal(raw, &r); err != nil {
|
||||
return inventory.ProvReport{}, false, fmt.Errorf("what %s's provisioner client said of %s is not its report: %w",
|
||||
node, module, err)
|
||||
}
|
||||
return r, true, nil
|
||||
}
|
||||
|
||||
// provLookup answers the clients' reports for one reading; a report that cannot be read is not counted, which
|
||||
// leaves the machine waiting, then silent, never applied — and is said.
|
||||
func provLookup(ctx context.Context) inventory.ProvLookup {
|
||||
return func(node, module string) (inventory.ProvReport, bool) {
|
||||
r, said, err := provAppliedOn(ctx, node, module)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "what %s's provisioner client said of %s could not be read: %v\n", node, module, err)
|
||||
return inventory.ProvReport{}, false
|
||||
}
|
||||
return r, said
|
||||
}
|
||||
}
|
||||
|
||||
// provisioningFailed is every module whose files a machine's provisioner client last said it failed to place, from
|
||||
// the stream that keeps their reports.
|
||||
func provisioningFailed(js nats.JetStreamContext) ([]inventory.ProvReport, error) {
|
||||
all, err := broker.ProvAppliedReports(js)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []inventory.ProvReport
|
||||
for subject, raw := range all {
|
||||
var r inventory.ProvReport
|
||||
if err := json.Unmarshal(raw, &r); err != nil {
|
||||
return nil, fmt.Errorf("what was said on %s is not a provisioner client's report: %w", subject, err)
|
||||
}
|
||||
if r.Status == inventory.ProvFailed {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// provisionedOutcome is a machine's report of one send as a delivery's end counts it (novox/hq ADR 0362 §1-5, issue
|
||||
// 840): the engine's own report, combined with its provisioner client's for a send that handed modules to it. The
|
||||
// outcome of a client's failure begins inventory.OutcomeProvisioningFailed; a report not yet applied on both sides
|
||||
// is none (false). A send handing nothing is the engine's report unchanged. onSend says the report is about the send.
|
||||
func provisionedOutcome(node string, sent inventory.SentDeclaration, r inventory.Reported, onSend bool,
|
||||
prov inventory.ProvLookup) (string, bool) {
|
||||
if len(sent.Provisioned) == 0 {
|
||||
return r.Outcome, onSend
|
||||
}
|
||||
engine := inventory.AppliedReport{Outcome: r.Outcome}
|
||||
if r.At != nil {
|
||||
engine.At = *r.At
|
||||
}
|
||||
got, ok := inventory.ProvisionedSend(node, sent, engine, onSend, prov)
|
||||
return got.Outcome, ok
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The controller hands a moved module to the provisioner, and counts it applied only once its files are placed
|
||||
// (design 57 P6, issue 672; novox/hq ADR 0349 §4, §15, ADR 0362).
|
||||
|
||||
// A declaration that hands files says which, and whose client fills them; one that hands none is byte for byte
|
||||
// what it was.
|
||||
func TestADeclarationNamesTheFilledFilesOnlyWhenItHandsAny(t *testing.T) {
|
||||
plain := sendable{Resources: []map[string]any{{"id": "notify.configuration", "type": "file", "path": "/x",
|
||||
"content": "a"}}, Sequence: 4}
|
||||
before, err := plain.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(before), "filled_files") || strings.Contains(string(before), "provisioner_account") {
|
||||
t.Fatalf("a declaration handing nothing names filled files: %s", before)
|
||||
}
|
||||
handing := plain
|
||||
handing.Filled, handing.Provisioner = []string{"/x"}, "mesh-provisioner-client"
|
||||
body, err := handing.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var envelope struct {
|
||||
Filled []string `json:"filled_files"`
|
||||
Provisioner string `json:"provisioner_account"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &envelope); err != nil || len(envelope.Filled) != 1 || envelope.Filled[0] != "/x" ||
|
||||
envelope.Provisioner != "mesh-provisioner-client" {
|
||||
t.Fatalf("the handing declaration says %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheProvisionersAccountIsTheClientsProcessUser(t *testing.T) {
|
||||
client := catalogue.Manifest{Module: "mesh-provisioner-client",
|
||||
OwnSecrets: catalogue.OwnSecrets{"provisioner-broker": {Path: "/var/lib/x/broker"}},
|
||||
Claims: []catalogue.Claim{{Name: "node-provisioner", Scope: "node"}},
|
||||
Resources: []map[string]any{{"id": "account", "type": "user", "name": "mesh-provisioner-client"},
|
||||
{"id": "client", "type": "process", "name": "mesh-provisioner-client", "user": "mesh-provisioner-client"}}}
|
||||
if got := provisionerAccount(catalogue.Resolution{Modules: []catalogue.Manifest{client}}); got != "mesh-provisioner-client" {
|
||||
t.Fatalf("the client's account is %q", got)
|
||||
}
|
||||
if got := provisionerAccount(catalogue.Resolution{}); got != "" {
|
||||
t.Fatalf("a machine without a client names %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The first-machine gate counts a handed module healthy only once its node's prov.applied for the new build has
|
||||
// come (ADR 0349 §15, kept by ADR 0362 for a walk finishing at its gate); a failure there fails it.
|
||||
func TestTheGateWaitsForTheProvisionersReport(t *testing.T) {
|
||||
now := time.Date(2026, 10, 14, 23, 0, 0, 0, time.UTC)
|
||||
since := now.Add(-time.Minute)
|
||||
reported := now.Add(-50 * time.Second)
|
||||
digest := "sha256:" + strings.Repeat("d", 64)
|
||||
facts := func(said map[string]inventory.ProvReport) gateFacts {
|
||||
return gateFacts{now: now,
|
||||
reports: map[string]inventory.Reported{"g14": {Node: "g14", Outcome: inventory.OutcomeApplied,
|
||||
At: &reported, Current: true}},
|
||||
sent: map[string]inventory.SentDeclaration{"g14": {Digest: "sent", At: since,
|
||||
Provisioned: map[string]string{"dunst": digest}}},
|
||||
provisioned: func(node, module string) (inventory.ProvReport, bool) {
|
||||
r, ok := said[node+"."+module]
|
||||
return r, ok
|
||||
}}
|
||||
}
|
||||
m := catalogue.Manifest{Module: "dunst"}
|
||||
if h, why := judgeHealth("dunst", "", m, "g14", since, facts(nil)); h != healthNotYet ||
|
||||
!strings.Contains(why, "has not reported dunst's files placed") {
|
||||
t.Fatalf("judged without the provisioner's report: %v %q", h, why)
|
||||
}
|
||||
failed := map[string]inventory.ProvReport{"g14.dunst": {Status: inventory.ProvFailed, Templates: digest,
|
||||
Reason: "a value is not there", At: now.Add(-10 * time.Second)}}
|
||||
if h, why := judgeHealth("dunst", "", m, "g14", since, facts(failed)); h != healthBroken ||
|
||||
!strings.Contains(why, "a value is not there") {
|
||||
t.Fatalf("a failed placing is not the build's failure there: %v %q", h, why)
|
||||
}
|
||||
placed := map[string]inventory.ProvReport{"g14.dunst": {Status: inventory.ProvApplied, Templates: digest,
|
||||
At: now.Add(-10 * time.Second)}}
|
||||
if h, why := judgeHealth("dunst", "", m, "g14", since, facts(placed)); strings.Contains(why, "provisioner") {
|
||||
t.Fatalf("placed, the module is still held on the provisioner: %v %q", h, why)
|
||||
}
|
||||
// A module the send did not hand is judged as before.
|
||||
if h, why := judgeHealth("picom", "", catalogue.Manifest{Module: "picom"}, "g14", since, facts(nil)); strings.Contains(why, "provisioner") {
|
||||
t.Fatalf("a module not handed waits on the provisioner: %v %q", h, why)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine whose provisioner client failed to place a module's files has a failed apply, said as apply-failing
|
||||
// once the failure outlasts an apply interval, with the client's reason (ADR 0362 §3).
|
||||
func TestAFailedPlacingRaisesApplyFailing(t *testing.T) {
|
||||
now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
|
||||
f := calm(now)
|
||||
f.provFailed = []inventory.ProvReport{{Node: "anchor", Module: "dunst", Status: inventory.ProvFailed,
|
||||
Reason: "the templates cannot be fetched", At: now.Add(-time.Minute)}}
|
||||
if got := watchApplyFailing(f); len(got) != 0 {
|
||||
t.Fatalf("raised within an apply interval: %+v", got)
|
||||
}
|
||||
f.provFailed[0].At = now.Add(-10 * time.Minute)
|
||||
got := watchApplyFailing(f)
|
||||
if len(got) != 1 || got[0].Key() != "machine.anchor.apply-failing" ||
|
||||
!strings.Contains(got[0].Summary, "dunst") || !strings.Contains(got[0].Said, "the templates cannot be fetched") {
|
||||
t.Fatalf("a failed placing is not raised as anchor's apply-failing: %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
func aProvisioner() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "mesh-provisioner", Version: "1",
|
||||
DefinesSeats: []catalogue.SeatDeclaration{{Name: broker.NodeProvisionerSeat, Scope: catalogue.ScopeNode,
|
||||
Serves: []catalogue.Verb{{Name: "get"}}}},
|
||||
Claims: []catalogue.Claim{{Name: broker.NodeProvisionerSeat, Scope: catalogue.ScopeNode}},
|
||||
OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/run/mesh-provisioner/broker"},
|
||||
broker.ProvisionerSecret: {Path: "/run/mesh-provisioner/client"}},
|
||||
}
|
||||
}
|
||||
|
||||
// A machine's provisioner client is issued an account of its own, under its own name, sealed to the machine as the
|
||||
// module's own secret its service reads (novox/hq ADR 0349 clause 11, design 57 P3); until it is, a push refuses
|
||||
// to invent one, as it does for a module's `broker`.
|
||||
func TestTheProvisionerClientsAccountIsIssuedForItsServiceAndNeverInvented(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
m := aProvisioner()
|
||||
register(t, open, m)
|
||||
if _, err := assign(ctx, open, "laptop", m.Module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := busCredentialIssued(ctx, inv, "laptop", m.Module, broker.ProvisionerSecret); err == nil {
|
||||
t.Fatal("a push would seal a placeholder for the client's account before it was issued")
|
||||
}
|
||||
|
||||
if err := issueProvisionerClient(ctx, inv, m, "laptop", broker.Broker{Fingerprint: "ab"}, "10.0.0.1:4222"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, minted, err := inv.BusUserHash(ctx, "provisioner.laptop"); err != nil || !minted {
|
||||
t.Fatalf("the client's account is not minted: %v %v", minted, err)
|
||||
}
|
||||
states, err := inv.SecretsOf(ctx, "laptop", m.Module)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.ContainsFunc(states, func(s inventory.SecretState) bool { return s.Name == broker.ProvisionerSecret }) {
|
||||
t.Fatalf("the client's account is not sealed to the machine for its service: %+v", states)
|
||||
}
|
||||
if err := busCredentialIssued(ctx, inv, "laptop", m.Module, broker.ProvisionerSecret); err != nil {
|
||||
t.Fatalf("an issued client account is refused: %v", err)
|
||||
}
|
||||
if !runsTheProvisionerClient(m) {
|
||||
t.Fatal("the module is not read as running the client")
|
||||
}
|
||||
unread := m
|
||||
unread.OwnSecrets = catalogue.OwnSecrets{"broker": {Path: "/run/broker"}}
|
||||
if runsTheProvisionerClient(unread) {
|
||||
t.Fatal("a module with nowhere to read the client's account is read as running it")
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
@@ -194,7 +195,7 @@ func serve(ctx context.Context) (err error) {
|
||||
// And what the catalogue decided a build meant. The builder's own result is already handled
|
||||
// above; this is the other half — the control plane is the only one of the three that knows
|
||||
// which machines run the thing, so it is the one that acts (novox/hq ADR 0072).
|
||||
if err := server.Follows(following{open}); err != nil {
|
||||
if err := server.Follows(following{open: open}); err != nil {
|
||||
return err
|
||||
}
|
||||
// And the merges the bus announced and never handed over, read back on a timer and acted on late
|
||||
@@ -202,7 +203,7 @@ func serve(ctx context.Context) (err error) {
|
||||
go catchingUpOnMerges(ctx, open, server)
|
||||
// And a catalogue that has just started, asking for what it missed. The same type answers
|
||||
// both: what a build meant and what the builds were are two questions about one record.
|
||||
if err := server.Answers(following{open}); err != nil {
|
||||
if err := server.Answers(following{open: open}); err != nil {
|
||||
return err
|
||||
}
|
||||
// And what providers say about consumers they keep failing, kept for `status` (novox/hq ADR
|
||||
@@ -256,7 +257,7 @@ func serve(ctx context.Context) (err error) {
|
||||
go forgetChecksOfWalk(context.WithoutCancel(ctx), open.inventory, p)
|
||||
}
|
||||
}
|
||||
if err := server.Checks(following{open}); err != nil {
|
||||
if err := server.Checks(following{open: open}); err != nil {
|
||||
return err
|
||||
}
|
||||
// Composed now and kept current, before the verb that answers from it is served.
|
||||
@@ -264,6 +265,12 @@ func serve(ctx context.Context) (err error) {
|
||||
// The facts snapshot a merge check is fed (novox/hq to-be 45 §9): kept current while this
|
||||
// controller holds the lease, read by the build seat from the artifact store.
|
||||
go exportingFacts(ctx, open, bus.Conn.ConnectedServerVersion)
|
||||
// The record mirrors (novox/hq ADR 0289, design 57 F3): the assignments and each registered module's
|
||||
// declared config on the bus, written from the store only when a record changed, by the lease holder.
|
||||
// The provisioner's plan of each handed module, as its machine was last sent it (design 57 P6), in the module's
|
||||
// assignment.
|
||||
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
|
||||
go mirroringRecords(ctx, inv, bus.Conn, bus)
|
||||
// Every call carries the lease's epoch, and its record is written only under the lease (novox/hq
|
||||
// to-be 45 §6).
|
||||
link.Calls.UnderLease(func() (uint64, error) { return theLease.epoch(ctx) })
|
||||
@@ -296,6 +303,8 @@ func serve(ctx context.Context) (err error) {
|
||||
return err
|
||||
}
|
||||
defer stopCLI()
|
||||
// Its read calls write no line each; how many each caller made is said once an hour (novox/hq issue 762).
|
||||
go sayReadCalls(ctx)
|
||||
// And says so on the bus (novox/hq ADR 0197): what it serves, as the NATS services protocol asks.
|
||||
stopAnnouncing, err := bus.Announce(seatAnnouncement(handlers), log.New(os.Stdout, "", log.LstdFlags))
|
||||
if err != nil {
|
||||
@@ -1386,6 +1395,13 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, se
|
||||
if issued > 0 {
|
||||
fmt.Printf(" issued %d membership(s)\n", issued)
|
||||
}
|
||||
// **And what every other machine hears of a module's stream** (novox/hq ADR 0363 §6): a grant the provider's
|
||||
// setting changed reaches a consumer on a machine this send does not reach. Its membership is issued again
|
||||
// as last issued, with only what it hears changed, so nothing else of it moves before its own send.
|
||||
reheard := reissueHearing(ctx, bus, records, where, sent)
|
||||
if reheard > 0 {
|
||||
fmt.Printf(" issued %d membership(s) again for what they hear of a module's stream\n", reheard)
|
||||
}
|
||||
if len(refused) > 0 {
|
||||
// Returned, never passed over (novox/hq issue 249): the declarations of the machines they are
|
||||
// for are not sent, and the rollout that asked is tried again rather than waiting for a push.
|
||||
@@ -1685,7 +1701,7 @@ func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body
|
||||
// send that is away look failed: it is said, loudly, and the machine's own record stands.
|
||||
err = inv.RecordSentWith(kept, record.ID, digest, builds, epoch, sent.toldGeneration, inventory.Send{
|
||||
Sequence: sent.sequence, Epoch: int64(sent.epoch), Sender: sent.sender, Generation: sent.generation,
|
||||
Digest: digest, Modules: sent.modules})
|
||||
Digest: digest, Modules: sent.modules, Provisioning: sent.plans})
|
||||
var unrecorded *inventory.SendNotRecordedError
|
||||
switch {
|
||||
case errors.As(err, &unrecorded):
|
||||
@@ -1744,3 +1760,78 @@ func schemaReach() (int, error) {
|
||||
}
|
||||
return reach, nil
|
||||
}
|
||||
|
||||
// reissueHearing issues again, on every machine not just sent, the membership of each assignment whose
|
||||
// `consumes` names a module owning a stream, when what it hears changed: read as last issued from the bus,
|
||||
// its `hears` replaced, and published. One that was never issued is left to its own send. Failures are said
|
||||
// and passed over: the send that carries the grant is not the consumer's, and its own next send issues it.
|
||||
func reissueHearing(ctx context.Context, bus link.OverNATS, records broker.Records, where broker.Placements,
|
||||
sent []readyNode) int {
|
||||
reached := map[string]bool{}
|
||||
for _, s := range sent {
|
||||
reached[s.node] = true
|
||||
}
|
||||
issued := 0
|
||||
for node, declared := range records.Assigned {
|
||||
if reached[node] {
|
||||
continue
|
||||
}
|
||||
for _, d := range declared {
|
||||
if !d.HearsAStream {
|
||||
continue
|
||||
}
|
||||
subject := broker.MembershipSubject(node, d.Module)
|
||||
last, err := bus.JS.GetLastMsg(broker.AssignmentsStream, subject)
|
||||
if err != nil {
|
||||
if !errors.Is(err, nats.ErrMsgNotFound) {
|
||||
fmt.Printf(" %s on %s: its membership could not be read to issue what it hears again: %v\n",
|
||||
d.Module, node, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
body, changed, err := withHearing(last.Data, broker.HeardIssuedFor(node, d))
|
||||
if err != nil {
|
||||
fmt.Printf(" %s on %s: its membership could not be read to issue what it hears again: %v\n",
|
||||
d.Module, node, err)
|
||||
continue
|
||||
}
|
||||
if !changed {
|
||||
continue
|
||||
}
|
||||
if err := bus.PublishMembership(ctx, node, d.Module, body); err != nil {
|
||||
fmt.Printf(" %s on %s: what it hears could not be issued again: %v\n", d.Module, node, err)
|
||||
continue
|
||||
}
|
||||
issued++
|
||||
}
|
||||
}
|
||||
return issued
|
||||
}
|
||||
|
||||
// withHearing is a membership as issued with its `hears` replaced, and whether that changed it. Pure.
|
||||
func withHearing(issued []byte, hears []broker.HeardIssued) ([]byte, bool, error) {
|
||||
var fields map[string]json.RawMessage
|
||||
if err := json.Unmarshal(issued, &fields); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
var had []broker.HeardIssued
|
||||
if raw, ok := fields["hears"]; ok {
|
||||
if err := json.Unmarshal(raw, &had); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
}
|
||||
if slices.Equal(had, hears) {
|
||||
return issued, false, nil
|
||||
}
|
||||
if len(hears) == 0 {
|
||||
delete(fields, "hears")
|
||||
} else {
|
||||
raw, err := json.Marshal(hears)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
fields["hears"] = raw
|
||||
}
|
||||
body, err := json.Marshal(fields)
|
||||
return body, true, err
|
||||
}
|
||||
|
||||
@@ -340,7 +340,10 @@ func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat str
|
||||
return "", fmt.Errorf("%s is marked cancelled and could not be deleted from the queue (seq %d): %w — a "+
|
||||
"holder taking it ends it as cancelled", ask.ID, ask.Seq, err)
|
||||
}
|
||||
recordCancelled(ctx, open, ask)
|
||||
// A check ended because its pull request has a newer head records nothing (novox/hq issue 726): open is nil.
|
||||
if open != nil {
|
||||
recordCancelled(ctx, open, ask)
|
||||
}
|
||||
return cancelSaid(ask, seat, removed), nil
|
||||
}
|
||||
|
||||
@@ -518,7 +521,9 @@ func rebuildCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
|
||||
// **A module a plan holds unbuilt, or failed, joins that plan** rather than running beside it: the
|
||||
// plan would ask it again, or stay failed on an outcome a rebuild has replaced.
|
||||
// plan would ask it again, or stay failed on an outcome a rebuild has replaced. A walk waiting for its word
|
||||
// is not joined (novox/hq issue 754): the rebuild runs beside it, and says so.
|
||||
var beside string
|
||||
if module != "" {
|
||||
joined, said, err := joinAPlan(ctx, open, module)
|
||||
if err != nil {
|
||||
@@ -528,6 +533,7 @@ func rebuildCommand(ctx context.Context, args []string) error {
|
||||
fmt.Println(said)
|
||||
return nil
|
||||
}
|
||||
beside = said
|
||||
}
|
||||
id, err := askABuild(ctx, source, path, ref)
|
||||
if err != nil {
|
||||
@@ -535,6 +541,10 @@ func rebuildCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
recordAsked(ctx, inventory.BuildRequest{ID: id, Repository: source.Repository, Seat: source.Seat, Path: path,
|
||||
Ref: ref, For: "rebuild"})
|
||||
if beside != "" {
|
||||
fmt.Printf("rebuild asked as %s, beside the walk: %s\n", id, beside)
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("rebuild asked as %s\n", id)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -616,6 +616,7 @@ func TestKillAndPauseAskTheHolderOnTheMachine(t *testing.T) {
|
||||
// A plan that stopped at its first machine is retried: the module sent to that machine again, the
|
||||
// send recorded as the first anew, and the plan goes on — unless a newer plan holds the module.
|
||||
func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := asksRecorded(t)
|
||||
@@ -636,8 +637,9 @@ func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) {
|
||||
stopped := inventory.Plan{ID: "plan-rollout", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
|
||||
Created: long, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Note: "a stopped at its first machine in tier 0: laptop refused what it was sent",
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "built", BuiltAt: &long, Commit: "c0ffee",
|
||||
First: []string{"laptop"}, FirstAt: &long, Why: "laptop refused what it was sent"}}}
|
||||
// Asked by the plan, as every module a plan holds built is (novox/hq issue 754).
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "built", AskedAt: &long, Build: "build-a", BuiltAt: &long,
|
||||
Commit: "c0ffee", First: []string{"laptop"}, FirstAt: &long, Why: "laptop refused what it was sent"}}}
|
||||
if err := open.inventory.SavePlan(ctx, &stopped); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/recordmirror"
|
||||
)
|
||||
|
||||
// The controller's record mirrors on the bus (novox/hq ADR 0289, design 54, design 57 F3): the assignments,
|
||||
// and each registered module's declared config, which mesh-config resolves a module's config values from.
|
||||
//
|
||||
// **Built from the store, on a tick of their own, by the lease holder alone.** A record changes through a
|
||||
// verb that runs as a command of its own, a reconcile, a merge's registration or a pending assignment made,
|
||||
// so no one call site sees every change; what every change has in common is that it is committed to the
|
||||
// store. So the serving controller reads the store every mirrorEvery and writes only what differs from what
|
||||
// the bucket holds (ADR 0289 §1): a quiet mesh writes nothing, and a change rolled back is never read. A
|
||||
// mirror is therefore at most mirrorEvery behind its store.
|
||||
|
||||
// mirrorEvery is how often the mirrors are compared with the store: two small reads of the store.
|
||||
const mirrorEvery = 10 * time.Second
|
||||
|
||||
// mirroringRecords keeps the mirrors current for as long as this controller serves: ctx ends when it stops.
|
||||
func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn, announce recordmirror.Announcer) {
|
||||
api, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
fmt.Printf("the record mirrors are not kept: %v\n", err)
|
||||
return
|
||||
}
|
||||
holds := func() bool {
|
||||
epoch, err := theLease.epoch(ctx)
|
||||
return err == nil && epoch > 0
|
||||
}
|
||||
epoch := func() uint64 {
|
||||
e, _ := theLease.epoch(ctx)
|
||||
return e
|
||||
}
|
||||
sources := map[string]func(context.Context) (map[string][]byte, error){
|
||||
recordmirror.Assignments.Name: inv.AssignmentsMirror,
|
||||
recordmirror.DeclaredConfig.Name: inv.DeclaredConfigMirror,
|
||||
recordmirror.Predefined.Name: inv.PredefinedMirror,
|
||||
}
|
||||
writers := map[string]*recordmirror.Writer{}
|
||||
failing := map[string]string{}
|
||||
tick := time.NewTicker(mirrorEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
if holds() {
|
||||
for _, m := range recordmirror.Mirrors() {
|
||||
why := mirrorOnce(ctx, api, m, writers, sources[m.Name], announce, epoch, holds)
|
||||
if why != failing[m.Name] {
|
||||
if why != "" {
|
||||
fmt.Printf("the %s mirror cannot be kept: %s\n", m.Name, why)
|
||||
} else {
|
||||
fmt.Printf("the %s mirror is kept again\n", m.Name)
|
||||
}
|
||||
failing[m.Name] = why
|
||||
}
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// mirrorOnce brings one mirror to what the store holds, and answers why it could not; empty when it could.
|
||||
func mirrorOnce(ctx context.Context, api jetstream.JetStream, m recordmirror.Mirror, writers map[string]*recordmirror.Writer,
|
||||
source func(context.Context) (map[string][]byte, error), announce recordmirror.Announcer, epoch func() uint64,
|
||||
holds func() bool) string {
|
||||
reading, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
w := writers[m.Name]
|
||||
if w == nil {
|
||||
kv, err := api.KeyValue(reading, m.Bucket)
|
||||
if err != nil {
|
||||
return fmt.Sprintf("the bucket %s cannot be read: %v", m.Bucket, err)
|
||||
}
|
||||
w = &recordmirror.Writer{Mirror: m, KV: kv, Announce: announce, Epoch: epoch, Holds: holds}
|
||||
writers[m.Name] = w
|
||||
}
|
||||
want, err := source(reading)
|
||||
if err != nil {
|
||||
return fmt.Sprintf("the store cannot be read: %v", err)
|
||||
}
|
||||
lost := w.LostEvents
|
||||
r, err := w.Sync(reading, want)
|
||||
if r.Written+r.Removed > 0 {
|
||||
fmt.Printf("the %s mirror moved: %d written, %d removed\n", m.Name, r.Written, r.Removed)
|
||||
}
|
||||
if w.LostEvents > lost {
|
||||
fmt.Printf("the %s mirror lost %d event(s): the bucket holds the change, its notice was refused twice\n",
|
||||
m.Name, w.LostEvents-lost)
|
||||
}
|
||||
if err != nil {
|
||||
return err.Error()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -158,6 +158,12 @@ func recordedKept(ctx context.Context, open *stores, node string) (map[string]st
|
||||
if f.identical(m, was, now.Commit) {
|
||||
continue
|
||||
}
|
||||
// **Under its own `record`, a move that recreates nothing and moves no data is not kept** (novox/hq ADR 0377
|
||||
// §1): it goes as a module that rolls out goes. A person's `record`, the bus, and a move that cannot be
|
||||
// judged are kept as before (§4, §5).
|
||||
if !every && f.rolls(m, was) {
|
||||
continue
|
||||
}
|
||||
out[m] = was
|
||||
}
|
||||
// **And the controller and the node-engine at the build the machine was last sent, over an older record**
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// Under a module's own `record`, a move that recreates no container and moves no data rolls out (novox/hq ADR
|
||||
// 0377, changing ADR 0242 §1 and ADR 0360 §1).
|
||||
//
|
||||
// A module records because its manifest says so, because it keeps irreplaceable data, because a person set it, or
|
||||
// because it provides the bus. The first two hold for a reason the manifest shows — a change that recreates mail's
|
||||
// containers interrupts everyone's mail, a new database image may upgrade its databases in place — and that hold
|
||||
// also held the moves that interrupt nothing: on 2026-10-15 the database server moved with only its tools bundle
|
||||
// changed, its walk said it "recreates none of its containers", and ace stayed behind until a person pushed.
|
||||
//
|
||||
// So **a move under a module's own `record` is judged per machine, against the build that machine was last sent**
|
||||
// (§2): when it recreates none of the module's containers and moves none of its data (catalogue.Holds, §3) it
|
||||
// goes as a module that rolls out goes — a walk sends it, and every other send treats it as a move waiting for its
|
||||
// walk — and otherwise it is kept at the build the machine runs, as before. The module's policy stays `record`. A
|
||||
// move that cannot be judged waits (§4); a person's `record` and the bus hold every move (§5); and each walk says
|
||||
// per machine which way a recorded module went (§6).
|
||||
|
||||
// recordedRollsWhy is what a walk says of a recorded module's move that went out (ADR 0377 §6).
|
||||
const recordedRollsWhy = "went out: it recreates none of its containers and moves none of its data (novox/hq ADR 0377)"
|
||||
|
||||
// recordVerdict is one machine's move of a recorded module: whether it rolls out, and what the walk says of it.
|
||||
type recordVerdict struct {
|
||||
rolls bool
|
||||
// said is what the walk says: recordedRollsWhy, or "waits: " and why — the recreate words, the data change
|
||||
// named, the person's or the bus's hold, or that it could not be judged.
|
||||
said string
|
||||
}
|
||||
|
||||
// recordedMove judges a module's move on a machine from the build it was last sent (`was`) to the one it would be
|
||||
// sent now. A module that rolls out rolls; one whose policy is a person's or the bus's waits, said; one under its
|
||||
// own `record` rolls when the move recreates nothing and moves no data, and waits when it does or cannot be judged.
|
||||
func (f moveFacts) recordedMove(module, was string) recordVerdict {
|
||||
now := f.current[module]
|
||||
switch {
|
||||
case now.RollOut:
|
||||
return recordVerdict{rolls: true}
|
||||
case now.From == catalogue.FromPerson:
|
||||
return recordVerdict{said: "waits: a person set it to record, and holds every move of it until they push (novox/hq ADR 0377 §5)"}
|
||||
case now.From == catalogue.FromBus:
|
||||
return recordVerdict{said: "waits: it provides the bus, which moves only by the bus step a person starts (novox/hq ADR 0377 §5)"}
|
||||
case !now.OwnRecord():
|
||||
return recordVerdict{said: "waits: its policy records"}
|
||||
case f.judge == nil:
|
||||
return recordVerdict{said: "waits: it could not be judged — the build records were not read (novox/hq ADR 0377 §4)"}
|
||||
}
|
||||
return f.judge(module, was, now.Commit)
|
||||
}
|
||||
|
||||
// rolls is whether a module's move on a machine from `was` goes as a move that rolls out (ADR 0377 §1).
|
||||
func (f moveFacts) rolls(module, was string) bool {
|
||||
return f.recordedMove(module, was).rolls
|
||||
}
|
||||
|
||||
// recordJudge is the judging of moves under a module's own `record`, reading the build records once per move and
|
||||
// the catalogue once (ADR 0377 §2–§4).
|
||||
func recordJudge(ctx context.Context, inv *inventory.Inventory) func(module, was, now string) recordVerdict {
|
||||
judged := map[string]recordVerdict{}
|
||||
var shelf map[string]catalogue.Manifest
|
||||
var shelfErr error
|
||||
shelfRead := false
|
||||
return func(module, was, now string) recordVerdict {
|
||||
key := module + "\x00" + was + "\x00" + now
|
||||
if v, seen := judged[key]; seen {
|
||||
return v
|
||||
}
|
||||
v := func() recordVerdict {
|
||||
cannot := func(why string) recordVerdict {
|
||||
return recordVerdict{said: "waits: it could not be judged — " + why + " (novox/hq ADR 0377 §4)"}
|
||||
}
|
||||
if was == "" {
|
||||
return cannot("what the machine was last sent of it is not known")
|
||||
}
|
||||
from, found, err := inv.ManifestAt(ctx, module, was)
|
||||
if err != nil {
|
||||
return cannot(fmt.Sprintf("its build %s, which the machine was last sent, cannot be read: %v", short(was), err))
|
||||
}
|
||||
if !found {
|
||||
return cannot(fmt.Sprintf("its build %s, which the machine was last sent, is not in the build records", short(was)))
|
||||
}
|
||||
if !shelfRead {
|
||||
shelf, shelfErr = inv.Catalogue(ctx)
|
||||
shelfRead = true
|
||||
}
|
||||
if shelfErr != nil {
|
||||
return cannot(fmt.Sprintf("the build it would be sent cannot be read: %v", shelfErr))
|
||||
}
|
||||
to, held := shelf[module]
|
||||
if !held {
|
||||
return cannot("the build it would be sent is not in the catalogue")
|
||||
}
|
||||
if why := catalogue.Holds(from, to); why != "" {
|
||||
return recordVerdict{said: "waits: " + why}
|
||||
}
|
||||
return recordVerdict{rolls: true, said: recordedRollsWhy}
|
||||
}()
|
||||
judged[key] = v
|
||||
return v
|
||||
}
|
||||
}
|
||||
|
||||
// judgeRecordedInWalk judges, for a walk carrying a module whose policy records, each machine running it whose
|
||||
// build would move (ADR 0377 §2), keeps on the plan which way each went (§6), says it on the controller's log, and
|
||||
// answers the machines its move goes out to. Judged once per walk: a walk read again answers what it kept.
|
||||
func judgeRecordedInWalk(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan, module string,
|
||||
state *inventory.PlanModule) ([]string, error) {
|
||||
if state.Recorded != nil {
|
||||
return wentOut(state.Recorded), nil
|
||||
}
|
||||
running, err := inv.Running(ctx, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f, err := readMoveFacts(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := f.current[module]
|
||||
said := map[string]string{}
|
||||
for _, n := range running {
|
||||
sent, known, err := inv.SentBuilds(ctx, n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
was, carried := sent[module]
|
||||
if known && carried && f.identical(module, was, now.Commit) {
|
||||
continue // no move there
|
||||
}
|
||||
v := f.recordedMove(module, was)
|
||||
if !known {
|
||||
v = recordVerdict{said: "waits: it could not be judged — what " + n + " was last sent is not known (novox/hq ADR 0377 §4)"}
|
||||
}
|
||||
said[n] = v.said
|
||||
fmt.Printf("%s: %s on %s %s\n", p.ID, module, n, v.said)
|
||||
}
|
||||
if len(said) > 0 {
|
||||
state.Recorded = said
|
||||
}
|
||||
return wentOut(said), nil
|
||||
}
|
||||
|
||||
// wentOut is the machines a recorded module's move went out to, sorted.
|
||||
func wentOut(recorded map[string]string) []string {
|
||||
var out []string
|
||||
for _, n := range sortedKeysOf(recorded) {
|
||||
if recorded[n] == recordedRollsWhy {
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// recordedSaid is, for a walk's note at its end, which way each recorded module it carried went on each machine
|
||||
// (ADR 0377 §6).
|
||||
func recordedSaid(p inventory.Plan) []string {
|
||||
var out []string
|
||||
for _, m := range sortedKeysOf(planModuleNames(p)) {
|
||||
s := p.Modules[m]
|
||||
for _, n := range sortedKeysOf(s.Recorded) {
|
||||
out = append(out, fmt.Sprintf("%s on %s %s", m, n, s.Recorded[n]))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func planModuleNames(p inventory.Plan) map[string]string {
|
||||
out := map[string]string{}
|
||||
for m, s := range p.Modules {
|
||||
if s != nil && len(s.Recorded) > 0 {
|
||||
out[m] = m
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// recordedWaiting is, on one machine, every move of a recorded module that waits for a person, each said with why
|
||||
// (ADR 0377 §6): what `status` names the machine behind for, with `push <node>` as the remedy. A move under a
|
||||
// module's own `record` that recreates nothing and moves no data is not among them: it goes with its walk.
|
||||
func recordedWaiting(ctx context.Context, inv *inventory.Inventory, f moveFacts, node string) ([]string, error) {
|
||||
sent, known, err := inv.SentBuilds(ctx, node)
|
||||
if err != nil || !known {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
for _, m := range sortedKeysOf(sent) {
|
||||
was := sent[m]
|
||||
now, held := f.current[m]
|
||||
if !held || now.RollOut || now.Commit == "" || was == "" || f.identical(m, was, now.Commit) {
|
||||
continue
|
||||
}
|
||||
if v := f.recordedMove(m, was); !v.rolls {
|
||||
out = append(out, fmt.Sprintf("%s %s → %s %s", m, short(was), short(now.Commit), v.said))
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// stringsKeys is a map of lists as a map of its keys, for sortedKeysOf.
|
||||
func stringsKeys(m map[string][]string) map[string]string {
|
||||
out := map[string]string{}
|
||||
for k := range m {
|
||||
out[k] = k
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,361 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// Under a module's own `record`, a move that recreates no container and moves no data rolls out (novox/hq ADR
|
||||
// 0377). Seen 2026-10-15: the database server moved with only its tools bundle changed, its walk said it
|
||||
// "recreates none of its containers", and ace stayed behind until a person pushed.
|
||||
|
||||
// aMailu is a mail module as the catalogue declares it, small: it records because its manifest says so, keeps its
|
||||
// mailboxes as data, mounts its environment file, and serves its tools from a bundle.
|
||||
func aMailu(policy bool) catalogue.Manifest {
|
||||
raw := `{"module":"mailu","version":"1","tools":["mailu_status"],
|
||||
"data":{"own":[{"id":"mail","path":"${dir:data-mail}","class":"valuable","why":"every mailbox"}]},
|
||||
"bundles":[{"name":"tools","source":"store/mailu-tools@sha256:aaa","digest":"sha256:aaa"}],
|
||||
"resources":[
|
||||
{"id":"state","type":"directory"},
|
||||
{"id":"mesh-state","type":"directory"},
|
||||
{"id":"data-mail","type":"directory"},
|
||||
{"id":"config-env","type":"file","path":"${dir:state}/mailu.env","content":"A=1"},
|
||||
{"id":"runtime-config","type":"file","path":"${dir:mesh-state}/config.json","content":"{}"},
|
||||
{"id":"imap","type":"container","name":"mailu-imap","image":"registry.invalid/imap@sha256:1",
|
||||
"volumes":["${dir:data-mail}:/mail"],"env-file":["${dir:state}/mailu.env"]}]}`
|
||||
var m catalogue.Manifest
|
||||
if err := json.Unmarshal([]byte(raw), &m); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if policy {
|
||||
m.Upgrade = &catalogue.UpgradePolicy{Policy: catalogue.PolicyRecord, Why: "people's mail: a person chooses the moment"}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func resourceOf(m *catalogue.Manifest, id string) map[string]any {
|
||||
for _, r := range m.Resources {
|
||||
if r["id"] == id {
|
||||
return r
|
||||
}
|
||||
}
|
||||
panic("no resource " + id)
|
||||
}
|
||||
|
||||
// recordedMesh is a mesh running mailu at build c1 on anchor and laptop, as each was last sent.
|
||||
type recordedMesh struct {
|
||||
open *stores
|
||||
sent [][]string
|
||||
}
|
||||
|
||||
func aRecordedMesh(t *testing.T, c1 catalogue.Manifest) *recordedMesh {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
rm := &recordedMesh{open: open}
|
||||
rm.register(t, c1, "c1", time.Now().Add(-2*time.Hour))
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.Assign(ctx, n, c1.Module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{c1.Module: "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return rm
|
||||
}
|
||||
|
||||
// register keeps a build of a module and makes it the mesh's.
|
||||
func (rm *recordedMesh) register(t *testing.T, m catalogue.Manifest, commit string, at time.Time) {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
raw, _ := json.Marshal(m)
|
||||
if err := rm.open.inventory.RecordBuild(ctx, inventory.Build{ID: "build-" + m.Module + "-" + commit, Module: m.Module,
|
||||
Commit: commit, Repository: "novox/mesh-catalog", Path: "modules/" + m.Module, Manifest: raw, Asked: at, At: at}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := rm.open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
|
||||
Path: "modules/" + m.Module, BuiltFrom: commit, Head: commit, Asked: at}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// keptOn is what an ordinary send — a walk's, a healer's — keeps on a machine: module → the build it keeps.
|
||||
func (rm *recordedMesh) keptOn(t *testing.T, node string) map[string]string {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
under, err := sendKeeps(ctx, rm.open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kept, err := recordedKept(under, rm.open, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return kept
|
||||
}
|
||||
|
||||
// walk opens a merge's walk of mailu's c2, built, and walks it with every send composed as an ordinary send
|
||||
// composes it — a recorded module kept where its move waits — and applied at once.
|
||||
func (rm *recordedMesh) walk(t *testing.T) inventory.Plan {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
inv := rm.open.inventory
|
||||
wasSend := sendRollout
|
||||
n := 0
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
rm.sent = append(rm.sent, append([]string(nil), names...))
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, node := range names {
|
||||
kept := rm.keptOn(t, node)
|
||||
builds := map[string]string{"mailu": current["mailu"].Commit}
|
||||
if was, held := kept["mailu"]; held {
|
||||
builds["mailu"] = was
|
||||
}
|
||||
n++
|
||||
digest := fmt.Sprintf("d-%s-%d", node, n)
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, node), digest, builds); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := inv.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
|
||||
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = wasSend })
|
||||
built := time.Now().UTC()
|
||||
plan := inventory.Plan{ID: "plan-mailu", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
|
||||
Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"mailu"}},
|
||||
Modules: map[string]*inventory.PlanModule{"mailu": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-mailu-c2"}}}
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 8; i++ {
|
||||
advancePlans(ctx, rm.open)
|
||||
}
|
||||
p, err := inv.PlanByID(ctx, "plan-mailu")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func (rm *recordedMesh) sentBuild(t *testing.T, node string) string {
|
||||
t.Helper()
|
||||
sent, _, err := rm.open.inventory.SentBuilds(t.Context(), node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return sent["mailu"]
|
||||
}
|
||||
|
||||
// recordedWaitsOn is what `status` names a machine behind for, waiting for a person (ADR 0377 §6).
|
||||
func (rm *recordedMesh) recordedWaitsOn(t *testing.T, node string) []string {
|
||||
t.Helper()
|
||||
f, err := readMoveFacts(t.Context(), rm.open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held, err := recordedWaiting(t.Context(), rm.open.inventory, f, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return held
|
||||
}
|
||||
|
||||
// §1, §6: a module that records because its manifest says so, whose move changes only its tools bundle, goes out
|
||||
// with its walk to every machine, without a push; the walk says per machine that it went out and why; the policy
|
||||
// still reads record; and status names no machine behind for it.
|
||||
func TestARecordedModulesBundleOnlyMoveRollsOutInItsWalk(t *testing.T) {
|
||||
c1 := aMailu(true)
|
||||
rm := aRecordedMesh(t, c1)
|
||||
c2 := aMailu(true)
|
||||
c2.Bundles[0].Digest, c2.Bundles[0].Source = "sha256:bbb", "store/mailu-tools@sha256:bbb"
|
||||
c2.Tools = append(c2.Tools, "mailu_queue")
|
||||
rm.register(t, c2, "c2", time.Now().Add(-time.Minute))
|
||||
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if _, kept := rm.keptOn(t, n)["mailu"]; kept {
|
||||
t.Fatalf("an ordinary send to %s keeps mailu at c1 for a move that recreates nothing", n)
|
||||
}
|
||||
}
|
||||
p := rm.walk(t)
|
||||
if p.State != inventory.PlanDone {
|
||||
t.Fatalf("the walk is %s: %s", p.State, p.Note)
|
||||
}
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if got := rm.sentBuild(t, n); got != "c2" {
|
||||
t.Fatalf("%s was sent mailu %s; want c2, with its walk", n, got)
|
||||
}
|
||||
if said := p.Modules["mailu"].Recorded[n]; said != recordedRollsWhy {
|
||||
t.Fatalf("the walk says of mailu on %s %q", n, said)
|
||||
}
|
||||
if held := rm.recordedWaitsOn(t, n); len(held) != 0 {
|
||||
t.Fatalf("status names %s behind for %v", n, held)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(p.Note, "mailu on anchor went out") || !strings.Contains(p.Note, "mailu on laptop went out") {
|
||||
t.Fatalf("the walk's note does not say which way mailu went: %q", p.Note)
|
||||
}
|
||||
if u, err := rm.open.inventory.UpgradeOf(t.Context(), "mailu"); err != nil || u.Policy() != catalogue.PolicyRecord {
|
||||
t.Fatalf("mailu's policy is %q (%v); want record still", u.Policy(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// §1, §2, §6: a move that recreates a container — a new image, or a changed declaration — still waits for a push;
|
||||
// the walk says per machine that it waits, with the recreate words; and status names the machine behind, with
|
||||
// `push <node>` as the remedy. A bundle-only change on top of it stays held: it is judged against what the machine
|
||||
// was last sent.
|
||||
func TestARecordedModulesMoveThatRecreatesAContainerWaits(t *testing.T) {
|
||||
for name, change := range map[string]func(m *catalogue.Manifest){
|
||||
"image": func(m *catalogue.Manifest) { resourceOf(m, "imap")["image"] = "registry.invalid/imap@sha256:2" },
|
||||
"spec": func(m *catalogue.Manifest) { resourceOf(m, "imap")["health"] = map[string]any{"kind": "runtime"} },
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
rm := aRecordedMesh(t, aMailu(true))
|
||||
c2 := aMailu(true)
|
||||
change(&c2)
|
||||
c2.Bundles[0].Digest = "sha256:bbb"
|
||||
rm.register(t, c2, "c2", time.Now().Add(-time.Minute))
|
||||
p := rm.walk(t)
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if got := rm.sentBuild(t, n); got != "c1" {
|
||||
t.Fatalf("%s was sent mailu %s; want c1 kept, waiting for a push", n, got)
|
||||
}
|
||||
if said := p.Modules["mailu"].Recorded[n]; !strings.HasPrefix(said, "waits: recreates 1 of mailu's 1") {
|
||||
t.Fatalf("the walk says of mailu on %s %q", n, said)
|
||||
}
|
||||
if held := rm.recordedWaitsOn(t, n); len(held) != 1 || !strings.Contains(held[0], "recreates") {
|
||||
t.Fatalf("status names %s behind for %v", n, held)
|
||||
}
|
||||
}
|
||||
// §2: the next build changes only the bundle, and the machines were last sent c1: still held.
|
||||
c3 := c2
|
||||
c3.Bundles = []catalogue.Bundle{{Name: "tools", Source: "store/mailu-tools@sha256:ccc", Digest: "sha256:ccc"}}
|
||||
rm.register(t, c3, "c3", time.Now())
|
||||
if kept := rm.keptOn(t, "anchor"); kept["mailu"] != "c1" {
|
||||
t.Fatalf("a bundle-only build on top of a held container change is not held: %v", kept)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// §3: a move touching a file a container mounts, a directory holding declared data, the data declaration or a
|
||||
// provider waits.
|
||||
func TestARecordedModulesMoveThatTouchesDataOrAProviderWaits(t *testing.T) {
|
||||
for name, c := range map[string]struct {
|
||||
change func(m *catalogue.Manifest)
|
||||
says string
|
||||
}{
|
||||
"a mounted file": {func(m *catalogue.Manifest) { resourceOf(m, "config-env")["content"] = "A=2" }, "which its container imap mounts"},
|
||||
"a data directory": {func(m *catalogue.Manifest) { resourceOf(m, "data-mail")["mode"] = "0700" }, "data-mail"},
|
||||
"the data declaration": {func(m *catalogue.Manifest) {
|
||||
m.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Path: "${dir:data-mail}", Class: "irreplaceable", Why: "x"}}}
|
||||
}, "data declaration"},
|
||||
"a provider": {func(m *catalogue.Manifest) { m.Requires = append(m.Requires, "postgres-database") }, "what it requires"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
rm := aRecordedMesh(t, aMailu(true))
|
||||
c2 := aMailu(true)
|
||||
c.change(&c2)
|
||||
rm.register(t, c2, "c2", time.Now().Add(-time.Minute))
|
||||
kept := rm.keptOn(t, "anchor")
|
||||
if kept["mailu"] != "c1" {
|
||||
t.Fatalf("an ordinary send does not keep mailu at c1: %v", kept)
|
||||
}
|
||||
held := rm.recordedWaitsOn(t, "anchor")
|
||||
if len(held) != 1 || !strings.Contains(held[0], c.says) {
|
||||
t.Fatalf("status says %v; want it to name %q", held, c.says)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// §5: a module a person set to record, and the bus, wait on a bundle-only move.
|
||||
func TestAPersonsRecordAndTheBusHoldEveryMove(t *testing.T) {
|
||||
t.Run("a person sets record", func(t *testing.T) {
|
||||
rm := aRecordedMesh(t, aMailu(false))
|
||||
c2 := aMailu(false)
|
||||
c2.Bundles[0].Digest = "sha256:bbb"
|
||||
rm.register(t, c2, "c2", time.Now().Add(-time.Minute))
|
||||
if err := rm.open.inventory.SetUpgradeOf(t.Context(), "mailu", inventory.Upgrade{Why: "the migration week", By: "jochen"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := rm.walk(t)
|
||||
if got := rm.sentBuild(t, "anchor"); got != "c1" {
|
||||
t.Fatalf("a person's record was walked past: anchor was sent %s", got)
|
||||
}
|
||||
if said := p.Modules["mailu"].Recorded["anchor"]; !strings.Contains(said, "a person set it to record") {
|
||||
t.Fatalf("the walk says %q", said)
|
||||
}
|
||||
if held := rm.recordedWaitsOn(t, "anchor"); len(held) != 1 {
|
||||
t.Fatalf("status names anchor behind for %v", held)
|
||||
}
|
||||
})
|
||||
t.Run("the bus", func(t *testing.T) {
|
||||
bus := aMailu(false)
|
||||
bus.Module = "nats"
|
||||
bus.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
||||
bus.Data = nil
|
||||
rm := aRecordedMesh(t, bus)
|
||||
c2 := bus
|
||||
c2.Bundles = []catalogue.Bundle{{Name: "tools", Source: "store/nats-tools@sha256:bbb", Digest: "sha256:bbb"}}
|
||||
rm.register(t, c2, "c2", time.Now().Add(-time.Minute))
|
||||
if kept := rm.keptOn(t, "anchor"); kept["nats"] != "c1" {
|
||||
t.Fatalf("the bus moved by an ordinary send: %v", kept)
|
||||
}
|
||||
f, err := readMoveFacts(t.Context(), rm.open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v := f.recordedMove("nats", "c1"); v.rolls || !strings.Contains(v.said, "bus") {
|
||||
t.Fatalf("the bus's move: %+v", v)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// §4: a move whose earlier build is not in the build records cannot be judged, waits, and is said so.
|
||||
func TestARecordedMoveThatCannotBeJudgedWaits(t *testing.T) {
|
||||
rm := aRecordedMesh(t, aMailu(true))
|
||||
c2 := aMailu(true)
|
||||
c2.Bundles[0].Digest = "sha256:bbb"
|
||||
rm.register(t, c2, "c2", time.Now().Add(-time.Minute))
|
||||
// anchor was last sent a build the records do not hold.
|
||||
if err := rm.open.inventory.RecordSent(t.Context(), nodeID(t, rm.open, "anchor"), "d-anchor-c0",
|
||||
map[string]string{"mailu": "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := readMoveFacts(t.Context(), rm.open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v := f.recordedMove("mailu", "c0"); v.rolls || !strings.Contains(v.said, "could not be judged") ||
|
||||
!strings.Contains(v.said, "not in the build records") {
|
||||
t.Fatalf("an unjudgeable move: %+v", v)
|
||||
}
|
||||
if _, _, err := planFor(keepingRecorded(t.Context()), rm.open, "anchor"); err == nil || !strings.Contains(err.Error(), "push anchor") {
|
||||
t.Fatalf("a send that cannot keep the unjudged build did not refuse with its remedy: %v", err)
|
||||
}
|
||||
// laptop, judged from c1, goes; anchor waits.
|
||||
p := rm.walk(t)
|
||||
if said := p.Modules["mailu"].Recorded["anchor"]; !strings.Contains(said, "could not be judged") {
|
||||
t.Fatalf("the walk says of anchor %q", said)
|
||||
}
|
||||
if !slices.Contains(wentOut(p.Modules["mailu"].Recorded), "laptop") || rm.sentBuild(t, "laptop") != "c2" {
|
||||
t.Fatalf("laptop, judged from c1, did not go: %+v", p.Modules["mailu"].Recorded)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// A membership issued again for what it hears changes nothing else of it (novox/hq ADR 0363 §6).
|
||||
func TestAMembershipIssuedAgainChangesOnlyWhatItHears(t *testing.T) {
|
||||
issued := []byte(`{"node":"ace","module":"reader","receives":{"x":[1]},"emits":"mesh.mod.reader.event.{event}"}`)
|
||||
hears := []broker.HeardIssued{{Stream: "MAIL", Consumer: "ace_reader"}}
|
||||
body, changed, err := withHearing(issued, hears)
|
||||
if err != nil || !changed {
|
||||
t.Fatalf("changed %v, %v", changed, err)
|
||||
}
|
||||
var got map[string]any
|
||||
if err := json.Unmarshal(body, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got["receives"] == nil || got["emits"] != "mesh.mod.reader.event.{event}" || got["hears"] == nil {
|
||||
t.Fatalf("issued again as %s", body)
|
||||
}
|
||||
if _, again, _ := withHearing(body, hears); again {
|
||||
t.Fatal("the same hearing issued again counts as a change")
|
||||
}
|
||||
gone, changed, err := withHearing(body, nil)
|
||||
if err != nil || !changed {
|
||||
t.Fatalf("a withdrawal: %v, %v", changed, err)
|
||||
}
|
||||
got = nil
|
||||
if err := json.Unmarshal(gone, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, still := got["hears"]; still {
|
||||
t.Fatalf("a withdrawn hearing is still issued: %s", gone)
|
||||
}
|
||||
}
|
||||
+591
-44
@@ -39,8 +39,8 @@ import (
|
||||
// from the same manifest is no move.
|
||||
//
|
||||
// **The release plan** is what walks the waiting moves through: whenever moves wait for a gate that no
|
||||
// started plan is walking, the mesh opens one — every such machine, one at a time, the control node last,
|
||||
// each sent and judged before the next. A release plan that fails stops the next one opening on its own:
|
||||
// started plan is walking, the mesh opens one — every such machine, the first sent and judged, then the rest
|
||||
// at once, each judged by its own gate (novox/hq ADR 0353, in place of one at a time). A release plan that fails stops the next one opening on its own:
|
||||
// a person releases it again (`upgrade release-backlog --why`), after the condition says why.
|
||||
|
||||
// sendScope is what a send may carry that no gate has seen.
|
||||
@@ -81,6 +81,18 @@ type moveFacts struct {
|
||||
srcs map[string]map[string]string
|
||||
passed map[string]map[string]bool
|
||||
plans []inventory.Plan
|
||||
// self is the release walking these moves, whose own gates are no other walk's (novox/hq ADR 0353).
|
||||
self string
|
||||
// judge judges a move under a module's own `record` (novox/hq ADR 0377): nil when the facts were not read,
|
||||
// and every such move then waits.
|
||||
judge func(module, was, now string) recordVerdict
|
||||
}
|
||||
|
||||
type walkingKey struct{}
|
||||
|
||||
// withWalking says a release plan is the one sending: its own gates never hold its own sends back.
|
||||
func withWalking(ctx context.Context, id string) context.Context {
|
||||
return context.WithValue(ctx, walkingKey{}, id)
|
||||
}
|
||||
|
||||
func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, error) {
|
||||
@@ -99,6 +111,8 @@ func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, er
|
||||
return f, err
|
||||
}
|
||||
f.plans, err = inv.OpenPlans(ctx)
|
||||
f.self, _ = ctx.Value(walkingKey{}).(string)
|
||||
f.judge = recordJudge(ctx, inv)
|
||||
return f, err
|
||||
}
|
||||
|
||||
@@ -162,7 +176,8 @@ func inRun(run map[string]bool, commit string) bool {
|
||||
}
|
||||
|
||||
// moves is what a machine's next send would move that no gate has seen: modules whose policy rolls out
|
||||
// (a recorded one is a person's push), that the machine was sent before, moving to a build not identical
|
||||
// (a recorded one is a person's push, unless its move under its own record recreates nothing and moves no data —
|
||||
// novox/hq ADR 0377), that the machine was sent before, moving to a build not identical
|
||||
// to the one it runs and that has passed no gate. A machine whose last send's builds are not known names
|
||||
// none: the cascade holds it whole (ADR 0221).
|
||||
//
|
||||
@@ -175,7 +190,7 @@ func (f moveFacts) moves(node string, modules []string, sent map[string]string,
|
||||
for _, m := range modules {
|
||||
now := f.current[m]
|
||||
was, carried := sent[m]
|
||||
if !now.RollOut || !carried || f.identical(m, was, now.Commit) || (!all && f.gated(m, now.Commit)) {
|
||||
if !carried || f.identical(m, was, now.Commit) || (!all && f.gated(m, now.Commit)) || !f.rolls(m, was) {
|
||||
continue
|
||||
}
|
||||
out = append(out, inventory.CarriedMove{Module: m, Node: node, From: was, To: now.Commit})
|
||||
@@ -193,17 +208,24 @@ func (f moveFacts) moves(node string, modules []string, sent map[string]string,
|
||||
// feet. A release keeps no module records: its open gate's carried moves are its walk.
|
||||
func (f moveFacts) walkedBy(module, node, to string) string {
|
||||
for _, p := range f.plans {
|
||||
if !p.Open() {
|
||||
if !p.Open() || (p.Release != nil && p.ID == f.self) {
|
||||
continue
|
||||
}
|
||||
if p.Release != nil {
|
||||
g := p.Release.Gate
|
||||
if g == nil || g.Verdict != "" {
|
||||
continue
|
||||
// The first machine's gate, and every gate of the rest still judging (novox/hq ADR 0353).
|
||||
gates := []*inventory.PlanGate{p.Release.Gate}
|
||||
for _, g := range p.Release.Gates {
|
||||
gates = append(gates, g)
|
||||
}
|
||||
for _, c := range g.Carried {
|
||||
if c.Module == module && !(slices.Contains(g.Machines, node) && sameCommit(c.To, to)) {
|
||||
return p.ID
|
||||
for _, g := range gates {
|
||||
// A send kept and not yet made walks nothing yet (ADR 0353 §8).
|
||||
if g == nil || g.Verdict != "" || g.Silent || g.Pending {
|
||||
continue
|
||||
}
|
||||
for _, c := range g.Carried {
|
||||
if c.Module == module && !(slices.Contains(g.Machines, node) && sameCommit(c.To, to)) {
|
||||
return p.ID
|
||||
}
|
||||
}
|
||||
}
|
||||
continue
|
||||
@@ -303,8 +325,8 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
|
||||
}
|
||||
}
|
||||
if len(refused) > 0 {
|
||||
return nil, fmt.Errorf("%w: %s — a walk sends them, one machine at a time, each judged (novox/hq ADR "+
|
||||
"0236); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; "))
|
||||
return nil, fmt.Errorf("%w: %s — a walk sends them, one machine first and then the rest at once, each judged (novox/hq ADR "+
|
||||
"0236, ADR 0353); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; "))
|
||||
}
|
||||
return kept, nil
|
||||
}
|
||||
@@ -320,19 +342,35 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
|
||||
// And the grants step first, when the send changes what the bus's user list must say (novox/hq issue 490):
|
||||
// answered for the gate to keep, nil when there was none.
|
||||
func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, []string, *inventory.GrantsStep, error) {
|
||||
inv := open.inventory
|
||||
f, err := readMoveFacts(ctx, inv)
|
||||
moves, err := gatedMoves(ctx, open, node, owns)
|
||||
if err != nil || len(moves) == 0 {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
sayRecreations(ctx, open, moves)
|
||||
sent, grants, err := sendJudged(ctx, open, node)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
return moves, sent, grants, nil
|
||||
}
|
||||
|
||||
// gatedMoves is what a gated send to one machine would carry, with the build each moves to, made of nothing:
|
||||
// gatedSend's reading, which a walk also keeps before it sends (novox/hq ADR 0353 §8). Refused the same way
|
||||
// while a plan that has started walks one of those builds elsewhere.
|
||||
func gatedMoves(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, error) {
|
||||
inv := open.inventory
|
||||
f, err := readMoveFacts(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
moves, err := machineMoves(ctx, open, f, node, true)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
return nil, err
|
||||
}
|
||||
// A move the send keeps is not made, and is no move for the gate to judge (novox/hq issue 201): a core module
|
||||
// kept over an older record judged as moving back to it would read the gate as superseded by its own send.
|
||||
if moves, err = withoutKept(ctx, open, node, moves); err != nil {
|
||||
return nil, nil, nil, err
|
||||
return nil, err
|
||||
}
|
||||
own := func(module string) bool {
|
||||
return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module })
|
||||
@@ -342,7 +380,7 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
|
||||
continue
|
||||
}
|
||||
if id := f.walkedBy(mv.Module, node, mv.To); id != "" {
|
||||
return nil, nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
||||
return nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
||||
mv.Module, short(mv.To), node, id)
|
||||
}
|
||||
}
|
||||
@@ -350,7 +388,7 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
|
||||
// plan, or a release, on this machine is not crossed with a newer build; this send waits for its gate.
|
||||
for _, o := range owns {
|
||||
if id := f.walkedBy(o.Module, node, o.To); id != "" {
|
||||
return nil, nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
||||
return nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
||||
o.Module, short(o.To), node, id)
|
||||
}
|
||||
}
|
||||
@@ -363,30 +401,26 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
|
||||
moves[i].Build = o.Build
|
||||
}
|
||||
}
|
||||
if len(moves) == 0 && len(owns) == 0 {
|
||||
return nil, nil, nil, nil
|
||||
if len(moves) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
for i := range moves {
|
||||
if moves[i].Build == "" {
|
||||
if moves[i].Build, err = inv.BuildOf(ctx, moves[i].Module, moves[i].To); err != nil {
|
||||
return nil, nil, nil, err
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(moves, func(i, j int) bool { return moves[i].Module < moves[j].Module })
|
||||
sayRecreations(ctx, open, moves)
|
||||
sent, grants, err := sendJudged(ctx, open, node)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
return moves, sent, grants, nil
|
||||
return moves, nil
|
||||
}
|
||||
|
||||
// passCarried keeps a pass as the verdict of every build the gate judged beside its own module.
|
||||
func passCarried(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) {
|
||||
seen := map[string]bool{}
|
||||
for _, c := range g.Carried {
|
||||
if c.Module == except || c.Build == "" || seen[c.Build] {
|
||||
// A build put back with another machine's failure is not this gate's to pass (novox/hq ADR 0353 §3).
|
||||
if c.Module == except || c.Build == "" || seen[c.Build] || slices.Contains(g.Returned, c.Module) {
|
||||
continue
|
||||
}
|
||||
seen[c.Build] = true
|
||||
@@ -695,7 +729,7 @@ func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Pl
|
||||
p := inventory.Plan{ID: fmt.Sprintf("release-%d", now.UnixNano()), Repository: releaseRepository,
|
||||
Created: now, State: inventory.PlanRolling, Tiers: [][]string{names}, Modules: map[string]*inventory.PlanModule{},
|
||||
Release: &inventory.PlanRelease{Order: order, By: by},
|
||||
Note: fmt.Sprintf("%d build(s) wait for a gate on %s; one machine at a time, each judged", len(names),
|
||||
Note: fmt.Sprintf("%d build(s) wait for a gate on %s; one machine first, then the rest at once, each judged", len(names),
|
||||
strings.Join(order, ", "))}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
return nil, err
|
||||
@@ -704,14 +738,28 @@ func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Pl
|
||||
return &p, nil
|
||||
}
|
||||
|
||||
// advanceRelease takes one step of a release plan: the next machine sent everything waiting there under a
|
||||
// gate, or the machine being judged judged once more; the plan done when every machine is.
|
||||
// advanceRelease takes one step of a release plan: the first machine sent everything waiting there under a
|
||||
// gate and judged; once it passed, every other machine sent at once, each under its own gate (novox/hq ADR
|
||||
// 0353 §1), and judged; the plan done when every machine is.
|
||||
//
|
||||
// **A send is kept before it is made** (ADR 0353 §8): the step that chooses a machine keeps its gate pending with
|
||||
// what it will carry, and the next step makes the send. A controller replaced between the send and keeping it
|
||||
// finds the gate pending, reads the machine's last send, and neither sends again nor leaves the machine unjudged.
|
||||
func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, error) {
|
||||
r := p.Release
|
||||
if r.Settings != nil {
|
||||
return advanceSettingsWalk(ctx, open, p)
|
||||
}
|
||||
ctx = withWalking(ctx, p.ID)
|
||||
now := time.Now().UTC()
|
||||
if r.Rest {
|
||||
return advanceRest(ctx, open, p, now)
|
||||
}
|
||||
if r.Gate == nil && !gateFirstMachine {
|
||||
// **No first machine** (novox/hq ADR 0360 §7): every machine heard from is sent at once, in one step.
|
||||
r.Rest = true
|
||||
return advanceRest(ctx, open, p, now)
|
||||
}
|
||||
if r.Gate == nil {
|
||||
heard, err := releaseHeard(ctx, open)
|
||||
if err != nil {
|
||||
@@ -724,7 +772,7 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
||||
r.Next++
|
||||
continue
|
||||
}
|
||||
moves, sent, grants, err := gatedSend(ctx, open, node, nil)
|
||||
moves, err := gatedMoves(ctx, open, node, nil)
|
||||
if errors.Is(err, errWalkedElsewhere) {
|
||||
note := fmt.Sprintf("waiting before %s: %v", node, err)
|
||||
changed := p.Note != note
|
||||
@@ -732,23 +780,15 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
||||
return changed, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("sending %s what waits there: %w", node, err)
|
||||
return false, fmt.Errorf("reading what waits on %s: %w", node, err)
|
||||
}
|
||||
if len(moves) == 0 {
|
||||
r.Done = append(r.Done, node)
|
||||
r.Next++
|
||||
continue
|
||||
}
|
||||
r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves, Sent: sentNow(ctx, open.inventory, sent),
|
||||
Grants: grants}
|
||||
p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves))
|
||||
if said := grantsSaid(grants); said != "" {
|
||||
p.Note += "; " + said
|
||||
}
|
||||
if said := recreationsSaid(moves); said != "" {
|
||||
p.Note += "; " + said
|
||||
}
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
r.Gate = &inventory.PlanGate{Machines: []string{node}, Carried: moves, Pending: true}
|
||||
p.Note = fmt.Sprintf("sending %s %d build(s) that wait for a gate, first", node, len(moves))
|
||||
return true, nil
|
||||
}
|
||||
p.State, p.Tier = inventory.PlanDone, len(p.Tiers)
|
||||
@@ -759,6 +799,19 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
||||
return true, nil
|
||||
}
|
||||
g := r.Gate
|
||||
if g.Pending {
|
||||
node := firstOf(g.Machines)
|
||||
out, err := sendKept(ctx, open, p, node, g, now)
|
||||
if err != nil || out == keptWaiting {
|
||||
return out == keptWaiting, err
|
||||
}
|
||||
if out == keptNothing {
|
||||
r.Gate = nil
|
||||
r.Done = append(r.Done, node)
|
||||
r.Next++
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
verdict, err := judgeMoves(ctx, open, g, nil, now)
|
||||
if err != nil {
|
||||
return false, err
|
||||
@@ -777,6 +830,11 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
||||
r.Done = append(r.Done, firstOf(g.Machines))
|
||||
r.Next++
|
||||
r.Gate = nil
|
||||
// **Then the rest at once** (novox/hq ADR 0353 §1): every other machine heard from, each judged by its own
|
||||
// gate, in place of one machine at a time (ADR 0236 §22).
|
||||
r.Rest = true
|
||||
p.Note = fmt.Sprintf("passed on %s; the rest are sent at once, each judged", firstOf(g.Machines))
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
return true, nil
|
||||
case inventory.GateSuperseded:
|
||||
// Another send moved a judged module on the judged machine (novox/hq issue 352): no verdict on what
|
||||
@@ -797,6 +855,495 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// What sendKept did with a kept send.
|
||||
const (
|
||||
keptSent = iota // sent now
|
||||
keptAlready // the machine was already sent it: judged from that send, nothing sent
|
||||
keptNothing // nothing waits there any more: no gate
|
||||
keptWaiting // another walk judges a build it would carry: it waits
|
||||
)
|
||||
|
||||
// sendKept makes a send a gate kept before it was made (novox/hq ADR 0353 §8) — unless the machine's last send
|
||||
// already carries what the gate is for, made before a controller was replaced and before it was kept: then
|
||||
// nothing is sent, and the gate judges from that send (§6).
|
||||
func sendKept(ctx context.Context, open *stores, p *inventory.Plan, node string, g *inventory.PlanGate,
|
||||
now time.Time) (int, error) {
|
||||
inv := open.inventory
|
||||
moves, sent, grants, err := gatedSend(ctx, open, node, nil)
|
||||
if errors.Is(err, errWalkedElsewhere) {
|
||||
p.Note = fmt.Sprintf("waiting before %s: %v", node, err)
|
||||
return keptWaiting, nil
|
||||
}
|
||||
if err != nil {
|
||||
return keptSent, fmt.Errorf("sending %s what waits there: %w", node, err)
|
||||
}
|
||||
if len(moves) > 0 {
|
||||
g.Machines, g.Carried, g.Since, g.Grants, g.Pending = sent, moves, &now, grants, false
|
||||
g.Sent = sentNow(ctx, inv, sent)
|
||||
p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves))
|
||||
if said := grantsSaid(grants); said != "" {
|
||||
p.Note += "; " + said
|
||||
}
|
||||
if said := recreationsSaid(moves); said != "" {
|
||||
p.Note += "; " + said
|
||||
}
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
return keptSent, nil
|
||||
}
|
||||
if !alreadySent(ctx, inv, node, g.Carried) {
|
||||
return keptNothing, nil
|
||||
}
|
||||
since := now
|
||||
if s, found, err := inv.SentTo(ctx, node); err == nil && found {
|
||||
if !s.At.IsZero() {
|
||||
since = s.At
|
||||
}
|
||||
g.Sent = map[string]inventory.SentDeclaration{node: s}
|
||||
}
|
||||
g.Since, g.Pending = &since, false
|
||||
p.Note = fmt.Sprintf("%s was already sent the %d build(s) kept for it, at %s: judged from that send, nothing sent again",
|
||||
node, len(g.Carried), since.Local().Format("15:04:05"))
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
return keptAlready, nil
|
||||
}
|
||||
|
||||
// alreadySent says a machine's last send carries every build of the moves: a send made and not kept.
|
||||
func alreadySent(ctx context.Context, inv *inventory.Inventory, node string, moves []inventory.CarriedMove) bool {
|
||||
if len(moves) == 0 {
|
||||
return false
|
||||
}
|
||||
sent, known, err := inv.SentBuilds(ctx, node)
|
||||
if err != nil || !known {
|
||||
return false
|
||||
}
|
||||
for _, mv := range moves {
|
||||
if !sameCommit(sent[mv.Module], mv.To) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// advanceRest takes one step of a release plan's rest (novox/hq ADR 0353): the sends kept for a step made, every
|
||||
// machine of the step judged by its own gate, one that fails put back by itself while the others go on to their
|
||||
// verdicts, and the next step — or the end — once every gate of the step has its verdict or is silent.
|
||||
func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time.Time) (bool, error) {
|
||||
r := p.Release
|
||||
// The sends kept for this step, in the plan's order: the control node, last in it, last (§1).
|
||||
var pending []string
|
||||
for _, n := range r.Order {
|
||||
if g := r.Gates[n]; g != nil && g.Pending {
|
||||
pending = append(pending, n)
|
||||
}
|
||||
}
|
||||
if len(pending) > 0 {
|
||||
// **A kept send carries only what was kept** (ADR 0353 §2): a build that passed no gate and came to wait on
|
||||
// a machine since its step was chosen could reach a second machine in it; the step is chosen again.
|
||||
if grew, err := keptStepGrew(ctx, open, r, pending); err != nil {
|
||||
return false, err
|
||||
} else if grew != "" {
|
||||
for _, n := range pending {
|
||||
delete(r.Gates, n)
|
||||
}
|
||||
opened, waiting, err := openRestStep(ctx, open, p)
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
p.Note = "chosen again: " + grew
|
||||
if !opened && waiting != "" {
|
||||
p.Note += "; " + waiting
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
var sent, already, waiting []string
|
||||
for _, node := range pending {
|
||||
out, err := sendKept(ctx, open, p, node, r.Gates[node], now)
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
switch out {
|
||||
case keptSent:
|
||||
sent = append(sent, node)
|
||||
case keptAlready:
|
||||
already = append(already, node)
|
||||
case keptNothing:
|
||||
delete(r.Gates, node)
|
||||
r.Done = append(r.Done, node)
|
||||
case keptWaiting:
|
||||
waiting = append(waiting, node)
|
||||
}
|
||||
}
|
||||
var said []string
|
||||
if len(sent) > 0 {
|
||||
said = append(said, "sent "+strings.Join(sent, ", ")+" at once, each judged there")
|
||||
}
|
||||
if len(already) > 0 {
|
||||
said = append(said, strings.Join(already, ", ")+" already sent it: judged from that send")
|
||||
}
|
||||
if len(waiting) > 0 {
|
||||
said = append(said, "waiting before "+strings.Join(waiting, ", ")+": another walk judges a build it carries")
|
||||
}
|
||||
if len(said) > 0 {
|
||||
p.Note = strings.Join(said, "; ")
|
||||
}
|
||||
return len(sent)+len(already) > 0 || len(waiting) < len(pending), nil
|
||||
}
|
||||
reports, err := open.inventory.LastReports(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
byNode := map[string]inventory.Reported{}
|
||||
for _, rep := range reports {
|
||||
byNode[rep.Node] = rep
|
||||
}
|
||||
changed, noteWas := false, p.Note
|
||||
var notes, judging []string
|
||||
batched, back := batchingRollbacks(ctx)
|
||||
for _, node := range r.Order {
|
||||
g := r.Gates[node]
|
||||
if g == nil || g.Verdict != "" || g.Silent {
|
||||
continue
|
||||
}
|
||||
// What another machine's failure already put back is not this gate's to judge (ADR 0353 §3).
|
||||
if len(g.Carried) > 0 && len(g.Returned) >= len(g.Carried) {
|
||||
decide(g, inventory.GateFailed, "what it was sent was put back with another machine's failure", now)
|
||||
r.Failed = append(r.Failed, node)
|
||||
changed = true
|
||||
continue
|
||||
}
|
||||
// **A machine that says nothing on its send is left silent, not failed** (ADR 0353 §4): it is not judged
|
||||
// until it reports, and past ApplySilentAfter it is listed and left.
|
||||
outcome, reported := byNode[node].Outcome, reportedOnSend(byNode[node], node, g)
|
||||
if sent, kept := g.Sent[node]; kept && !gateFirstMachine {
|
||||
// A module handed to the provisioner is applied only once its client said so too (ADR 0362, issue 840).
|
||||
carried := make([]string, 0, len(g.Carried))
|
||||
for _, c := range g.Carried {
|
||||
carried = append(carried, c.Module)
|
||||
}
|
||||
outcome, reported = provisionedOutcome(node, sent.Only(carried...), byNode[node], reported, provLookup(ctx))
|
||||
}
|
||||
if !reported {
|
||||
if now.Sub(*g.Since) > inventory.ApplySilentAfter {
|
||||
g.Silent = true
|
||||
r.Silent = append(r.Silent, node)
|
||||
changed = true
|
||||
notes = append(notes, fmt.Sprintf("%s said nothing on its send within %s: left silent", node,
|
||||
inventory.ApplySilentAfter))
|
||||
continue
|
||||
}
|
||||
judging = append(judging, node+": not yet reported on its send")
|
||||
continue
|
||||
}
|
||||
if !gateFirstMachine {
|
||||
// **Judged by its report alone** (novox/hq ADR 0360 §7): applied passes; failed or refused is named and
|
||||
// fails the walk, with nothing put back — apply-failing says it.
|
||||
switch outcome {
|
||||
case inventory.OutcomeApplied:
|
||||
decide(g, inventory.GatePassed, node+" reported it applied its send (novox/hq ADR 0360)", now)
|
||||
passCarried(ctx, open, p, g, "")
|
||||
r.Done = append(r.Done, node)
|
||||
changed = true
|
||||
case inventory.OutcomeFailed, inventory.OutcomeRefused:
|
||||
decide(g, inventory.GateFailed, fmt.Sprintf("%s %s its send", node, outcome), now)
|
||||
r.Failed = append(r.Failed, node)
|
||||
changed = true
|
||||
notes = append(notes, fmt.Sprintf("%s %s its send: nothing put back, apply-failing says it", node, outcome))
|
||||
default:
|
||||
if strings.HasPrefix(outcome, inventory.OutcomeProvisioningFailed) {
|
||||
decide(g, inventory.GateFailed, fmt.Sprintf("%s: %s", node, outcome), now)
|
||||
r.Failed = append(r.Failed, node)
|
||||
changed = true
|
||||
notes = append(notes, fmt.Sprintf("%s: %s: nothing put back, apply-failing says it", node, outcome))
|
||||
continue
|
||||
}
|
||||
judging = append(judging, node+": not yet reported on its send")
|
||||
}
|
||||
continue
|
||||
}
|
||||
verdict, err := judgeMoves(ctx, open, g, nil, now)
|
||||
if err != nil {
|
||||
return changed, err
|
||||
}
|
||||
switch verdict {
|
||||
case "":
|
||||
if putBackBroken(ctx, open, p, g, "", nil) {
|
||||
changed = true
|
||||
notes = append(notes, p.Note)
|
||||
}
|
||||
judging = append(judging, node+": "+gateLine(g))
|
||||
case inventory.GatePassed:
|
||||
passCarried(ctx, open, p, g, "")
|
||||
r.Done = append(r.Done, node)
|
||||
changed = true
|
||||
case inventory.GateSuperseded:
|
||||
// Another send moved a judged module there (novox/hq issue 352): nothing judged, nothing put back on
|
||||
// it; what still waits is released again.
|
||||
r.Done = append(r.Done, node)
|
||||
changed = true
|
||||
notes = append(notes, fmt.Sprintf("superseded on %s: %s", node, g.Why))
|
||||
default:
|
||||
// **Put back by itself** (ADR 0353 §3): what was found wanting goes back at once, and every other
|
||||
// machine of the step goes on to its own verdict.
|
||||
stateWas := p.State
|
||||
p.Note = ""
|
||||
failCarried(batched, open, p, g, "")
|
||||
p.State = stateWas
|
||||
r.Failed = append(r.Failed, node)
|
||||
changed = true
|
||||
notes = append(notes, fmt.Sprintf("failed its gate on %s: %s — %s", node, g.Why, p.Note))
|
||||
returnedElsewhere(ctx, open, r, node, g)
|
||||
}
|
||||
}
|
||||
p.Note = ""
|
||||
sendRollbacks(ctx, open, p, back)
|
||||
if p.Note != "" {
|
||||
notes = append(notes, strings.TrimPrefix(p.Note, "; "))
|
||||
}
|
||||
if len(judging) > 0 {
|
||||
note := strings.Join(append(notes, "judging "+strings.Join(judging, "; ")), "; ")
|
||||
p.Note = note
|
||||
return changed || noteWas != note, nil
|
||||
}
|
||||
// Every gate of the step has its verdict or is silent: the next step, unless a machine failed (§3).
|
||||
if len(r.Failed) == 0 {
|
||||
opened, waiting, err := openRestStep(ctx, open, p)
|
||||
if err != nil {
|
||||
return changed, err
|
||||
}
|
||||
if opened {
|
||||
return true, nil
|
||||
}
|
||||
if waiting != "" {
|
||||
note := strings.Join(append(notes, waiting), "; ")
|
||||
p.Note = note
|
||||
return changed || noteWas != note, nil
|
||||
}
|
||||
}
|
||||
p.Tier = len(p.Tiers)
|
||||
end := fmt.Sprintf("released on %s", orNone(strings.Join(r.Done, ", ")))
|
||||
if len(r.Silent) > 0 {
|
||||
end += "; silent on their send, left: " + strings.Join(r.Silent, ", ")
|
||||
}
|
||||
if len(r.Skipped) > 0 {
|
||||
end += "; not heard from, left as they were: " + strings.Join(r.Skipped, ", ")
|
||||
}
|
||||
p.State = inventory.PlanDone
|
||||
if len(r.Failed) > 0 {
|
||||
p.State = inventory.PlanFailed
|
||||
end = "failed on " + strings.Join(r.Failed, ", ") + "; " + end
|
||||
} else if len(r.Silent) > 0 && len(r.Gates) == len(r.Silent) {
|
||||
// **A rest that said nothing is no success** (ADR 0353 §4): failure is loud; the condition says it.
|
||||
p.State = inventory.PlanFailed
|
||||
end = "no machine of the rest reported on its send; " + end
|
||||
}
|
||||
p.Note = strings.Join(append(notes, end), "; ")
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// keptStepGrew says why a step's kept sends would now carry a build no gate passed that they were not kept with;
|
||||
// empty when none would.
|
||||
func keptStepGrew(ctx context.Context, open *stores, r *inventory.PlanRelease, pending []string) (string, error) {
|
||||
if !gateFirstMachine {
|
||||
return "", nil // every machine goes at once (novox/hq ADR 0360 §7): a send carries what waits when it is made
|
||||
}
|
||||
f, err := readMoveFacts(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, node := range pending {
|
||||
moves, err := gatedMoves(ctx, open, node, nil)
|
||||
if errors.Is(err, errWalkedElsewhere) {
|
||||
continue // sendKept says it waits
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
kept := r.Gates[node].Carried
|
||||
for _, mv := range moves {
|
||||
// The node-engine and the resolver keep one machine at a time, passed or not (ADR 0353 §9).
|
||||
if (f.gated(mv.Module, mv.To) && !oneAtATime(mv.Module, shelf[mv.Module])) || slices.ContainsFunc(kept, func(c inventory.CarriedMove) bool {
|
||||
return c.Module == mv.Module && sameCommit(c.To, mv.To)
|
||||
}) {
|
||||
continue
|
||||
}
|
||||
return fmt.Sprintf("%s's build %s, which this step did not keep for it, came to wait on %s", mv.Module, short(mv.To), node), nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// reportedOnSend says a machine's last report is about the send its gate judges (novox/hq issue 352): the
|
||||
// declaration kept on the gate, or one sequenced after it; a gate that kept none reads the send made last.
|
||||
func reportedOnSend(r inventory.Reported, node string, g *inventory.PlanGate) bool {
|
||||
if r.At == nil {
|
||||
return false
|
||||
}
|
||||
if s, kept := g.Sent[node]; kept {
|
||||
return s.ReportsOn(r)
|
||||
}
|
||||
return r.Current
|
||||
}
|
||||
|
||||
// returnedElsewhere marks, on every other gate of the rest still judging, the builds a failed machine's
|
||||
// rollback put back wherever they were sent (ADR 0236 §5): no longer theirs to judge (ADR 0353 §3).
|
||||
func returnedElsewhere(ctx context.Context, open *stores, r *inventory.PlanRelease, failed string, g *inventory.PlanGate) {
|
||||
for _, c := range g.Carried {
|
||||
if c.Build == "" {
|
||||
continue
|
||||
}
|
||||
if marked, err := open.inventory.GateFailed(ctx, c.Build); err != nil || !marked {
|
||||
continue
|
||||
}
|
||||
for node, other := range r.Gates {
|
||||
if node == failed || other.Verdict != "" || slices.Contains(other.Returned, c.Module) {
|
||||
continue
|
||||
}
|
||||
for _, oc := range other.Carried {
|
||||
if oc.Module == c.Module && sameCommit(oc.To, c.To) {
|
||||
other.Returned = append(other.Returned, c.Module)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// openRestStep chooses the next step of a release plan's rest and keeps its sends (novox/hq ADR 0353 §1, §2):
|
||||
// every machine left that is heard from and has something waiting, except that a build no gate has passed goes
|
||||
// to one machine of the step only. Answers whether a step was kept, or why the walk waits.
|
||||
func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, string, error) {
|
||||
r := p.Release
|
||||
heard, err := releaseHeard(ctx, open)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
f, err := readMoveFacts(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
waiting := map[string][]inventory.CarriedMove{}
|
||||
var nodes, walked []string
|
||||
for _, node := range r.Order[min(r.Next, len(r.Order)):] {
|
||||
if r.Gates[node] != nil || slices.Contains(r.Done, node) || slices.Contains(r.Skipped, node) ||
|
||||
slices.Contains(r.Failed, node) || slices.Contains(r.Silent, node) {
|
||||
continue
|
||||
}
|
||||
if !heard[node] {
|
||||
r.Skipped = append(r.Skipped, node)
|
||||
continue
|
||||
}
|
||||
moves, err := gatedMoves(ctx, open, node, nil)
|
||||
if errors.Is(err, errWalkedElsewhere) {
|
||||
walked = append(walked, fmt.Sprintf("%s (%v)", node, err))
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return false, "", fmt.Errorf("reading what waits on %s: %w", node, err)
|
||||
}
|
||||
if len(moves) == 0 {
|
||||
r.Done = append(r.Done, node)
|
||||
continue
|
||||
}
|
||||
waiting[node] = moves
|
||||
nodes = append(nodes, node)
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
// **Every machine at once** (novox/hq ADR 0360 §2, §7): no build waits for one machine to pass first.
|
||||
step := nodes
|
||||
if gateFirstMachine {
|
||||
step = restStep(nodes, waiting, f.gated, func(module string) bool { return oneAtATime(module, shelf[module]) })
|
||||
}
|
||||
if len(step) == 0 {
|
||||
if len(walked) > 0 {
|
||||
return false, "waiting before " + strings.Join(walked, "; "), nil
|
||||
}
|
||||
return false, "", nil
|
||||
}
|
||||
if r.Gates == nil {
|
||||
r.Gates = map[string]*inventory.PlanGate{}
|
||||
}
|
||||
for _, node := range step {
|
||||
r.Gates[node] = &inventory.PlanGate{Machines: []string{node}, Carried: waiting[node], Pending: true}
|
||||
}
|
||||
p.Note = fmt.Sprintf("sending %s at once, each judged there", strings.Join(step, ", "))
|
||||
if left := len(nodes) - len(step); left > 0 {
|
||||
p.Note += fmt.Sprintf("; %d machine(s) wait for a build that passed no gate yet to pass on one", left)
|
||||
}
|
||||
return true, "", nil
|
||||
}
|
||||
|
||||
// oneAtATime is a module whose builds go to the rest one machine at a time (novox/hq ADR 0353 §9): the
|
||||
// node-engine, which applies every send, and a module holding the mesh's resolver, which names the bus.
|
||||
func oneAtATime(module string, m catalogue.Manifest) bool {
|
||||
if module == hostModule {
|
||||
return true
|
||||
}
|
||||
return slices.ContainsFunc(m.Claims, func(c catalogue.Claim) bool { return c.Name == resolverSeat })
|
||||
}
|
||||
|
||||
// restStep is the machines of one step of the rest (novox/hq ADR 0353 §2), in the order given: each machine, unless
|
||||
// a build it would carry has passed no gate and an earlier machine of the step already carries it — that build's
|
||||
// first machine. Pure.
|
||||
//
|
||||
// **The node-engine and the mesh's resolver still go one machine at a time** (ADR 0353 §9): a step carries a
|
||||
// build of either to one machine only, judged before the next, since a fault that cuts a machine off the bus
|
||||
// shows only as its silence and no rollback reaches it.
|
||||
func restStep(nodes []string, waiting map[string][]inventory.CarriedMove, passed func(module, commit string) bool,
|
||||
alone func(module string) bool) []string {
|
||||
first := map[string]string{}
|
||||
var out []string
|
||||
for _, n := range nodes {
|
||||
var unpassed []string
|
||||
for _, mv := range waiting[n] {
|
||||
if !passed(mv.Module, mv.To) || alone(mv.Module) {
|
||||
unpassed = append(unpassed, mv.Module+"@"+mv.To)
|
||||
}
|
||||
}
|
||||
if slices.ContainsFunc(unpassed, func(k string) bool { _, taken := first[k]; return taken }) {
|
||||
continue
|
||||
}
|
||||
for _, k := range unpassed {
|
||||
first[k] = n
|
||||
}
|
||||
out = append(out, n)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// silentObservation is what the gate's probe says of the newest release whose rest went silent (novox/hq ADR 0353
|
||||
// §4): machines that never reported on their send, left as they were — loud, never a success said by nothing.
|
||||
func silentObservation(ctx context.Context, inv *inventory.Inventory) ([]conditions.Observation, error) {
|
||||
recent, err := inv.RecentPlans(ctx, 20)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, p := range recent {
|
||||
if p.Release == nil || p.Release.Settings != nil || p.Open() {
|
||||
continue
|
||||
}
|
||||
if len(p.Release.Silent) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
machines := append([]string(nil), p.Release.Silent...)
|
||||
sort.Strings(machines)
|
||||
o := conditions.Observation{Scope: conditions.ScopeMesh, ID: "release", Token: "silent", Kind: "release-silent",
|
||||
Severity: conditions.Warning, Resolver: conditions.ResolverOperator, Also: machines,
|
||||
Summary: fmt.Sprintf("%s: %s said nothing on the send of the builds that waited for a gate within %s, and were "+
|
||||
"left as they were", p.ID, strings.Join(machines, ", "), inventory.ApplySilentAfter)}
|
||||
w := releaseSilentWords(machines)
|
||||
o.Headline, o.Explanation, o.Resolved, o.Needs = w.Headline, w.Explanation, w.Resolved, w.Needs
|
||||
return []conditions.Observation{o}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// backlogObservation is what the gate's probe says of a release held after a failure.
|
||||
func backlogObservation() []conditions.Observation {
|
||||
if backlogNow.held == "" || len(backlogNow.waiting) == 0 {
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A pull request into a release branch is judged by that branch** (novox/hq ADR 0358, issue 704): a base that is
|
||||
// not its repository's trunk, where mesh-controller has a branch of that name, is a release branch; the check then
|
||||
// reads every repository beside it at its branch of that name where it has one, else at its trunk. Any other base is
|
||||
// checked exactly as before.
|
||||
|
||||
const aRelease = "feat/57-config-and-provisioning"
|
||||
|
||||
// theCore is the mesh's core as a check reads it beside: the controller, the node-engine, the tool runner and the
|
||||
// catalogue's bus, on the git seat.
|
||||
func theCore() []inventory.Entry {
|
||||
onSeat := func(module, repository string) inventory.Entry {
|
||||
e := fromRepo(module, repository, "")
|
||||
e.Source.Seat = "git"
|
||||
return e
|
||||
}
|
||||
nats := onSeat("nats", "novox/mesh-catalog")
|
||||
nats.Source.Path = "modules/nats"
|
||||
return []inventory.Entry{onSeat("mesh-controller", "novox/mesh-controller"), onSeat("mesh-host", "novox/mesh-host"),
|
||||
onSeat("node-tools", "novox/mesh-tools"), nats, onSeat("gitea", "novox/mesh-catalog")}
|
||||
}
|
||||
|
||||
// aForgeWithBranches stands in for the forge: which branches each repository has, and how often it was asked.
|
||||
func aForgeWithBranches(t *testing.T, branches map[string][]string, failing error) *int {
|
||||
t.Helper()
|
||||
asked := 0
|
||||
was := forgeHasBranch
|
||||
t.Cleanup(func() { forgeHasBranch = was })
|
||||
forgeHasBranch = func(_ context.Context, owner, repo, branch string) (bool, error) {
|
||||
asked++
|
||||
if failing != nil {
|
||||
return false, failing
|
||||
}
|
||||
for _, b := range branches[owner+"/"+repo] {
|
||||
if b == branch {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
return &asked
|
||||
}
|
||||
|
||||
func TestABaseIsAReleaseBranchWhenTheControllerHasABranchOfThatName(t *testing.T) {
|
||||
entries := theCore()
|
||||
scopeOf := func(p link.PullUpdated) checkScope { return pullScope(p, entries, nil, nil) }
|
||||
pull := func(repo, base string) link.PullUpdated {
|
||||
return link.PullUpdated{Owner: "novox", Repo: repo, Number: 286, Base: base, Commit: "abc",
|
||||
Paths: []string{"modules/gitea/index.ts"}}
|
||||
}
|
||||
|
||||
asked := aForgeWithBranches(t, map[string][]string{"novox/mesh-controller": {"main", aRelease}}, nil)
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
p link.PullUpdated
|
||||
release string
|
||||
asks int
|
||||
}{
|
||||
{"a pull request into the trunk asks nothing and is no release", pull("mesh-catalog", "main"), "", 0},
|
||||
{"a catalogue pull request into the release branch", pull("mesh-catalog", aRelease), aRelease, 1},
|
||||
{"a controller pull request into the release branch", pull("mesh-controller", aRelease), aRelease, 1},
|
||||
{"a base mesh-controller has no branch of is checked as today", pull("mesh-catalog", "feat/other"), "", 1},
|
||||
{"no base: as today", pull("mesh-catalog", ""), "", 0},
|
||||
} {
|
||||
*asked = 0
|
||||
got, err := releaseBranchOf(t.Context(), c.p, scopeOf(c.p), entries)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", c.what, err)
|
||||
}
|
||||
if got != c.release || *asked != c.asks {
|
||||
t.Errorf("%s: release %q after %d question(s) to the forge, not %q after %d", c.what, got, *asked,
|
||||
c.release, c.asks)
|
||||
}
|
||||
}
|
||||
|
||||
// The branch a module built from the repository follows is its trunk: never asked about.
|
||||
photos := fromRepo("photos", "novox/photos", "")
|
||||
photos.Source.Ref = "nox-mesh"
|
||||
withPhotos := append(theCore(), photos)
|
||||
p := link.PullUpdated{Owner: "novox", Repo: "photos", Base: "nox-mesh", Commit: "abc", Paths: []string{"x.js"}}
|
||||
*asked = 0
|
||||
if got, err := releaseBranchOf(t.Context(), p, pullScope(p, withPhotos, nil, nil), withPhotos); err != nil ||
|
||||
got != "" || *asked != 0 {
|
||||
t.Fatalf("a module's own trunk read as release %q (%v) after %d question(s)", got, err, *asked)
|
||||
}
|
||||
|
||||
// A forge that cannot answer is an error, never "no release branch".
|
||||
aForgeWithBranches(t, nil, errors.New("nothing serves gitea.gitea_api"))
|
||||
q := pull("mesh-catalog", aRelease)
|
||||
if _, err := releaseBranchOf(t.Context(), q, scopeOf(q), entries); err == nil ||
|
||||
!strings.Contains(err.Error(), "cannot be told") {
|
||||
t.Fatalf("an unanswered forge was read as an answer: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBesideAReleaseBranchCheckEachRepositoryIsReadAtThatBranchElseItsTrunk(t *testing.T) {
|
||||
entries := theCore()
|
||||
clone := func(s inventory.Source) (string, error) { return "http://git/" + s.Repository + ".git", nil }
|
||||
running := func(module string) string { return "running-" + module }
|
||||
|
||||
beside, err := besideOf(entries, running, clone, aRelease)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
at := func(repo string, ref, orElse string) link.CheckedOut {
|
||||
return link.CheckedOut{Repository: "http://git/novox/" + repo + ".git", Ref: ref, Else: orElse}
|
||||
}
|
||||
want := map[string]link.CheckedOut{
|
||||
// The judge: mesh-controller's release branch, which it has.
|
||||
"mesh-controller": at("mesh-controller", aRelease, ""),
|
||||
"mesh-controller-main": at("mesh-controller", "main", ""),
|
||||
// A controller pull request into the release branch reads mesh-host at that branch where it has one.
|
||||
"mesh-host": at("mesh-host", aRelease, "main"),
|
||||
"mesh-tools": at("mesh-tools", aRelease, "main"),
|
||||
"mesh-catalog": at("mesh-catalog", aRelease, "main"),
|
||||
// mesh-lab at its trunk, always: its replays run with the container runtime's socket.
|
||||
"mesh-lab": at("mesh-lab", "main", ""),
|
||||
"mesh-sdk": at("mesh-sdk", aRelease, "main"),
|
||||
}
|
||||
if !reflect.DeepEqual(beside, want) {
|
||||
t.Fatalf("beside a release-branch check:\n got %v\nwant %v", beside, want)
|
||||
}
|
||||
|
||||
// Any other base: exactly as before — what the mesh runs, the catalogue's and the lab's main.
|
||||
beside, err = besideOf(entries, running, clone, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want = map[string]link.CheckedOut{
|
||||
"mesh-controller": at("mesh-controller", "running-mesh-controller", ""),
|
||||
"mesh-controller-main": at("mesh-controller", "main", ""),
|
||||
"mesh-host": at("mesh-host", "running-mesh-host", ""),
|
||||
"mesh-tools": at("mesh-tools", "running-node-tools", ""),
|
||||
"mesh-catalog": at("mesh-catalog", "main", ""),
|
||||
"mesh-lab": at("mesh-lab", "main", ""),
|
||||
"mesh-sdk": at("mesh-sdk", sdkPinned(), ""),
|
||||
}
|
||||
if !reflect.DeepEqual(beside, want) {
|
||||
t.Fatalf("beside a trunk check:\n got %v\nwant %v", beside, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReleaseBranchVerdictNamesTheBranchTheJudgeAndWhatItRead(t *testing.T) {
|
||||
asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog", Number: 286, Base: aRelease, Release: aRelease}
|
||||
c := checkedOf(link.BuildResult{ID: "b", Ref: "aaaaaaaaaaaa0000", Checked: asked, Check: &link.CheckOutcome{
|
||||
Verdict: "pass", Head: "aaaaaaaaaaaa", Trunk: "111111111111", Gate: &link.CheckLayer{Verdict: "pass"},
|
||||
RepoCheck: &link.CheckLayer{Verdict: "pass"}, Release: aRelease, JudgedBy: "fcdecb436117",
|
||||
Beside: map[string]string{"mesh-host": "222222222222", "mesh-lab": "333333333333"},
|
||||
BesideAt: map[string]string{"mesh-host": aRelease, "mesh-lab": "main"},
|
||||
Validator: "mesh-host's validator of " + aRelease + " at 222222222222"}})
|
||||
if c.Release != aRelease || c.JudgedBy != "fcdecb436117" {
|
||||
t.Fatalf("the verdict names release %q judged by %q", c.Release, c.JudgedBy)
|
||||
}
|
||||
if c.BesideAt["mesh-host"] != aRelease || c.BesideAt["mesh-lab"] != "main" || c.Beside["mesh-host"] != "222222222222" ||
|
||||
c.Validator != "mesh-host's validator of "+aRelease+" at 222222222222" {
|
||||
t.Fatalf("the verdict names what it read beside it as %v at %v", c.Beside, c.BesideAt)
|
||||
}
|
||||
}
|
||||
|
||||
// **Clause 5** (ADR 0358 §5): a release branch's own pull request into its trunk is no release-branch check: the forge
|
||||
// is asked nothing, the request names no release branch, and the repositories beside it are read as for any trunk
|
||||
// pull request. Its delivery group's composed check is asked as for any other (askGroupCheck, unchanged).
|
||||
func TestAReleaseBranchsPullRequestIntoItsTrunkIsCheckedAsAnyTrunkPullRequest(t *testing.T) {
|
||||
entries := theCore()
|
||||
asked := aForgeWithBranches(t, map[string][]string{"novox/mesh-controller": {"main", aRelease}}, nil)
|
||||
for _, repo := range []string{"mesh-catalog", "mesh-controller", "mesh-host"} {
|
||||
p := link.PullUpdated{Owner: "novox", Repo: repo, Number: 400, Head: aRelease, Base: "main", Commit: "abc",
|
||||
Paths: []string{"modules/gitea/index.ts"}}
|
||||
got, err := releaseBranchOf(t.Context(), p, pullScope(p, entries, nil, nil), entries)
|
||||
if err != nil || got != "" || *asked != 0 {
|
||||
t.Fatalf("%s's release branch into main read as release %q (%v) after %d question(s)", repo, got, err, *asked)
|
||||
}
|
||||
}
|
||||
clone := func(s inventory.Source) (string, error) { return "http://git/" + s.Repository + ".git", nil }
|
||||
beside, err := besideOf(entries, func(module string) string { return "running-" + module }, clone, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for dir, b := range beside {
|
||||
if b.Ref == aRelease || b.Else != "" {
|
||||
t.Fatalf("beside a trunk pull request %s is read at %+v", dir, b)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **A forge answer that names no branch is an error**, never "no release branch"; a 404 is "no such branch".
|
||||
func TestTheForgesAnswerAboutABranchIsReadStrictly(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
answer link.Answer
|
||||
has bool
|
||||
fails bool
|
||||
}{
|
||||
{"the branch named", link.Answer{Result: []byte(`{"result":{"name":"` + aRelease + `"}}`)}, true, false},
|
||||
{"not found", link.Answer{Error: "Gitea API /repos/novox/mesh-controller/branches/x: 404 {\"message\":\"not found\"}"},
|
||||
false, false},
|
||||
{"an empty branch name", link.Answer{Result: []byte(`{"result":{"name":""}}`)}, false, true},
|
||||
{"no result", link.Answer{Result: []byte(`{}`)}, false, true},
|
||||
{"another branch named", link.Answer{Result: []byte(`{"result":{"name":"main"}}`)}, false, true},
|
||||
{"unreadable", link.Answer{Result: []byte(`not json`)}, false, true},
|
||||
{"another error", link.Answer{Error: "Gitea API: 500 internal"}, false, true},
|
||||
} {
|
||||
has, err := branchAnswered(c.answer, aRelease)
|
||||
if has != c.has || (err != nil) != c.fails {
|
||||
t.Errorf("%s: has %v, error %v", c.what, has, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **The trunk is the repository's own, not a name**: a repository whose trunk is not main, one that follows main, and
|
||||
// a controller whose trunk is not main.
|
||||
func TestATrunkIsTheBranchTheRepositoryFollows(t *testing.T) {
|
||||
asked := aForgeWithBranches(t, map[string][]string{"novox/mesh-controller": {"main", aRelease, "master"}}, nil)
|
||||
legacy := fromRepo("legacy", "novox/legacy", "")
|
||||
legacy.Source.Ref = "master"
|
||||
onMain := fromRepo("photos", "novox/photos", "")
|
||||
onMain.Source.Ref = "main"
|
||||
entries := append(theCore(), legacy, onMain)
|
||||
for _, c := range []struct {
|
||||
what, repo, base, release string
|
||||
asks int
|
||||
}{
|
||||
{"a repository whose trunk is master, into master", "legacy", "master", "", 0},
|
||||
{"a repository whose trunk is master, into the release branch", "legacy", aRelease, aRelease, 1},
|
||||
{"a repository whose module follows main, into main", "photos", "main", "", 0},
|
||||
{"a repository whose module follows main, into the release branch", "photos", aRelease, aRelease, 1},
|
||||
} {
|
||||
*asked = 0
|
||||
p := link.PullUpdated{Owner: "novox", Repo: c.repo, Base: c.base, Commit: "abc", Paths: []string{"x"}}
|
||||
got, err := releaseBranchOf(t.Context(), p, pullScope(p, entries, nil, nil), entries)
|
||||
if err != nil || got != c.release || *asked != c.asks {
|
||||
t.Errorf("%s: release %q (%v) after %d question(s), not %q after %d", c.what, got, err, *asked, c.release, c.asks)
|
||||
}
|
||||
}
|
||||
|
||||
// A controller whose trunk is not main: main may be a release branch there, and its own trunk never is.
|
||||
core := theCore()
|
||||
core[0].Source.Ref = "trunk"
|
||||
for _, c := range []struct {
|
||||
base, release string
|
||||
asks int
|
||||
}{{"trunk", "", 0}, {"main", "main", 1}} {
|
||||
*asked = 0
|
||||
p := link.PullUpdated{Owner: "novox", Repo: "hq", Base: c.base, Commit: "abc", Paths: []string{"x"}}
|
||||
got, err := releaseBranchOf(t.Context(), p, pullScope(p, core, nil, nil), core)
|
||||
if err != nil || got != c.release || *asked != c.asks {
|
||||
t.Errorf("hq into %s, the controller following trunk: release %q (%v) after %d question(s)", c.base, got, err,
|
||||
*asked)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// trunkOf: the branch a source follows, and main when it records none or records a commit (issue 215's rule).
|
||||
func TestATrunkIsMainWhenTheSourceRecordsNone(t *testing.T) {
|
||||
for ref, want := range map[string]string{"": "main", "main": "main", "master": "master", "nox-mesh": "nox-mesh",
|
||||
"0123456789abcdef0123456789abcdef01234567": "main", " ": "main"} {
|
||||
if got := trunkOf(inventory.Source{Ref: ref}); got != want {
|
||||
t.Errorf("a source recording %q follows %q, not %q", ref, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -9,11 +9,13 @@ import (
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
@@ -26,6 +28,10 @@ import (
|
||||
// alone cannot (a tier waiting for machines to report); a controller replaced mid-plan finds the
|
||||
// plan where it left it.
|
||||
|
||||
// gateFirstMachine sends a module that rolls out to one machine first and judges it there (ADR 0236). Off since
|
||||
// novox/hq ADR 0360: the gate's code is kept for the walks a controller before it started, and its tests turn it on.
|
||||
var gateFirstMachine = false
|
||||
|
||||
// planWaitBound is how long a plan may wait on one thing before `status` names it red.
|
||||
const planWaitBound = 30 * time.Minute
|
||||
|
||||
@@ -42,29 +48,15 @@ func tiersOf(set []string, edges []inventory.Edge) [][]string {
|
||||
for _, m := range set {
|
||||
deps[m] = map[string]bool{}
|
||||
}
|
||||
// The build seat's holders follow the controller that defines their worker (EdgeWorkerOf,
|
||||
// novox/hq issue 206), so the built-by edge from that controller to such a holder yields: the
|
||||
// controller is built by whichever build machine is running, as the runtime image always was.
|
||||
worker := map[string]map[string]bool{}
|
||||
for _, e := range edges {
|
||||
if e.Kind == inventory.EdgeWorkerOf && in[e.From] && in[e.To] {
|
||||
if worker[e.To] == nil {
|
||||
worker[e.To] = map[string]bool{}
|
||||
}
|
||||
worker[e.To][e.From] = true
|
||||
}
|
||||
}
|
||||
for _, e := range edges {
|
||||
// A code dependency — B packages A's source — rebuilds B with A, in the same tier: B's
|
||||
// build needs nothing of A's first. The other kinds order: stands-on and declared after
|
||||
// the base is built, built-by after the build machine is built and running — except for
|
||||
// what the build machine itself stands on, and for the controller whose worker the build
|
||||
// machine binds. The runtime image is built by the builder and the builder is built on the
|
||||
// runtime image; the image comes first, built by the builder that is running.
|
||||
// build needs nothing of A's first. Stands-on and declared order after the base is built, and the
|
||||
// build seat's holder follows the controller that defines its worker (EdgeWorkerOf, issue 206).
|
||||
if !in[e.From] || !in[e.To] || e.From == e.To || e.Kind == inventory.EdgePackages {
|
||||
continue
|
||||
}
|
||||
if e.Kind == inventory.EdgeBuiltBy && (isBaseOf(e.From, e.To, edges, in) || worker[e.From][e.To]) {
|
||||
// **Being built by the builder orders nothing** (novox/hq ADR 0360 §3): whichever builder runs builds it.
|
||||
if e.Kind == inventory.EdgeBuiltBy {
|
||||
continue
|
||||
}
|
||||
deps[e.From][e.To] = true
|
||||
@@ -105,29 +97,6 @@ func tiersOf(set []string, edges []inventory.Edge) [][]string {
|
||||
return tiers
|
||||
}
|
||||
|
||||
// isBaseOf says whether `to` stands on `base`, directly or through other bases in the set, along
|
||||
// the build edges alone.
|
||||
func isBaseOf(base, to string, edges []inventory.Edge, in map[string]bool) bool {
|
||||
seen := map[string]bool{}
|
||||
var walk func(string) bool
|
||||
walk = func(m string) bool {
|
||||
if m == base {
|
||||
return true
|
||||
}
|
||||
if seen[m] {
|
||||
return false
|
||||
}
|
||||
seen[m] = true
|
||||
for _, e := range edges {
|
||||
if e.From == m && in[e.To] && (e.Kind == inventory.EdgeStandsOn || e.Kind == inventory.EdgeDeclared) && walk(e.To) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
return walk(to)
|
||||
}
|
||||
|
||||
// reachableFrom is the moved modules plus everything that depends on them, through every layer:
|
||||
// what a merge rebuilds. Along the code and build edges only: a module *built by* the build machine
|
||||
// is not changed by a new build machine, so a built-by edge orders and gates a plan and never
|
||||
@@ -163,7 +132,7 @@ func reachableFrom(moved []string, edges []inventory.Edge) []string {
|
||||
}
|
||||
|
||||
// hasCycle says whether the tiers' last tier holds modules that still depend on each other. A packages
|
||||
// edge orders nothing (tiersOf), so a module and what packages its source share a tier by rule: that is
|
||||
// or built-by edge orders nothing (tiersOf), so a module and what packages its source share a tier by rule: that is
|
||||
// no cycle, and saying one was is a false report in every such plan's log.
|
||||
func hasCycle(tiers [][]string, edges []inventory.Edge) bool {
|
||||
if len(tiers) == 0 {
|
||||
@@ -174,7 +143,7 @@ func hasCycle(tiers [][]string, edges []inventory.Edge) bool {
|
||||
last[m] = true
|
||||
}
|
||||
for _, e := range edges {
|
||||
if e.Kind == inventory.EdgePackages {
|
||||
if e.Kind == inventory.EdgePackages || e.Kind == inventory.EdgeBuiltBy {
|
||||
continue
|
||||
}
|
||||
if last[e.From] && last[e.To] {
|
||||
@@ -220,12 +189,6 @@ func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool)
|
||||
return nil
|
||||
}
|
||||
inTier := map[string]bool{}
|
||||
all := map[string]bool{}
|
||||
for _, tier := range p.Tiers {
|
||||
for _, m := range tier {
|
||||
all[m] = true
|
||||
}
|
||||
}
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
inTier[m] = true
|
||||
}
|
||||
@@ -240,8 +203,8 @@ func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool)
|
||||
for _, e := range edges {
|
||||
// A delivery group's order inside a walk (novox/hq ADR 0276 decision 5) gates as built-by does: the
|
||||
// member before is running on its machines before the member after is asked.
|
||||
ordered := e.Kind == edgeGroupOrder ||
|
||||
e.Kind == inventory.EdgeBuiltBy && !isBaseOf(e.From, e.To, edges, all)
|
||||
// A built-by edge waits for nothing since novox/hq ADR 0360 §3.
|
||||
ordered := e.Kind == edgeGroupOrder
|
||||
if later[e.From] && inTier[e.To] && ordered && !seen[e.To] && rollsOut(e.To) {
|
||||
seen[e.To] = true
|
||||
out = append(out, e.To)
|
||||
@@ -251,6 +214,20 @@ func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool)
|
||||
return out
|
||||
}
|
||||
|
||||
// noMoveWhy is what a plan says of a module of its tier whose build was made from the source every machine
|
||||
// running it runs (novox/hq issue 280): nothing is sent for it.
|
||||
// noMovePrefix opens it, and is what noMove reads: one prefix for the writer and the reader.
|
||||
const (
|
||||
noMovePrefix = "no move"
|
||||
noMoveWhy = noMovePrefix + ": made from the source every machine running it runs"
|
||||
)
|
||||
|
||||
// noMove is whether the plan marked the module no move and sent it nothing. Read from what the plan says,
|
||||
// so a plan kept before the mark had a name of its own is read the same (novox/hq issue 648).
|
||||
func noMove(s *inventory.PlanModule) bool {
|
||||
return s != nil && s.FirstAt == nil && s.Gate == nil && strings.HasPrefix(s.Why, noMovePrefix)
|
||||
}
|
||||
|
||||
// applied says whether every machine running the module has reported since the module was built.
|
||||
// appliedEach is applied with a moment of its own for each machine: the reports that count are the ones
|
||||
// after that machine was sent the build (novox/hq issue 256).
|
||||
@@ -282,10 +259,11 @@ func applied(module string, builtAt time.Time, running []string, reports []inven
|
||||
return len(waiting) == 0, waiting
|
||||
}
|
||||
|
||||
// askTier asks the build machine for every module of the tier, and marks each asked. A module
|
||||
// askTier asks the build machine for the named modules of the plan's tier — all of it, or what of it is still
|
||||
// unasked (novox/hq issue 754) — and marks each asked. A module
|
||||
// the catalogue no longer holds, or whose ask could not be made, is a failure of the plan: a tier
|
||||
// half asked is a tier that will never complete.
|
||||
func askTier(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) error {
|
||||
func askTier(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan, names []string) error {
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -294,12 +272,62 @@ func askTier(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) e
|
||||
for _, e := range entries {
|
||||
byName[e.Manifest.Module] = e
|
||||
}
|
||||
for _, name := range p.Tiers[p.Tier] {
|
||||
for _, name := range names {
|
||||
askModule(ctx, p, name, byName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// unaskedOf are the modules of a plan's current tier it has still to ask: those it never asked, and one it holds
|
||||
// built that it never asked for (novox/hq issue 754) — a build taken in by a controller from before a walk took
|
||||
// only its own asks, somebody else's, which the walk would otherwise send as its own (ADR 0219). A tier partly asked is asked the rest, not
|
||||
// waited on for ever: a walk that waits for every build of a tier it never asked whole never moves again.
|
||||
func unaskedOf(p inventory.Plan) []string {
|
||||
if p.Tier >= len(p.Tiers) {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
s := p.Modules[m]
|
||||
switch {
|
||||
case s == nil || s.State == "":
|
||||
out = append(out, m)
|
||||
case s.State == "built" && s.Build == "" && s.AskedAt == nil:
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// askedInTier says the plan asked a module of its current tier itself: under an id, or at a time it kept.
|
||||
func askedInTier(p inventory.Plan) bool {
|
||||
if p.Tier >= len(p.Tiers) {
|
||||
return false
|
||||
}
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if s := p.Modules[m]; s != nil && (s.Build != "" || s.AskedAt != nil) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// takesOutcome says whether a build's outcome is the plan's for a module of its current tier (novox/hq ADR 0219,
|
||||
// issues 219 and 754). **A walk that waits for its delivery's word asks nothing, and so is answered by nothing**
|
||||
// (ADR 0239): a rebuild that lands in the wait is somebody else's build. **Nor is a module the plan never asked
|
||||
// answered**: taken, the plan held a module built that it never asked, and waited for the rest of the tier for
|
||||
// ever. Asked under an id, the plan takes that id's outcome and no other; asked before ids were kept, a build
|
||||
// asked at or after the plan's ask.
|
||||
func takesOutcome(p inventory.Plan, state *inventory.PlanModule, asked time.Time, id string) bool {
|
||||
if p.Waiting() || state == nil || (state.Build == "" && state.AskedAt == nil) {
|
||||
return false
|
||||
}
|
||||
if state.Build != "" {
|
||||
return state.Build == id
|
||||
}
|
||||
return !askedBefore(asked, state.AskedAt)
|
||||
}
|
||||
|
||||
// askABuild is how a plan asks for one build, not waited for, and learns the id it asked under. A
|
||||
// variable so a test of what a plan does around an ask needs no build machine.
|
||||
//
|
||||
@@ -372,15 +400,73 @@ var checkedBuildsOf = func(ctx context.Context, pulls []link.PullRef) ([]link.Ch
|
||||
return link.ReadCheckBuilds(ctx, js, buildSeatHeld(ctx), pulls, time.Now().Add(-builder.CheckBuildsKept))
|
||||
}
|
||||
|
||||
// mergedHeadsOf are the head commits the walk's merges merged, by pull request (novox/hq issue 703): a pull request
|
||||
// merged twice in one walk, or whose head the announcement did not say, has none, and takes no check's build.
|
||||
func mergedHeadsOf(p *inventory.Plan) map[link.PullRef]string {
|
||||
heads := map[link.PullRef]string{}
|
||||
if p.Delivery == nil {
|
||||
return heads
|
||||
}
|
||||
seen := map[link.PullRef]bool{}
|
||||
for _, m := range p.Delivery.Merges {
|
||||
pull, ok := pullOf(m)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if seen[pull] && heads[pull] != m.Head {
|
||||
heads[pull] = ""
|
||||
continue
|
||||
}
|
||||
seen[pull] = true
|
||||
heads[pull] = m.Head
|
||||
}
|
||||
return heads
|
||||
}
|
||||
|
||||
// walkVerdicts are the checks' builds a walk read, by the walk's plan: read once per walk (novox/hq issue 649), not
|
||||
// once per module it asks for. Kept a day; a walk lasts minutes.
|
||||
var walkVerdicts = struct {
|
||||
sync.Mutex
|
||||
read map[string]verdictsRead
|
||||
}{read: map[string]verdictsRead{}}
|
||||
|
||||
type verdictsRead struct {
|
||||
at time.Time
|
||||
builds []link.CheckedBuild
|
||||
}
|
||||
|
||||
// reuseWithVerdicts is a walk's leave with the builds its pull requests' verdicts name: the only builds the builder
|
||||
// may take (ADR 0350 §2). When they cannot be read the leave names none, and the walk builds, saying why.
|
||||
func reuseWithVerdicts(ctx context.Context, r *link.ReuseAsk) *link.ReuseAsk {
|
||||
builds, err := checkedBuildsOf(ctx, r.Pulls)
|
||||
if err != nil {
|
||||
fmt.Printf(" the checks' builds of %d pull request(s) could not be read, so the walk builds: %v\n", len(r.Pulls), err)
|
||||
return r
|
||||
// may take (ADR 0350 §2), and **only those of a check of the head each merge merged** (novox/hq issue 703): a build
|
||||
// a check made at an earlier head of the same pull request is never registered. Read once per walk. When they
|
||||
// cannot be read the leave names none, and the walk builds, saying why.
|
||||
func reuseWithVerdicts(ctx context.Context, walk string, r *link.ReuseAsk, heads map[link.PullRef]string) *link.ReuseAsk {
|
||||
walkVerdicts.Lock()
|
||||
read, known := walkVerdicts.read[walk]
|
||||
walkVerdicts.Unlock()
|
||||
if !known || walk == "" {
|
||||
builds, err := checkedBuildsOf(ctx, r.Pulls)
|
||||
if err != nil {
|
||||
fmt.Printf(" the checks' builds of %d pull request(s) could not be read, so the walk builds: %v\n", len(r.Pulls), err)
|
||||
return r
|
||||
}
|
||||
read = verdictsRead{at: time.Now(), builds: builds}
|
||||
if walk != "" {
|
||||
walkVerdicts.Lock()
|
||||
for id, w := range walkVerdicts.read {
|
||||
if time.Since(w.at) > 24*time.Hour {
|
||||
delete(walkVerdicts.read, id)
|
||||
}
|
||||
}
|
||||
walkVerdicts.read[walk] = read
|
||||
walkVerdicts.Unlock()
|
||||
}
|
||||
}
|
||||
r.Builds = nil
|
||||
for _, b := range read.builds {
|
||||
if link.SameHead(b.Head, heads[b.Pull]) {
|
||||
r.Builds = append(r.Builds, b)
|
||||
}
|
||||
}
|
||||
r.Builds = builds
|
||||
return r
|
||||
}
|
||||
|
||||
@@ -423,7 +509,8 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
|
||||
}
|
||||
// **The walk takes the check's build of the same key** (novox/hq ADR 0350 §2): its ask says so, with the pull
|
||||
// requests it answers, whose checks name the part of a differing key.
|
||||
id, err := askABuild(withReuse(ctx, reuseWithVerdicts(ctx, reuseAskOf(p))), source, e.Source.Path, ref)
|
||||
id, err := askABuild(withReuse(ctx, reuseWithVerdicts(ctx, p.ID, reuseAskOf(p), mergedHeadsOf(p))), source,
|
||||
e.Source.Path, ref)
|
||||
if err != nil {
|
||||
state.State = "failed"
|
||||
state.Why = err.Error()
|
||||
@@ -490,22 +577,15 @@ func planBuiltSaying(ctx context.Context, open *stores, module, commit, failed s
|
||||
if !inTier {
|
||||
continue
|
||||
}
|
||||
state := p.Modules[module]
|
||||
if state == nil {
|
||||
state = &inventory.PlanModule{}
|
||||
p.Modules[module] = state
|
||||
}
|
||||
// **The plan's own ask is its outcome, by id** (novox/hq ADR 0219); another build of the module
|
||||
// is, as before, when it was asked at or after the plan's ask (issue 219).
|
||||
// **A module asked under an id is answered by that id's outcome and no other** (novox/hq ADR
|
||||
// 0219): a replay, a rebuild beside the plan, or an older ask finishing late is somebody else's
|
||||
// build, made from other source, and settling the plan with it would send that. A plan from
|
||||
// before ids were kept is matched as it was: by when the build was asked (issue 219).
|
||||
if state.Build != "" {
|
||||
if state.Build != id {
|
||||
continue
|
||||
}
|
||||
} else if askedBefore(asked, state.AskedAt) {
|
||||
// before ids were kept is matched as it was: by when the build was asked (issue 219). A walk
|
||||
// waiting for its word, or a module it never asked, takes no outcome at all (issue 754).
|
||||
state := p.Modules[module]
|
||||
if !takesOutcome(*p, state, asked, id) {
|
||||
continue
|
||||
}
|
||||
if failed != "" && deletedAtSource(failed) {
|
||||
@@ -595,7 +675,7 @@ func advancePlans(ctx context.Context, open *stores) {
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
// What waits for a gate, released one machine at a time (ADR 0236).
|
||||
// What waits for a gate, released one machine first, then the rest at once (ADR 0236, ADR 0353).
|
||||
if _, err := releaseBacklog(ctx, open, ""); err != nil {
|
||||
fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err)
|
||||
}
|
||||
@@ -711,17 +791,11 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
return true, nil
|
||||
}
|
||||
tier := p.Tiers[p.Tier]
|
||||
// Not yet asked: ask.
|
||||
unasked := 0
|
||||
for _, m := range tier {
|
||||
if s := p.Modules[m]; s == nil || s.State == "" {
|
||||
unasked++
|
||||
}
|
||||
}
|
||||
if unasked == len(tier) {
|
||||
// Not yet asked: ask — the whole tier, or what of it is still unasked (novox/hq issue 754).
|
||||
if unasked := unaskedOf(*p); len(unasked) > 0 {
|
||||
// **One walk at a time** (novox/hq ADR 0276): a walk about to ask its first tier while another started
|
||||
// walk is open waits for that one to end, let go or not, and says so.
|
||||
if p.Tier == 0 {
|
||||
if p.Tier == 0 && !askedInTier(*p) {
|
||||
if behind, err := startedBeside(ctx, inv, p.ID, p.Created); err != nil {
|
||||
return false, err
|
||||
} else if behind != "" {
|
||||
@@ -731,7 +805,10 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
return changed, nil
|
||||
}
|
||||
}
|
||||
if err := askTier(ctx, inv, p); err != nil {
|
||||
if len(unasked) < len(tier) {
|
||||
fmt.Printf("%s: tier %d was asked in part; asking the rest: %s\n", p.ID, p.Tier, strings.Join(unasked, ", "))
|
||||
}
|
||||
if err := askTier(ctx, inv, p, unasked); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
@@ -788,13 +865,32 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
together := map[string]bool{}
|
||||
for _, m := range tier {
|
||||
state := p.Modules[m]
|
||||
if state == nil || state.SentAt != nil || state.State == planDeleted || !rollsOut(m) {
|
||||
if state == nil || state.SentAt != nil || state.State == planDeleted {
|
||||
continue
|
||||
}
|
||||
// **A recorded module goes where its move recreates nothing and moves no data** (novox/hq ADR 0377): judged
|
||||
// per machine against the build it was last sent, said per machine, and sent only there.
|
||||
var goes map[string]bool
|
||||
if !rollsOut(m) {
|
||||
out, err := judgeRecordedInWalk(ctx, inv, p, m, state)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if len(out) == 0 {
|
||||
continue
|
||||
}
|
||||
goes = map[string]bool{}
|
||||
for _, n := range out {
|
||||
goes[n] = true
|
||||
}
|
||||
}
|
||||
running, err := inv.Running(ctx, m)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if goes != nil {
|
||||
running = slices.DeleteFunc(running, func(n string) bool { return !goes[n] })
|
||||
}
|
||||
runningOf[m] = running
|
||||
// **A build whose source is unchanged is never a move** (novox/hq issue 280): made from the source
|
||||
// of the build every machine running it was sent, it is registered with that build's artifacts —
|
||||
@@ -805,7 +901,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
} else if same {
|
||||
now := time.Now().UTC()
|
||||
state.SentAt = &now
|
||||
state.Why = "no move: made from the source every machine running it runs"
|
||||
state.Why = noMoveWhy
|
||||
fmt.Printf("%s: %s built from the source every machine running it runs: no move, nothing sent\n", p.ID, m)
|
||||
continue
|
||||
}
|
||||
@@ -814,6 +910,12 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
// **Every machine at once** (novox/hq ADR 0360 §1, §2): a module that rolls out is sent to every machine
|
||||
// running it in one step, with no first machine and no gate, whatever module it is. A module a controller
|
||||
// before ADR 0360 already sent to its first machine finishes under its gate.
|
||||
if !gateFirstMachine && state.FirstAt == nil {
|
||||
policy.Together = true
|
||||
}
|
||||
var reports []inventory.Reported
|
||||
if !policy.Together {
|
||||
// Read for the choice of the first machine as well as for its report.
|
||||
@@ -995,6 +1097,25 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
if state.BuiltAt != nil {
|
||||
since = *state.BuiltAt
|
||||
}
|
||||
// **A build that is no move is waited on by the send that last carried it** (novox/hq issue 648).
|
||||
// Nothing was sent for it, so no report follows its marking or its build: a machine that does not
|
||||
// happen to report again held the walk for ever. Each machine runs it once it answered its last
|
||||
// send — the one whose builds made it no move — whenever that was.
|
||||
if noMove(state) {
|
||||
byNode := map[string]inventory.Reported{}
|
||||
for _, r := range reports {
|
||||
byNode[r.Node] = r
|
||||
}
|
||||
if ok, on := appliedEach(m, func(node string) time.Time {
|
||||
if s := byNode[node].Sent; s != nil {
|
||||
return *s
|
||||
}
|
||||
return time.Time{}
|
||||
}, running, reports); !ok {
|
||||
waiting = append(waiting, fmt.Sprintf("%s on %s", m, strings.Join(on, ", ")))
|
||||
}
|
||||
continue
|
||||
}
|
||||
// **Each machine from its own send** (novox/hq issue 256). With one machine first (ADR 0218)
|
||||
// a module is sent twice — the first machine, then the rest — and SentAt is the second.
|
||||
// Asked of every machine, the first machine's report, made between the two sends, read as
|
||||
@@ -1024,15 +1145,110 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
return changed, nil
|
||||
}
|
||||
}
|
||||
// **A walk ends on its machines' reports** (novox/hq ADR 0360 §4, §5): after its last tier, every machine it sent
|
||||
// has reported on that send, or is silent past ApplySilentAfter; never a health reading.
|
||||
end := ""
|
||||
if p.Tier == len(p.Tiers)-1 {
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
waiting, failed, silent := walkReports(*p, reports, time.Now().UTC(), provLookup(ctx))
|
||||
if len(waiting) > 0 {
|
||||
note := "sent; waiting for " + strings.Join(waiting, ", ") + " to report on their send"
|
||||
changed := p.State != inventory.PlanRolling || p.Note != note
|
||||
p.State = inventory.PlanRolling
|
||||
p.Note = note
|
||||
return changed, nil
|
||||
}
|
||||
var said []string
|
||||
if len(failed) > 0 {
|
||||
said = append(said, "failed or refused its send (apply-failing says it): "+strings.Join(failed, ", "))
|
||||
}
|
||||
if len(silent) > 0 {
|
||||
said = append(said, fmt.Sprintf("said nothing on its send within %s: %s", inventory.ApplySilentAfter,
|
||||
strings.Join(silent, ", ")))
|
||||
}
|
||||
said = append(said, recordedSaid(*p)...)
|
||||
end = strings.Join(said, "; ")
|
||||
// **The new controller's user list follows its walk** (ADR 0236 §34, read without the gate since ADR 0360):
|
||||
// once a walk that moved the controller ends, the bus's machine is sent the list it composes, if behind.
|
||||
if s := p.Modules[catalogue.ControllerSeatName]; s != nil && s.SentAt != nil && s.Gate == nil && !noMove(s) {
|
||||
carryUserList(ctx, open, p)
|
||||
}
|
||||
}
|
||||
p.Tier++
|
||||
p.State = inventory.PlanBuilding
|
||||
p.Note = ""
|
||||
p.Note = end
|
||||
if p.Tier < len(p.Tiers) {
|
||||
fmt.Printf("%s: tier %d done; asking tier %d: %s\n", p.ID, p.Tier-1, p.Tier, strings.Join(p.Tiers[p.Tier], ", "))
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// walkReports is, for every machine a walk sent its rest, whether it reported on that send (novox/hq ADR 0360 §4):
|
||||
// those still waiting, those that reported a failed or refused apply, and those silent past ApplySilentAfter,
|
||||
// each sorted. Pure.
|
||||
func walkReports(p inventory.Plan, reports []inventory.Reported, now time.Time, prov inventory.ProvLookup) (waiting, failed, silent []string) {
|
||||
byNode := map[string]inventory.Reported{}
|
||||
for _, r := range reports {
|
||||
byNode[r.Node] = r
|
||||
}
|
||||
// The modules of this delivery sent to each machine: a machine is judged on those, not on every module its
|
||||
// declaration hands to the provisioner (ADR 0362 §1-2, issue 840).
|
||||
carried := map[string][]string{}
|
||||
for m, s := range p.Modules {
|
||||
if s == nil {
|
||||
continue
|
||||
}
|
||||
for node := range s.Rest {
|
||||
carried[node] = append(carried[node], m)
|
||||
}
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, s := range p.Modules {
|
||||
if s == nil {
|
||||
continue
|
||||
}
|
||||
for node, sent := range s.Rest {
|
||||
sent = sent.Only(carried[node]...)
|
||||
if seen[node] {
|
||||
continue
|
||||
}
|
||||
r := byNode[node]
|
||||
at := sent.At
|
||||
if at.IsZero() && s.SentAt != nil {
|
||||
at = *s.SentAt
|
||||
}
|
||||
// A module handed to the provisioner is applied only once its client said so too (ADR 0362 §1-5).
|
||||
outcome, reported := provisionedOutcome(node, sent, r, r.At != nil && sent.ReportsOn(r), prov)
|
||||
switch {
|
||||
case reported:
|
||||
seen[node] = true
|
||||
switch {
|
||||
case outcome == inventory.OutcomeFailed || outcome == inventory.OutcomeRefused:
|
||||
failed = append(failed, node)
|
||||
case strings.HasPrefix(outcome, inventory.OutcomeProvisioningFailed):
|
||||
failed = append(failed, node+" ("+outcome+")")
|
||||
}
|
||||
case !at.IsZero() && now.Sub(at) > inventory.ApplySilentAfter:
|
||||
seen[node] = true
|
||||
silent = append(silent, node)
|
||||
default:
|
||||
if !slices.Contains(waiting, node) {
|
||||
waiting = append(waiting, node)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// A machine one module's send was reported on and another's not yet is the same send: reported.
|
||||
waiting = slices.DeleteFunc(waiting, func(n string) bool { return seen[n] })
|
||||
sort.Strings(waiting)
|
||||
sort.Strings(failed)
|
||||
sort.Strings(silent)
|
||||
return waiting, failed, silent
|
||||
}
|
||||
|
||||
// restOf is, for one module, every machine of its rest that the send reached and what it carried there.
|
||||
func restOf(to, sent []string, what map[string]inventory.SentDeclaration) map[string]inventory.SentDeclaration {
|
||||
out := map[string]inventory.SentDeclaration{}
|
||||
@@ -1056,10 +1272,36 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
||||
runningOf map[string][]string) error {
|
||||
inv := open.inventory
|
||||
before, beforeKnown, _ := inv.SentBuilds(ctx, node)
|
||||
// **A machine already sent what this send is for is not sent again** (novox/hq ADR 0353 §6): sent by this plan
|
||||
// before a controller was replaced and before it was kept, or by another walk. Judged from that send, and from
|
||||
// the readings of a gate already judging it there.
|
||||
// Only a send the machine reported it applied is one it has; one it never reported on is sent, as before
|
||||
// (issue 280: what it ran before is then that build, and a failure leaves it as it was).
|
||||
if beforeKnown && slices.IndexFunc(modules, func(m string) bool { return !sameCommit(before[m], p.Modules[m].Commit) }) < 0 &&
|
||||
appliedLastSend(ctx, inv, node) {
|
||||
if others, err := gatedMoves(ctx, open, node, nil); err == nil && len(others) == 0 {
|
||||
adoptSend(ctx, open, p, node, modules, before, runningOf)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
// **What the machine ran before is kept before the send is made** (novox/hq ADR 0353 §6): a controller replaced
|
||||
// between the send and keeping it reads it here, and a failure still puts the machine back.
|
||||
kept := false
|
||||
for _, m := range modules {
|
||||
s := p.Modules[m]
|
||||
if beforeKnown && (s.Previous == "" || !sameCommit(before[m], s.Commit)) && s.Previous != before[m] {
|
||||
s.Previous, kept = before[m], true
|
||||
}
|
||||
}
|
||||
if kept {
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
return fmt.Errorf("what %s ran before is not kept, so nothing is sent: %w", node, err)
|
||||
}
|
||||
}
|
||||
owns := make([]inventory.CarriedMove, 0, len(modules))
|
||||
for _, m := range modules {
|
||||
s := p.Modules[m]
|
||||
owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: before[m], To: s.Commit, Build: s.Build})
|
||||
owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: s.Previous, To: s.Commit, Build: s.Build})
|
||||
}
|
||||
// A gated send (ADR 0236): everything waiting on the machine goes with the tier, and the gate judges
|
||||
// all of it there.
|
||||
@@ -1074,10 +1316,7 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
||||
sentWhat := sentNow(ctx, inv, sent)
|
||||
for _, m := range modules {
|
||||
s := p.Modules[m]
|
||||
// What the first machine ran before: what a failed gate puts back (ADR 0236).
|
||||
if beforeKnown {
|
||||
s.Previous = before[m]
|
||||
}
|
||||
// What the first machine ran before (ADR 0236), kept above before the send.
|
||||
s.First, s.FirstAt = sent, &now
|
||||
s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning(sent, runningOf[m]),
|
||||
From: s.Previous, To: s.Commit, Since: &now, Sent: sentWhat}
|
||||
@@ -1106,6 +1345,113 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
||||
return nil
|
||||
}
|
||||
|
||||
// adoptSend records a first send the machine already had (novox/hq ADR 0353 §6, §7): nothing is sent; the gate counts
|
||||
// from the machine's last send, and takes over the readings of an open gate of another walk judging the same build
|
||||
// there — whose carried move also says what the machine ran before, for a rollback.
|
||||
func adoptSend(ctx context.Context, open *stores, p *inventory.Plan, node string, modules []string,
|
||||
before map[string]string, runningOf map[string][]string) {
|
||||
inv := open.inventory
|
||||
since := time.Now().UTC()
|
||||
last, found, err := inv.SentTo(ctx, node)
|
||||
if err == nil && found && !last.At.IsZero() {
|
||||
since = last.At
|
||||
}
|
||||
lead := modules[0]
|
||||
other, id, from := judgingThere(ctx, inv, p.ID, lead, node, p.Modules[lead].Commit, p.Modules[lead].Build)
|
||||
for _, m := range modules {
|
||||
s := p.Modules[m]
|
||||
// What it ran before: kept by this walk before its send was made (§6); else what the other walk's send
|
||||
// moved it from; else the build itself, so a failure leaves it as it was (issue 280).
|
||||
switch {
|
||||
case s.Previous != "":
|
||||
case m == lead && from != "":
|
||||
s.Previous = from
|
||||
default:
|
||||
s.Previous = before[m]
|
||||
}
|
||||
at := since
|
||||
s.First, s.FirstAt = []string{node}, &at
|
||||
s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning([]string{node}, runningOf[m]),
|
||||
From: s.Previous, To: s.Commit, Since: &at}
|
||||
if found {
|
||||
s.Gate.Sent = map[string]inventory.SentDeclaration{node: last}
|
||||
}
|
||||
s.GatedBy = ""
|
||||
if m == lead {
|
||||
s.Gate.TakeReadings(other, modules, id)
|
||||
} else {
|
||||
s.GatedBy = lead
|
||||
}
|
||||
}
|
||||
p.State = inventory.PlanRolling
|
||||
p.Note = fmt.Sprintf("tier %d built; %s already runs %s's build: nothing sent again, judged from that send",
|
||||
p.Tier, node, strings.Join(modules, ", "))
|
||||
if id != "" {
|
||||
p.Note += ", going on from the readings of " + id
|
||||
}
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
}
|
||||
|
||||
// appliedLastSend says a machine reported it applied the declaration it was sent last.
|
||||
func appliedLastSend(ctx context.Context, inv *inventory.Inventory, node string) bool {
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
for _, r := range reports {
|
||||
if r.Node == node {
|
||||
return r.Current && r.At != nil && r.Outcome == inventory.OutcomeApplied
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// judgingThere is an open gate of another walk judging a module's build on a machine (novox/hq ADR 0353 §7), the
|
||||
// walk's id, and what the machine ran before; nil when none does.
|
||||
func judgingThere(ctx context.Context, inv *inventory.Inventory, self, module, node, commit, build string) (*inventory.PlanGate, string, string) {
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return nil, "", ""
|
||||
}
|
||||
// The same build, by its id where both name one, else by its commit.
|
||||
same := func(b, c string) bool {
|
||||
if b != "" && build != "" {
|
||||
return b == build
|
||||
}
|
||||
return sameCommit(c, commit)
|
||||
}
|
||||
usable := func(g *inventory.PlanGate) bool {
|
||||
return g != nil && !g.Pending && g.Since != nil && (g.Verdict == "" || g.Verdict == inventory.GatePassed) &&
|
||||
slices.Contains(g.Machines, node)
|
||||
}
|
||||
for _, q := range plans {
|
||||
if q.ID == self {
|
||||
continue
|
||||
}
|
||||
if r := q.Release; r != nil {
|
||||
gates := []*inventory.PlanGate{r.Gate}
|
||||
for _, g := range r.Gates {
|
||||
gates = append(gates, g)
|
||||
}
|
||||
for _, g := range gates {
|
||||
if !usable(g) {
|
||||
continue
|
||||
}
|
||||
for _, c := range g.Carried {
|
||||
if c.Module == module && c.Node == node && same(c.Build, c.To) {
|
||||
return g, q.ID, c.From
|
||||
}
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
if s := q.Modules[module]; s != nil && usable(s.Gate) && same(s.Build, s.Commit) {
|
||||
return s.Gate, q.ID, s.Previous
|
||||
}
|
||||
}
|
||||
return nil, "", ""
|
||||
}
|
||||
|
||||
// passedWith keeps, on a module sent with others, the verdict of the gate that judged the send: the gate
|
||||
// on the first of them passed, and its pass was kept for every build it carried (passCarried).
|
||||
func passedWith(module string, s *inventory.PlanModule, lead string, g *inventory.PlanGate) {
|
||||
@@ -1594,9 +1940,19 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
// A release plan's walk (ADR 0236): machines done, the one judged, those to come.
|
||||
fmt.Printf(" machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "),
|
||||
orNone(strings.Join(r.Done, ", ")), orNone(strings.Join(r.Skipped, ", ")))
|
||||
if r.Gate != nil {
|
||||
fmt.Printf(" %s\n", gateLine(r.Gate))
|
||||
for _, c := range r.Gate.Carried {
|
||||
if len(r.Failed)+len(r.Silent) > 0 {
|
||||
fmt.Printf(" failed: %s; silent: %s\n", orNone(strings.Join(r.Failed, ", ")), orNone(strings.Join(r.Silent, ", ")))
|
||||
}
|
||||
gates := []*inventory.PlanGate{r.Gate}
|
||||
for _, n := range r.Order {
|
||||
gates = append(gates, r.Gates[n])
|
||||
}
|
||||
for _, g := range gates {
|
||||
if g == nil {
|
||||
continue
|
||||
}
|
||||
fmt.Printf(" %s\n", gateLine(g))
|
||||
for _, c := range g.Carried {
|
||||
fmt.Printf(" %-22s %s → %s\n", c.Module, short(c.From), short(c.To))
|
||||
if c.Recreates != "" {
|
||||
fmt.Printf(" %-22s %s\n", "", c.Recreates)
|
||||
@@ -1636,6 +1992,12 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
case s != nil && s.Gate != nil:
|
||||
fmt.Printf(" %-22s %s\n", "", gateLine(s.Gate))
|
||||
}
|
||||
// Which way a recorded module went on each machine (novox/hq ADR 0377 §6).
|
||||
if s != nil {
|
||||
for _, n := range sortedKeysOf(s.Recorded) {
|
||||
fmt.Printf(" %-22s on %s %s\n", "", n, s.Recorded[n])
|
||||
}
|
||||
}
|
||||
// What the send to its first machine did to its containers (ADR 0245).
|
||||
if s != nil && s.Gate != nil {
|
||||
for _, c := range s.Gate.Carried {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -49,8 +50,10 @@ func TestAMergeIsPlannedInTiersAlongTheThreeKindsOfDependency(t *testing.T) {
|
||||
if pos["mesh-tools"] != 0 || pos["builder"] != 1 {
|
||||
t.Fatalf("the runtime then the builder: %v", tiers)
|
||||
}
|
||||
if !(pos["shop"] > pos["builder"] && pos["postgres"] > pos["builder"]) {
|
||||
t.Fatalf("what the builder builds comes after the builder: %v", tiers)
|
||||
// Built-by orders nothing (novox/hq ADR 0360 §3): what the builder builds is beside it, after the runtime it
|
||||
// stands on.
|
||||
if pos["shop"] != 1 || pos["postgres"] != 1 {
|
||||
t.Fatalf("what the builder builds waits for the builder: %v", tiers)
|
||||
}
|
||||
if pos["shop-plugin"] <= pos["shop"] {
|
||||
t.Fatalf("a plugin after what it is declared on: %v", tiers)
|
||||
@@ -67,24 +70,23 @@ func TestAMergeIsPlannedInTiersAlongTheThreeKindsOfDependency(t *testing.T) {
|
||||
// A change to a package all three build from moves all three. The builder holds
|
||||
// the build seat, whose worker the controller defines, so it follows the controller (worker-of,
|
||||
// novox/hq issue 206), and the controller's built-by edge to it yields: the controller is built by the
|
||||
// build machine that is running. The proxy is built by the new builder: the controller, the builder,
|
||||
// the proxy — the live plan of every controller merge. (This read "the builder, then the controller
|
||||
// and the proxy together" before issue 206, and the fixture had no worker-of edge.)
|
||||
// build machine that is running. The proxy is built by whichever builder runs, so it goes beside the
|
||||
// controller (novox/hq ADR 0360 §3; it read "the controller, the builder, the proxy" before).
|
||||
small := reachableFrom([]string{"mesh-controller", "builder", "route-proxy"}, edges)
|
||||
smallTiers := tiersOf(small, edges)
|
||||
if got := tiered(smallTiers); got != "mesh-controller | builder | route-proxy" {
|
||||
t.Fatalf("the controller, then the builder, then the proxy: %v", smallTiers)
|
||||
if got := tiered(smallTiers); got != "mesh-controller,route-proxy | builder" {
|
||||
t.Fatalf("the controller and the proxy, then the builder: %v", smallTiers)
|
||||
}
|
||||
// The builder alone moved: the builder, and nothing it builds.
|
||||
if only := reachableFrom([]string{"builder"}, edges); len(only) != 1 {
|
||||
t.Fatalf("a build machine change rebuilds the build machine alone: %v", only)
|
||||
}
|
||||
|
||||
// Only a runtime dependency gates on deployment, and only when the module rolls out.
|
||||
// Nothing waits on deployment for a built-by edge (novox/hq ADR 0360 §3): the builder gates no tier after it.
|
||||
p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-tools", Commit: "abc"}, []string{"mesh-tools"}, edges)
|
||||
p.Tier = pos["builder"]
|
||||
rollsOut := func(m string) bool { return m == "builder" }
|
||||
if g := gates(p, edges, rollsOut); len(g) != 1 || g[0] != "builder" {
|
||||
if g := gates(p, edges, rollsOut); len(g) != 0 {
|
||||
t.Fatalf("the builder gates the tier after it: %v", g)
|
||||
}
|
||||
p.Tier = 0
|
||||
@@ -253,10 +255,10 @@ func TestAChangeToTheBuildAgentRebuildsTheBuildAgentAlone(t *testing.T) {
|
||||
t.Fatalf("%v planned %v: the build agent alone", paths, p.Tiers)
|
||||
}
|
||||
}
|
||||
// With what it builds moved beside it, the agent comes first and they follow: an order, not a widening.
|
||||
// With what it builds moved beside it, both go in one tier (novox/hq ADR 0360 §3): built-by orders nothing.
|
||||
p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Commit: "abc"},
|
||||
[]string{"build-agent", "redis"}, edges)
|
||||
if len(p.Tiers) != 2 || p.Tiers[0][0] != "build-agent" || p.Tiers[1][0] != "redis" || len(p.Modules) != 2 {
|
||||
t.Fatalf("the agent, then what moved beside it: %v", p.Tiers)
|
||||
if len(p.Tiers) != 1 || !slices.Equal(p.Tiers[0], []string{"build-agent", "redis"}) || len(p.Modules) != 2 {
|
||||
t.Fatalf("the agent and what moved beside it, in one tier: %v", p.Tiers)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// handingAnchor makes every send to anchor hand the provisioner the given modules (module -> templates digest), and
|
||||
// the clients' words the given reports (key `node/module`), as the stream would give them (issue 840, ADR 0362).
|
||||
func handingAnchor(t *testing.T, b *backlogMesh, handed map[string]string) map[string]inventory.ProvReport {
|
||||
t.Helper()
|
||||
says := map[string]inventory.ProvReport{}
|
||||
wasSays := provAppliedOn
|
||||
provAppliedOn = func(_ context.Context, node, module string) (inventory.ProvReport, bool, error) {
|
||||
r, ok := says[node+"/"+module]
|
||||
return r, ok, nil
|
||||
}
|
||||
inner := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
got, err := inner(ctx, open, names)
|
||||
if err != nil {
|
||||
return got, err
|
||||
}
|
||||
for _, n := range names {
|
||||
if n != "anchor" {
|
||||
continue
|
||||
}
|
||||
inv := open.inventory
|
||||
s, _, err := inv.SentTo(ctx, n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
builds, _, err := inv.SentBuilds(ctx, n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
plans := map[string]inventory.Provisioning{}
|
||||
for m, d := range handed {
|
||||
plans[m] = inventory.Provisioning{Templates: inventory.TemplatesArchive{Digest: d}}
|
||||
}
|
||||
raw, err := json.Marshal(plans)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
id := nodeID(t, open, n)
|
||||
if err := inv.RecordSentWith(ctx, id, s.Digest, builds, 0, 0, inventory.Send{Node: id, Provisioning: raw}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return got, nil
|
||||
}
|
||||
t.Cleanup(func() { sendRollout, provAppliedOn = inner, wasSays })
|
||||
return says
|
||||
}
|
||||
|
||||
func stepRelease(t *testing.T, b *backlogMesh, n int) inventory.Plan {
|
||||
t.Helper()
|
||||
for i := 0; i < n; i++ {
|
||||
advancePlans(t.Context(), b.open)
|
||||
}
|
||||
return b.release(t)
|
||||
}
|
||||
|
||||
// Issue 840 (novox/hq ADR 0362 §1-5): the release's rest gate, which every delivery passes since ADR 0360, counts a
|
||||
// machine's send of a provisioned module applied only once the engine and the machine's provisioner client both said
|
||||
// so; a failed client is a failed apply named with its reason; none past ApplySilentAfter is silent, never done.
|
||||
func TestTheRestGateWaitsForTheProvisioner(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
applied := inventory.ProvReport{Status: inventory.ProvApplied, Templates: "t1", At: time.Now()}
|
||||
|
||||
t.Run("a engine applied, the client has not said: no verdict, the release stays open", func(t *testing.T) {
|
||||
b := aBacklog(t)
|
||||
handingAnchor(t, b, map[string]string{"app": "t1"})
|
||||
p := stepRelease(t, b, 8)
|
||||
g := p.Release.Gates["anchor"]
|
||||
if !p.Open() || g == nil || g.Verdict != "" || g.Silent || slices.Contains(p.Release.Done, "anchor") {
|
||||
t.Fatalf("anchor is judged before its provisioner reported: %s %s %+v", p.State, p.Note, g)
|
||||
}
|
||||
if !slices.Contains(p.Release.Done, "laptop") {
|
||||
t.Fatalf("laptop hands nothing and passes on the engine's report: %+v", p.Release)
|
||||
}
|
||||
})
|
||||
t.Run("b the client failed: a failed apply named with the reason", func(t *testing.T) {
|
||||
b := aBacklog(t)
|
||||
says := handingAnchor(t, b, map[string]string{"app": "t1"})
|
||||
says["anchor/app"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t1", Reason: "no value", At: time.Now()}
|
||||
p := stepRelease(t, b, 8)
|
||||
g := p.Release.Gates["anchor"]
|
||||
if g == nil || g.Verdict != inventory.GateFailed || !slices.Contains(p.Release.Failed, "anchor") ||
|
||||
!strings.Contains(p.Note, "provisioning failed (app: no value)") {
|
||||
t.Fatalf("the failure is not named: %s %q %+v %+v", p.State, p.Note, g, p.Release)
|
||||
}
|
||||
})
|
||||
t.Run("c no client report past ApplySilentAfter: silent, never done", func(t *testing.T) {
|
||||
b := aBacklog(t)
|
||||
handingAnchor(t, b, map[string]string{"app": "t1"})
|
||||
p := stepRelease(t, b, 8)
|
||||
g := p.Release.Gates["anchor"]
|
||||
if g == nil {
|
||||
t.Fatalf("no gate: %+v", p.Release)
|
||||
}
|
||||
since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute)
|
||||
g.Since = &since
|
||||
if err := b.open.inventory.SavePlan(t.Context(), &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p = stepRelease(t, b, 1)
|
||||
if !slices.Equal(p.Release.Silent, []string{"anchor"}) || slices.Contains(p.Release.Done, "anchor") {
|
||||
t.Fatalf("anchor is not silent: %s %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
})
|
||||
t.Run("d both applied: the gate passes", func(t *testing.T) {
|
||||
b := aBacklog(t)
|
||||
says := handingAnchor(t, b, map[string]string{"app": "t1"})
|
||||
says["anchor/app"] = applied
|
||||
p := stepRelease(t, b, 8)
|
||||
if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") || len(p.Release.Failed) > 0 {
|
||||
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
})
|
||||
t.Run("e a send handing nothing passes on the engine's report", func(t *testing.T) {
|
||||
b := aBacklog(t)
|
||||
p := stepRelease(t, b, 8)
|
||||
if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") {
|
||||
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
})
|
||||
t.Run("f a module the delivery does not carry does not touch its end", func(t *testing.T) {
|
||||
// anchor's declaration also hands `same`, rebuilt with the very bytes and so not carried: its failed report
|
||||
// is none of this delivery's.
|
||||
b := aBacklog(t)
|
||||
says := handingAnchor(t, b, map[string]string{"app": "t1", "same": "t9"})
|
||||
says["anchor/app"] = applied
|
||||
says["anchor/same"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t9", Reason: "stuck", At: time.Now()}
|
||||
p := stepRelease(t, b, 8)
|
||||
if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") || len(p.Release.Failed) > 0 {
|
||||
t.Fatalf("another module's failure ended this delivery: %s %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Issue 840: a walk's end waits on the modules it sent to a machine, not on every module that machine's declaration
|
||||
// hands to the provisioner.
|
||||
func TestAWalkEndWaitsOnlyOnTheModulesItSent(t *testing.T) {
|
||||
now := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)
|
||||
at := now.Add(-10 * time.Second)
|
||||
handed := map[string]string{"app": "t1", "other": "t2"}
|
||||
p := inventory.Plan{Modules: map[string]*inventory.PlanModule{"app": {Rest: map[string]inventory.SentDeclaration{
|
||||
"ace": {Digest: "d-ace", At: now.Add(-time.Minute), Provisioned: handed}}}}}
|
||||
reports := []inventory.Reported{{Node: "ace", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-ace"}}
|
||||
says := map[string]inventory.ProvReport{"ace/app": {Status: inventory.ProvApplied, Templates: "t1", At: at}}
|
||||
prov := func(node, module string) (inventory.ProvReport, bool) { r, ok := says[node+"/"+module]; return r, ok }
|
||||
// other has said nothing: not this walk's to wait for.
|
||||
if waiting, failed, silent := walkReports(p, reports, now, prov); len(waiting)+len(failed)+len(silent) != 0 {
|
||||
t.Fatalf("missing: waiting %v, failed %v, silent %v", waiting, failed, silent)
|
||||
}
|
||||
says["ace/other"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t2", Reason: "stuck", At: at}
|
||||
if waiting, failed, silent := walkReports(p, reports, now, prov); len(waiting)+len(failed)+len(silent) != 0 {
|
||||
t.Fatalf("failed: waiting %v, failed %v, silent %v", waiting, failed, silent)
|
||||
}
|
||||
// once the walk carries it too, it does.
|
||||
p.Modules["other"] = &inventory.PlanModule{Rest: p.Modules["app"].Rest}
|
||||
if _, failed, _ := walkReports(p, reports, now, prov); len(failed) != 1 || !strings.Contains(failed[0], "other: stuck") {
|
||||
t.Fatalf("a carried module's failure is not named: %v", failed)
|
||||
}
|
||||
}
|
||||
@@ -111,8 +111,13 @@ func aBacklog(t *testing.T) *backlogMesh {
|
||||
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// A broken machine reports its apply failed (novox/hq ADR 0360 §7: the report is the verdict).
|
||||
outcome := inventory.OutcomeApplied
|
||||
if b.broken[node] {
|
||||
outcome = inventory.OutcomeFailed
|
||||
}
|
||||
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
|
||||
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
|
||||
Outcome: outcome, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
@@ -164,6 +169,7 @@ func (b *backlogMesh) release(t *testing.T) inventory.Plan {
|
||||
// The backlog goes out one machine at a time: the first judged before the second is sent, each build's
|
||||
// pass kept; a rebuild with the same bytes is no move at all; and a send outside a gate is refused.
|
||||
func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
@@ -219,6 +225,7 @@ func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) {
|
||||
// A release that fails on its first machine puts back what it carried there, goes no further, and the
|
||||
// next one waits for a person, said as a condition; a person releases it with why.
|
||||
func TestAFailedReleaseIsPutBackAndTheNextWaitsForAPerson(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
|
||||
@@ -31,6 +31,7 @@ import (
|
||||
// nothing is put back, the walk goes on to the workstation, and openrazer's condition says, in one sentence
|
||||
// for the operator, that a new login is needed on the laptop.
|
||||
func TestReplay318(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
// at its gate with the pass's own words, and put it back. Written with only what the controller had before
|
||||
// its fix, so it is laid over the commit before.
|
||||
func TestReplay335(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
t.Run("the first machine's later reports", testAPassedGateIsNotJudgedAgainFromTheFirstMachinesLaterReports)
|
||||
t.Run("a walk stopped after the pass", testAWalkStoppedAfterItsGatePassedKeepsThePass)
|
||||
}
|
||||
@@ -77,6 +78,7 @@ func testAWalkStoppedAfterItsGatePassedKeepsThePass(t *testing.T) {
|
||||
// the first send is older than the wait for a first machine's report. Read before the pass is taken over, the
|
||||
// carried module's step said the first machine never applied it, and the passed build was put back.
|
||||
func TestACarriedModuleTakesItsLeadsPassBeforeItsStepIsRead(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
tm := aTierMesh(t, "m01", "m02")
|
||||
ctx := t.Context()
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
// had before its fix — the stores, register, assign, a plan saved and advanced, the watchdogs' gathering and
|
||||
// seeing — so it is laid over the commit before.
|
||||
func TestReplay336(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
|
||||
@@ -139,6 +139,7 @@ func TestReplay273AConsumerBesideItsStoreStaysBoundToIt(t *testing.T) {
|
||||
// one, kept below. Heard as bytes, so an older controller reads them as it reads any report — this file is
|
||||
// written only with what the controller had before the judging, for the prover to lay over that commit.
|
||||
func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
|
||||
@@ -0,0 +1,382 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0353, kept behind gateFirstMachine for walks a controller before ADR 0360 started: after its first machine passes, a build goes to every other machine at once, each judged by
|
||||
// its own gate and a failing one put back by itself; and a gate's readings outlive the controller that took them —
|
||||
// a machine already sent what a send is for is not sent again, and its judging goes on from what was read.
|
||||
//
|
||||
// Seen 2026-10-14: release-1791930194184991192 sent ace, g14, shanks and the control node one after another in
|
||||
// eleven minutes; the new controller its last send started sent the control node again the controller build it
|
||||
// had just applied, and judged it from its first reading.
|
||||
|
||||
// aBacklogOnThree is the backlog fixture with a third machine, desk, running app at c1 like the others.
|
||||
func aBacklogOnThree(t *testing.T) *backlogMesh {
|
||||
t.Helper()
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
if _, err := inv.AddNode(ctx, "desk"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "desk", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, b.open, "desk"), "d-desk", map[string]string{"app": "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wasHeard := releaseHeard
|
||||
releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
|
||||
return map[string]bool{"anchor": true, "laptop": true, "desk": true}, nil
|
||||
}
|
||||
t.Cleanup(func() { releaseHeard = wasHeard })
|
||||
return b
|
||||
}
|
||||
|
||||
// events records every send and every judging, in order.
|
||||
func events(t *testing.T) *[]string {
|
||||
t.Helper()
|
||||
var said []string
|
||||
wasSend, wasGather := sendRollout, gatherGateFacts
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
said = append(said, "send "+strings.Join(names, ","))
|
||||
return wasSend(ctx, open, names)
|
||||
}
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
said = append(said, "judge")
|
||||
return wasGather(ctx, open, component)
|
||||
}
|
||||
t.Cleanup(func() { sendRollout, gatherGateFacts = wasSend, wasGather })
|
||||
return &said
|
||||
}
|
||||
|
||||
// Once the first machine passed, the rest are both sent before either is judged (ADR 0353 §1); before, the second
|
||||
// was judged to its verdict before the third was sent.
|
||||
func TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses(t *testing.T) {
|
||||
withTheGate(t)
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklogOnThree(t)
|
||||
ctx := t.Context()
|
||||
said := events(t)
|
||||
gateEvery = 0
|
||||
for i := 0; i < 12; i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
p := b.release(t)
|
||||
if p.State != inventory.PlanDone || len(p.Release.Done) != 3 {
|
||||
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
desk, laptop := slices.Index(*said, "send desk"), slices.Index(*said, "send laptop")
|
||||
if desk < 0 || laptop < 0 || slices.Index(*said, "send anchor") > min(desk, laptop) {
|
||||
t.Fatalf("sends and judgings: %v", *said)
|
||||
}
|
||||
if slices.Contains((*said)[min(desk, laptop):max(desk, laptop)], "judge") {
|
||||
t.Fatalf("the rest were sent one at a time, each judged before the next: %v", *said)
|
||||
}
|
||||
if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine of the rest whose gate fails is put back at once, and the other machine of the step goes on to its own
|
||||
// verdict (ADR 0353 §3); before, the walk stopped at the failure and the machine after it was never sent.
|
||||
func TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn(t *testing.T) {
|
||||
withTheGate(t)
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklogOnThree(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
gateEvery = 0
|
||||
b.broken["desk"] = true // desk fails what it is sent in the rest's step
|
||||
advancePlans(ctx, b.open)
|
||||
for i := 0; i < 12 && b.release(t).Open(); i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
p := b.release(t)
|
||||
if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) {
|
||||
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
if !slices.ContainsFunc(b.sent, func(s []string) bool { return slices.Equal(s, []string{"laptop"}) }) {
|
||||
t.Fatalf("laptop was never sent: %v", b.sent)
|
||||
}
|
||||
if !slices.Contains(p.Release.Done, "laptop") {
|
||||
t.Fatalf("laptop has no verdict of its own: %+v", p.Release)
|
||||
}
|
||||
// late waited on laptop alone and was healthy there: its pass is kept, whatever desk did.
|
||||
if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
|
||||
}
|
||||
// app failed on desk: marked, and put back wherever it was sent (ADR 0236).
|
||||
if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed {
|
||||
t.Fatal("app's build is not marked failed")
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c1" {
|
||||
t.Fatalf("desk was not put back: %v", sent)
|
||||
}
|
||||
}
|
||||
|
||||
// A walk whose first machine already runs its build under another walk's gate sends nothing and goes on from that
|
||||
// gate's readings (ADR 0353 §7); before, it sent the machine again and judged from its first reading.
|
||||
func TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere(t *testing.T) {
|
||||
withTheGate(t)
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
advancePlans(ctx, b.open) // the release sends anchor app c2, and reads it once
|
||||
release := b.release(t)
|
||||
judging := release.Release.Gate
|
||||
if judging == nil || judging.Pending || judging.Passes == 0 {
|
||||
t.Fatalf("the release is not judging anchor: %+v", judging)
|
||||
}
|
||||
built := time.Now().UTC()
|
||||
plan := inventory.Plan{ID: "plan-app", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
|
||||
Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
|
||||
Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2",
|
||||
Build: "build-app-c2"}}}
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, b.open)
|
||||
if len(b.sent) != 1 {
|
||||
t.Fatalf("anchor was sent again what it runs: %v", b.sent)
|
||||
}
|
||||
p, err := inv.PlanByID(ctx, "plan-app")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g := p.Modules["app"].Gate
|
||||
if g == nil || g.Since == nil || !g.Since.Equal(*judging.Since) || g.Passes < judging.Passes || g.TookOver != release.ID {
|
||||
t.Fatalf("the walk's gate starts from zero: %+v (the release's: %+v)", g, judging)
|
||||
}
|
||||
if p.Modules["app"].Previous != "c1" {
|
||||
t.Fatalf("what anchor ran before is %q, not c1", p.Modules["app"].Previous)
|
||||
}
|
||||
}
|
||||
|
||||
// A walk's first send that was made and not kept — the controller replaced between the two — is not made again
|
||||
// by the controller that takes over: the gate counts from that send (ADR 0353 §5, §6).
|
||||
func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) {
|
||||
withTheGate(t)
|
||||
judgeLeavesAsTheCore(t)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
// The old controller sends anchor app c2 and is replaced before it keeps the send.
|
||||
made := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
if _, err := made(ctx, open, names); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, context.Canceled
|
||||
}
|
||||
advancePlans(ctx, g.open)
|
||||
sendRollout = made
|
||||
if len(g.sent) != 1 || g.plan(t).Modules["app"].FirstAt != nil {
|
||||
t.Fatalf("the fixture's send was not made, or was kept: %v", g.sent)
|
||||
}
|
||||
last, _, err := g.open.inventory.SentTo(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, g.open)
|
||||
if len(g.sent) != 1 {
|
||||
t.Fatalf("the send was made again: %v", g.sent)
|
||||
}
|
||||
s := g.plan(t).Modules["app"]
|
||||
if s.FirstAt == nil || s.Gate == nil || s.Gate.Since == nil || !s.Gate.Since.Equal(last.At) {
|
||||
t.Fatalf("the gate does not count from the send made: %+v %+v", s, s.Gate)
|
||||
}
|
||||
// What anchor ran before was kept before the send, so a failure still puts it back (§5).
|
||||
if s.Previous != "c1" || s.Gate.From != "c1" {
|
||||
t.Fatalf("what anchor ran before is %q (gate from %q), not c1", s.Previous, s.Gate.From)
|
||||
}
|
||||
}
|
||||
|
||||
// A release's send kept before it was made, and made, is judged by the controller that takes over without being
|
||||
// made again (ADR 0353 §8).
|
||||
func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) {
|
||||
withTheGate(t)
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
p, err := releaseBacklog(ctx, b.open, "")
|
||||
if err != nil || p == nil {
|
||||
t.Fatalf("no release opened: %v", err)
|
||||
}
|
||||
if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gate == nil || !p.Release.Gate.Pending {
|
||||
t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err)
|
||||
}
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The send is made, and the controller is replaced before it keeps it.
|
||||
if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, b.open)
|
||||
if len(b.sent) != 1 {
|
||||
t.Fatalf("anchor was sent again: %v", b.sent)
|
||||
}
|
||||
g := b.release(t).Release.Gate
|
||||
if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 {
|
||||
t.Fatalf("anchor is not judged on the send made: %+v", g)
|
||||
}
|
||||
}
|
||||
|
||||
// A build no gate has passed goes to one machine of a step, its first; a machine that would be its second waits
|
||||
// for the next step, and one whose every build passed goes now (ADR 0353 §2).
|
||||
func TestABuildThatPassedNoGateGoesToOneMachineOfAStep(t *testing.T) {
|
||||
withTheGate(t)
|
||||
waiting := map[string][]inventory.CarriedMove{
|
||||
"desk": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}},
|
||||
"laptop": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}},
|
||||
"shanks": {{Module: "app", To: "c2"}},
|
||||
"novox": {{Module: "app", To: "c2"}, {Module: "ctl", To: "k1"}},
|
||||
}
|
||||
passed := func(module, commit string) bool { return module == "app" && commit == "c2" }
|
||||
never := func(string) bool { return false }
|
||||
step := restStep([]string{"desk", "laptop", "shanks", "novox"}, waiting, passed, never)
|
||||
if !slices.Equal(step, []string{"desk", "shanks", "novox"}) {
|
||||
t.Fatalf("the step is %v: laptop would be new's second machine before it passed", step)
|
||||
}
|
||||
if step := restStep([]string{"laptop"}, waiting, func(string, string) bool { return true }, never); !slices.Equal(step, []string{"laptop"}) {
|
||||
t.Fatalf("a machine whose every build passed waits: %v", step)
|
||||
}
|
||||
}
|
||||
|
||||
// The node-engine and the mesh's resolver go to the rest one machine at a time, even passed (ADR 0353 §9): a step
|
||||
// carries either to one machine only, while what else passed goes everywhere at once.
|
||||
func TestTheNodeEngineAndTheResolverGoToTheRestOneAtATime(t *testing.T) {
|
||||
withTheGate(t)
|
||||
waiting := map[string][]inventory.CarriedMove{
|
||||
"desk": {{Module: hostModule, To: "h2"}},
|
||||
"laptop": {{Module: hostModule, To: "h2"}, {Module: "app", To: "c2"}},
|
||||
"shanks": {{Module: "app", To: "c2"}},
|
||||
"novox": {{Module: "dns", To: "d2"}},
|
||||
"ace": {{Module: "dns", To: "d2"}},
|
||||
}
|
||||
resolver := catalogue.Manifest{Module: "dns", Claims: []catalogue.Claim{{Name: resolverSeat}}}
|
||||
alone := func(m string) bool {
|
||||
if m == "dns" {
|
||||
return oneAtATime(m, resolver)
|
||||
}
|
||||
return oneAtATime(m, catalogue.Manifest{Module: m})
|
||||
}
|
||||
step := restStep([]string{"desk", "laptop", "shanks", "novox", "ace"}, waiting,
|
||||
func(string, string) bool { return true }, alone)
|
||||
if !slices.Equal(step, []string{"desk", "shanks", "novox"}) {
|
||||
t.Fatalf("the step is %v: the node-engine and the resolver each reach one machine of it", step)
|
||||
}
|
||||
if oneAtATime("app", catalogue.Manifest{Module: "app"}) {
|
||||
t.Fatal("a leaf module is held to one machine at a time")
|
||||
}
|
||||
}
|
||||
|
||||
// A first machine that fails still stops the walk (ADR 0236 §5, kept by ADR 0353 §3): the rest are never sent.
|
||||
func TestAFirstMachineThatFailsStillStopsTheWalk(t *testing.T) {
|
||||
withTheGate(t)
|
||||
b := aBacklogOnThree(t)
|
||||
ctx := t.Context()
|
||||
gateEvery = 0
|
||||
b.broken["anchor"] = true
|
||||
advancePlans(ctx, b.open)
|
||||
for i := 0; i < 6 && b.release(t).Open(); i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
if p := b.release(t); p.State != inventory.PlanFailed || p.Release.Rest {
|
||||
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
for _, s := range b.sent {
|
||||
if !slices.Equal(s, []string{"anchor"}) {
|
||||
t.Fatalf("sent %v: the rest after a failed first machine", b.sent)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The control node is sent last in a step of the rest (ADR 0353 §1): the machine holding the controller is
|
||||
// ordered after every other.
|
||||
func TestTheControlNodeIsSentLastInAStep(t *testing.T) {
|
||||
withTheGate(t)
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklogOnThree(t)
|
||||
ctx := t.Context()
|
||||
register(t, b.open, catalogue.Manifest{Module: catalogue.ControllerSeatName, Version: "1"})
|
||||
if _, err := b.open.inventory.Assign(ctx, "anchor", catalogue.ControllerSeatName); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said := events(t)
|
||||
gateEvery = 0
|
||||
for i := 0; i < 12 && (len(b.sent) == 0 || b.release(t).Open()); i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
if p := b.release(t); !slices.Equal(p.Release.Order, []string{"desk", "laptop", "anchor"}) {
|
||||
t.Fatalf("the order is %v", p.Release.Order)
|
||||
}
|
||||
laptop, anchor := slices.Index(*said, "send laptop"), slices.Index(*said, "send anchor")
|
||||
if laptop < 0 || anchor < laptop || slices.Contains((*said)[laptop:anchor], "judge") {
|
||||
t.Fatalf("the control node is not sent last in the rest's step: %v", *said)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put
|
||||
// back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4).
|
||||
func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) {
|
||||
withTheGate(t)
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
reporting := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing
|
||||
b.sent = append(b.sent, names)
|
||||
current, _ := open.inventory.CurrentBuilds(ctx)
|
||||
return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep",
|
||||
map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit})
|
||||
}
|
||||
return reporting(ctx, open, names)
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = reporting })
|
||||
gateEvery = 0
|
||||
for i := 0; i < 8; i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
p := b.release(t)
|
||||
g := p.Release.Gates["laptop"]
|
||||
if !p.Open() || g == nil || g.Verdict != "" || g.Silent {
|
||||
t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g)
|
||||
}
|
||||
since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute)
|
||||
g.Since = &since
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, b.open)
|
||||
p = b.release(t)
|
||||
if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 {
|
||||
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed {
|
||||
t.Fatal("a silent machine's build was marked failed")
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" {
|
||||
t.Fatalf("laptop was put back: %v", sent)
|
||||
}
|
||||
obs, err := silentObservation(ctx, inv)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) {
|
||||
t.Fatalf("the silence is not said: %+v", obs)
|
||||
}
|
||||
}
|
||||
@@ -379,6 +379,19 @@ func rolloutMint(ctx context.Context, again bool) error {
|
||||
}
|
||||
modules++
|
||||
|
||||
case broker.KindProvisioner:
|
||||
// A machine's provisioner client, on an account of its own (novox/hq ADR 0349 clause 11), delivered
|
||||
// to the module running it.
|
||||
m, inShelf := shelf[p.Module]
|
||||
if !inShelf || !runsTheProvisionerClient(m) {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
if err := issueProvisionerClient(ctx, inv, m, p.Node, known, busAddress); err != nil {
|
||||
return err
|
||||
}
|
||||
modules++
|
||||
|
||||
default:
|
||||
skipped++
|
||||
}
|
||||
|
||||
+141
-11
@@ -6,6 +6,7 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
@@ -39,6 +40,9 @@ func rotateCommand(ctx context.Context, args []string) error {
|
||||
// of one provision, each with its own credential; one module that leaked its credential (novox/hq
|
||||
// issue 268) is no reason to restart every other one on the machine.
|
||||
module := set.String("module", "", "only this consuming module's credential")
|
||||
// Why, which the vault asks of every rotation (issue 767) and the hand-act log records; required only where a
|
||||
// provider's credential is the vault's, so nothing the operator typed before changes meaning.
|
||||
why := addHandActFlags(set)
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -89,17 +93,6 @@ func rotateCommand(ctx context.Context, args []string) error {
|
||||
|
||||
// Every machine at both ends, named before anything changes. A person about to rotate a
|
||||
// production credential is entitled to know the blast radius before it is the past tense.
|
||||
affected := map[string]bool{}
|
||||
for _, h := range holders {
|
||||
affected[h.Consumer] = true
|
||||
affected[h.Provider] = true
|
||||
}
|
||||
machines := make([]string, 0, len(affected))
|
||||
for name := range affected {
|
||||
machines = append(machines, name)
|
||||
}
|
||||
sort.Strings(machines)
|
||||
|
||||
fmt.Printf("rotating %s for %d holder(s):\n", provision, len(holders))
|
||||
for _, h := range holders {
|
||||
// The module, because a machine may hold several credentials for one provision and
|
||||
@@ -107,6 +100,35 @@ func rotateCommand(ctx context.Context, args []string) error {
|
||||
fmt.Printf(" %s on %s, from %s%s\n", h.ConsumerModule, h.Consumer, h.Provider, asLocal(h.Local))
|
||||
}
|
||||
|
||||
// **The vault's, where the provider is filled the new way** (issue 767, ADR 0349 clause 8): its credential is
|
||||
// the vault's, so the vault remakes it and its holders take it from there; the controller writes no value of its
|
||||
// own and sends nothing for it. Which holders go where is read before anything changes.
|
||||
toVault, today, err := rotateRoutes(holders,
|
||||
func(provider, provision string) (string, error) { return offeringModuleOn(ctx, provider, provision) },
|
||||
filledTheNewWay)
|
||||
if err != nil {
|
||||
return fmt.Errorf("nothing was rotated: %w", err)
|
||||
}
|
||||
if len(toVault) > 0 && !why.given() {
|
||||
return errors.New("not rotated: the vault holds this credential and rotates it only with a reason, " +
|
||||
"`--why \"…\"`; nothing was asked")
|
||||
}
|
||||
if why.given() {
|
||||
why.record(ctx, "rotate", append([]string{provision}, rotateNarrowArgs(*only, *module)...))
|
||||
}
|
||||
for _, g := range toVault {
|
||||
words, err := rotateProvisionInVault(theVault(ctx), g, *only, *module, actingAs(), *why.why)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(words)
|
||||
}
|
||||
if len(today) == 0 {
|
||||
return nil
|
||||
}
|
||||
holders = today
|
||||
machines := machinesOf(holders)
|
||||
|
||||
for _, h := range holders {
|
||||
if err := inv.RotateSecret(ctx, h.Provision, h.Consumer, h.ConsumerModule, h.Provider, h.Local); err != nil {
|
||||
// Partly rotated, and said so plainly. What is gone is remade on the next push, so
|
||||
@@ -137,6 +159,21 @@ func rotateCommand(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// machinesOf is every machine at both ends of these holders, in order.
|
||||
func machinesOf(holders []inventory.Holder) []string {
|
||||
affected := map[string]bool{}
|
||||
for _, h := range holders {
|
||||
affected[h.Consumer] = true
|
||||
affected[h.Provider] = true
|
||||
}
|
||||
machines := make([]string, 0, len(affected))
|
||||
for name := range affected {
|
||||
machines = append(machines, name)
|
||||
}
|
||||
sort.Strings(machines)
|
||||
return machines
|
||||
}
|
||||
|
||||
// ofModule is the holders whose consuming module is this one; all of them when none is named.
|
||||
func ofModule(holders []inventory.Holder, module string) []inventory.Holder {
|
||||
if module == "" {
|
||||
@@ -165,3 +202,96 @@ func asLocal(local string) string {
|
||||
}
|
||||
return " (as " + local + ")"
|
||||
}
|
||||
|
||||
// rotateGroup is the holders of one provider's credential for one provision whose provider is filled the new way:
|
||||
// the vault's to rotate (issue 767).
|
||||
type rotateGroup struct {
|
||||
Provider, Provision, Module string
|
||||
Holders []inventory.Holder
|
||||
}
|
||||
|
||||
// rotateRoutes says which holders the vault rotates, grouped by provider and provision, and which today's path
|
||||
// does: the vault's are those whose provider's offering module on its machine is filled the new way (the switch
|
||||
// `accept --provider` reads too). Anything that cannot be read is an error, never today's path by default: a value
|
||||
// remade where its holders no longer look would be reported rotated and change nothing.
|
||||
func rotateRoutes(holders []inventory.Holder, offering func(provider, provision string) (string, error),
|
||||
ofTheVault vaultSwitch) ([]rotateGroup, []inventory.Holder, error) {
|
||||
type key struct{ provider, provision string }
|
||||
decided := map[key]int{} // index into groups, or -1 for today's path
|
||||
var groups []rotateGroup
|
||||
var today []inventory.Holder
|
||||
for _, h := range holders {
|
||||
k := key{h.Provider, h.Provision}
|
||||
i, seen := decided[k]
|
||||
if !seen {
|
||||
module, err := offering(h.Provider, h.Provision)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("which module offers %s on %s cannot be read: %w", h.Provision, h.Provider, err)
|
||||
}
|
||||
ours := false
|
||||
if module != "" {
|
||||
if ours, err = ofTheVault(module); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
i = -1
|
||||
if ours {
|
||||
i = len(groups)
|
||||
groups = append(groups, rotateGroup{Provider: h.Provider, Provision: h.Provision, Module: module})
|
||||
}
|
||||
decided[k] = i
|
||||
}
|
||||
if i < 0 {
|
||||
today = append(today, h)
|
||||
continue
|
||||
}
|
||||
groups[i].Holders = append(groups[i].Holders, h)
|
||||
}
|
||||
return groups, today, nil
|
||||
}
|
||||
|
||||
// rotateNarrowArgs is --consumer and --module as typed, for the hand-act log.
|
||||
func rotateNarrowArgs(consumer, module string) []string {
|
||||
var out []string
|
||||
if consumer != "" {
|
||||
out = append(out, "--consumer", consumer)
|
||||
}
|
||||
if module != "" {
|
||||
out = append(out, "--module", module)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// rotateProvisionInVault asks the vault to rotate one provider's credential for a provision (issue 767): a shared
|
||||
// value is made anew for every holder, and the vault refuses it for one consumer alone; for a provider whose
|
||||
// consumers keep a login each (ADR 0158 §11), consumer and module name whose own login is remade, and only theirs.
|
||||
// The vault says secret-changed and each holder takes the new value from it; nothing is sealed or sent here.
|
||||
func rotateProvisionInVault(ask vaultAsker, g rotateGroup, consumer, module, by, why string) (string, error) {
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return "", errors.New("not rotated: the vault rotates a credential only with a reason, `--why \"…\"`; nothing was asked")
|
||||
}
|
||||
args := provisionArgs(g.Provider, g.Provision)
|
||||
args["by"], args["why"] = by, strings.TrimSpace(why)
|
||||
if consumer != "" {
|
||||
args["consumer"] = consumer
|
||||
}
|
||||
if module != "" {
|
||||
args["consumer_module"] = module
|
||||
}
|
||||
changed, err := endVaultCall(ask, "rotate", args)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s from %s not rotated, and nothing else was: %w", g.Provision, g.Provider, err)
|
||||
}
|
||||
if len(changed.Each) > 0 {
|
||||
each := make([]string, 0, len(changed.Each))
|
||||
for _, c := range changed.Each {
|
||||
each = append(each, fmt.Sprintf("%s (generation %d)", c.Name, c.Generation))
|
||||
}
|
||||
return fmt.Sprintf("rotated in the vault, each consumer's own login of %s from %s, asked by %s: %s; the values "+
|
||||
"are sealed and not shown, and each holder takes its own from the vault", g.Provision, g.Provider,
|
||||
changed.By, strings.Join(each, ", ")), nil
|
||||
}
|
||||
return fmt.Sprintf("rotated %s in the vault (generation %d), one value for every holder of %s from %s, asked by "+
|
||||
"%s; the value is sealed and not shown, and each holder takes it from the vault", changed.Name,
|
||||
changed.Generation, g.Provision, g.Provider, changed.By), nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// `rotate <provision>` reaches the vault for a provider filled the new way (issue 767), and the controller writes no
|
||||
// value of its own for it: its holders go to the vault, named by provider and provision, and no other holder does.
|
||||
func TestARotationOfAProvisionTheVaultOwnsGoesToTheVaultAndNotTheControllersStore(t *testing.T) {
|
||||
holders := []inventory.Holder{
|
||||
{Provision: "postgres-database", Consumer: "g14", ConsumerModule: "gitea", Provider: "novox"},
|
||||
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "umami", Provider: "novox"},
|
||||
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "n8n", Provider: "ace"},
|
||||
}
|
||||
offering := func(provider, provision string) (string, error) {
|
||||
if provider == "novox" {
|
||||
return "postgres", nil
|
||||
}
|
||||
return "postgres-old", nil
|
||||
}
|
||||
var asked []string
|
||||
moved := func(module string) (bool, error) {
|
||||
asked = append(asked, module)
|
||||
return module == "postgres", nil
|
||||
}
|
||||
vault, today, err := rotateRoutes(holders, offering, moved)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(vault) != 1 || vault[0].Provider != "novox" || vault[0].Provision != "postgres-database" ||
|
||||
vault[0].Module != "postgres" || len(vault[0].Holders) != 2 {
|
||||
t.Fatalf("to the vault: %+v", vault)
|
||||
}
|
||||
if len(today) != 1 || today[0].Provider != "ace" {
|
||||
t.Fatalf("today's path rotates a holder the vault owns, or misses one it does not: %+v", today)
|
||||
}
|
||||
if fmt.Sprint(asked) != "[postgres postgres-old]" {
|
||||
t.Fatalf("the switch was read for %v, not the offering module", asked)
|
||||
}
|
||||
}
|
||||
|
||||
// A switch that cannot be read, or a provider whose offering module cannot be found, routes nothing: an error
|
||||
// before anything changes, never today's path by default.
|
||||
func TestARotationWhoseOwnerCannotBeReadChangesNothing(t *testing.T) {
|
||||
holders := []inventory.Holder{{Provision: "postgres-database", Consumer: "g14", ConsumerModule: "gitea",
|
||||
Provider: "novox"}}
|
||||
broken := func(string) (bool, error) { return false, errors.New("the catalogue is unreadable") }
|
||||
if _, _, err := rotateRoutes(holders, func(string, string) (string, error) { return "postgres", nil }, broken); err == nil {
|
||||
t.Fatal("an unreadable switch fell back to today's path")
|
||||
}
|
||||
fine := func(string) (bool, error) { return true, nil }
|
||||
if _, _, err := rotateRoutes(holders, func(string, string) (string, error) { return "", errors.New("no store") },
|
||||
fine); err == nil {
|
||||
t.Fatal("an unreadable provider fell back to today's path")
|
||||
}
|
||||
}
|
||||
|
||||
// What the vault is told: the provider and provision, and --consumer and --module as the one consumer whose own
|
||||
// login is remade; nothing named when the operator named nobody, so a shared value is rotated for every holder.
|
||||
func TestTheVaultIsToldTheProvisionAndTheConsumerTheOperatorNamed(t *testing.T) {
|
||||
group := rotateGroup{Provider: "novox", Provision: "postgres-database", Module: "postgres"}
|
||||
for _, c := range []struct {
|
||||
consumer, module string
|
||||
want, absent []string
|
||||
}{
|
||||
{"", "", []string{`"provider":"novox"`, `"provision":"postgres-database"`, `"why":"yearly"`,
|
||||
`"by":"g14/claude-code"`}, []string{`"consumer"`, `"consumer_module"`}},
|
||||
{"g14", "", []string{`"consumer":"g14"`}, []string{`"consumer_module"`}},
|
||||
{"", "gitea", []string{`"consumer_module":"gitea"`}, []string{`"consumer":`}},
|
||||
{"g14", "gitea", []string{`"consumer":"g14"`, `"consumer_module":"gitea"`}, nil},
|
||||
} {
|
||||
var said string
|
||||
ask := func(verb string, args map[string]any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(map[string]any{"verb": verb, "args": args})
|
||||
said = string(raw)
|
||||
return json.Marshal(vaultChanged{Name: "provision/novox/postgres-database", By: "g14/claude-code",
|
||||
Each: []vaultChanged{{Name: "pair/novox/postgres-database/g14/gitea", Generation: 3}}})
|
||||
}
|
||||
words, err := rotateProvisionInVault(ask, group, c.consumer, c.module, "g14/claude-code", "yearly")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(said, `"verb":"rotate"`) {
|
||||
t.Fatalf("%+v: the vault was asked %s", c, said)
|
||||
}
|
||||
for _, w := range c.want {
|
||||
if !strings.Contains(said, w) {
|
||||
t.Errorf("%+v: the vault was not told %s: %s", c, w, said)
|
||||
}
|
||||
}
|
||||
for _, a := range c.absent {
|
||||
if strings.Contains(said, a) {
|
||||
t.Errorf("%+v: the vault was told %s: %s", c, a, said)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(words, "pair/novox/postgres-database/g14/gitea (generation 3)") {
|
||||
t.Errorf("%+v: the operator was not told whose login was remade: %s", c, words)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The vault rotates only with a why: asked of the vault without one, nothing is sent; and the vault's refusal (a
|
||||
// shared value named for one consumer) is the operator's answer, not a rotation in the controller's store.
|
||||
func TestARotationInTheVaultSaysWhyAndPassesOnItsRefusal(t *testing.T) {
|
||||
group := rotateGroup{Provider: "anchor", Provision: "stand-in-api", Module: "stand-in"}
|
||||
sent := 0
|
||||
ask := func(string, map[string]any) (json.RawMessage, error) {
|
||||
sent++
|
||||
return nil, errors.New("not rotated: provision/anchor/stand-in-api is one value every holder shares")
|
||||
}
|
||||
if _, err := rotateProvisionInVault(ask, group, "", "", "g14/claude-code", " "); err == nil || sent != 0 {
|
||||
t.Fatalf("a rotation without a why reached the vault: %v", err)
|
||||
}
|
||||
if _, err := rotateProvisionInVault(ask, group, "g14", "", "g14/claude-code", "suspected"); err == nil ||
|
||||
!strings.Contains(err.Error(), "every holder shares") {
|
||||
t.Fatalf("the vault's refusal was not said: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0366 §3: a module's own runtime stated unhealthy by its runtime-health raises the module's usual
|
||||
// condition, module.<module>.<machine>.unhealthy, through ADR 0240's path, saying the tool's reason, and the first
|
||||
// statement that says it healthy clears it.
|
||||
func TestAnUnhealthyOwnRuntimeRaisesTheModulesUsualCondition(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv, k := open.inventory, conditionsFrom
|
||||
const key = "module.mesh-vault.anchor.unhealthy"
|
||||
said := func(at time.Time, state, reason string) link.Health {
|
||||
return link.Health{Contract: link.LivenessContract, At: at, Resources: []link.ResourceHealth{{
|
||||
Module: "mesh-vault", Resource: "mesh-vault.own-runtime", Kind: "process", Target: "mesh-vault-runtime.service",
|
||||
State: state, Reason: reason, Since: at, Check: "tool"}}}
|
||||
}
|
||||
why := "re-sealing to the operator key failed on 3 passes in a row: the operator key is not readable"
|
||||
for i := 0; i < 2; i++ {
|
||||
at := h0.Add(time.Duration(i) * time.Minute)
|
||||
if err := stateHealth(ctx, inv, k, "anchor", said(at, link.StateUnhealthy, why), at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
c, had, err := k.Get(ctx, key)
|
||||
if err != nil || !had {
|
||||
t.Fatalf("two unhealthy statements of its own runtime raised no %s: %v", key, err)
|
||||
}
|
||||
evidence := ""
|
||||
for _, e := range c.Evidence {
|
||||
evidence += e.Said + "\n"
|
||||
}
|
||||
if !strings.Contains(c.Summary+evidence, "operator key is not readable") {
|
||||
t.Errorf("the condition does not say the tool's reason: %s / %s", c.Summary, evidence)
|
||||
}
|
||||
at := h0.Add(3 * time.Minute)
|
||||
if err := stateHealth(ctx, inv, k, "anchor", said(at, link.StateHealthy, ""), at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
list, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, o := range list {
|
||||
if o.Key == key {
|
||||
t.Fatalf("a healthy statement did not clear %s", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A module moves only when its program changes** (novox/hq issue 721). A merge check builds every module the
|
||||
// delivery plan moves and compares each with the build the mesh holds of it; a walk is cut without a module whose
|
||||
// check found its whole image the same as that build, while the mesh still holds it, and says so among the modules
|
||||
// it left unchanged. A merge of one sentence in a package a program imports but never reaches moves no build of it.
|
||||
|
||||
// sameAsRunning are the modules of a walk its checks found the same as the build the mesh holds, each with what its
|
||||
// check said. **Only when every pull request the walk answers was checked at the head it merged, and every one of
|
||||
// those checks built the module and found it the same as the build the mesh still holds**: a merge the walk answers
|
||||
// without a pull request, a check of another head, a check that built it differently, or a held build that moved
|
||||
// since, leaves the module in the walk.
|
||||
func sameAsRunning(plan *inventory.Plan, builds []link.CheckedBuild, held map[string]string) map[string]string {
|
||||
if plan.Delivery == nil || len(plan.Delivery.Merges) == 0 {
|
||||
return nil
|
||||
}
|
||||
for _, m := range plan.Delivery.Merges {
|
||||
if _, ok := pullOf(m); !ok {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
pulls := reuseAskOf(plan).Pulls
|
||||
heads := mergedHeadsOf(plan)
|
||||
out := map[string]string{}
|
||||
for name := range plan.Modules {
|
||||
said := ""
|
||||
for _, pull := range pulls {
|
||||
b, ok := checkedBuildOf(builds, pull, heads[pull], name)
|
||||
if !ok || !stillHeld(b.Built, name, held) {
|
||||
said = ""
|
||||
break
|
||||
}
|
||||
if said == "" {
|
||||
said = b.Built.Same
|
||||
}
|
||||
}
|
||||
if said != "" {
|
||||
out[name] = said
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// checkedBuildOf is the newest build a check of the pull request at head made of the module; false when none did.
|
||||
func checkedBuildOf(builds []link.CheckedBuild, pull link.PullRef, head, module string) (link.CheckedBuild, bool) {
|
||||
if head == "" {
|
||||
return link.CheckedBuild{}, false
|
||||
}
|
||||
for _, b := range builds {
|
||||
if b.Pull == pull && b.Built.Module == module && link.SameHead(b.Head, head) {
|
||||
return b, true
|
||||
}
|
||||
}
|
||||
return link.CheckedBuild{}, false
|
||||
}
|
||||
|
||||
// stillHeld is whether a check found every artifact of the module the same as a build the mesh still holds.
|
||||
func stillHeld(b link.CheckBuilt, module string, held map[string]string) bool {
|
||||
if b.Same == "" || b.NotBuilt != "" || len(b.Made) == 0 || len(b.SameAs) != len(b.Made) {
|
||||
return false
|
||||
}
|
||||
for _, m := range b.Made {
|
||||
was := pinnedDigest(b.SameAs[m.Name])
|
||||
if was == "" || was != pinnedDigest(held[module+"/"+m.Name]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// pinnedDigest is a reference's pinned digest, empty when it is not pinned.
|
||||
func pinnedDigest(ref string) string {
|
||||
if i := strings.LastIndex(ref, "@"); i >= 0 {
|
||||
return ref[i+1:]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// leaveSameAsRunning takes from a walk being cut every module its checks found the same as the build the mesh holds
|
||||
// (sameAsRunning), and every dependent in the walk only for one of those (withDependentsOf): what was taken, with why. **Anything not read leaves the walk as it was**, said.
|
||||
func leaveSameAsRunning(ctx context.Context, inv *inventory.Inventory, plan *inventory.Plan, moved []string,
|
||||
edges []inventory.Edge) map[string]string {
|
||||
if plan.Delivery == nil || len(plan.Modules) == 0 {
|
||||
return nil
|
||||
}
|
||||
pulls := reuseAskOf(plan).Pulls
|
||||
if len(pulls) == 0 {
|
||||
return nil
|
||||
}
|
||||
builds, err := checkedBuildsOf(ctx, pulls)
|
||||
if err != nil {
|
||||
fmt.Printf(" the checks' builds could not be read, so every module moves: %v\n", err)
|
||||
// empty-on-error: said above; nothing left out of the walk is the safe answer, every module moves as before
|
||||
return nil
|
||||
}
|
||||
held, err := inv.RegisteredHeld(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf(" what the mesh holds could not be read, so every module moves: %v\n", err)
|
||||
// empty-on-error: said above; nothing left out of the walk is the safe answer, every module moves as before
|
||||
return nil
|
||||
}
|
||||
same := withDependentsOf(sameAsRunning(plan, builds, held), plan, moved, edges)
|
||||
if len(same) == 0 {
|
||||
return nil
|
||||
}
|
||||
names := make([]string, 0, len(same))
|
||||
for n := range same {
|
||||
names = append(names, n)
|
||||
}
|
||||
sort.Strings(names)
|
||||
withhold(plan, names, nil)
|
||||
for _, n := range names {
|
||||
fmt.Printf(" %s left out: %s\n", n, same[n])
|
||||
}
|
||||
return same
|
||||
}
|
||||
|
||||
// withDependentsOf adds to the modules found the same every module of the walk that is in it only as a dependent of
|
||||
// one of them (novox/hq ADR 0267 rule 3): **a dependent is left out when its base comes out the same**, since the
|
||||
// base it stands on stays the build the mesh holds. A module a merge moves itself, or a dependent of a base that
|
||||
// still moves, stays. moved are the modules the walk's merges move; edges the graph the walk was planned along.
|
||||
func withDependentsOf(same map[string]string, plan *inventory.Plan, moved []string, edges []inventory.Edge) map[string]string {
|
||||
if len(same) == 0 {
|
||||
return same
|
||||
}
|
||||
var still []string
|
||||
for _, n := range moved {
|
||||
if same[n] == "" {
|
||||
still = append(still, n)
|
||||
}
|
||||
}
|
||||
keep := map[string]bool{}
|
||||
for _, n := range reachableFrom(still, edges) {
|
||||
keep[n] = true
|
||||
}
|
||||
// Until nothing more is found: a dependent of a dependent left out is left out too.
|
||||
for grew := true; grew; {
|
||||
grew = false
|
||||
for n := range plan.Modules {
|
||||
if keep[n] || same[n] != "" {
|
||||
continue
|
||||
}
|
||||
var bases []string
|
||||
for _, e := range edges {
|
||||
ordering := e.Kind == inventory.EdgeBuiltBy || e.Kind == inventory.EdgeWorkerOf ||
|
||||
e.Kind == inventory.EdgePackages || e.Kind == edgeGroupOrder
|
||||
if !ordering && e.From == n && same[e.To] != "" && !slices.Contains(bases, e.To) {
|
||||
bases = append(bases, e.To)
|
||||
}
|
||||
}
|
||||
sort.Strings(bases)
|
||||
if len(bases) == 0 {
|
||||
// In the walk for no base found the same: whatever brought it in is not said here, so it moves.
|
||||
continue
|
||||
}
|
||||
same[n] = "unchanged: it is in the walk only for " + strings.Join(bases, ", ") +
|
||||
", which came out the same as the build the mesh holds (ADR 0267 rule 3)"
|
||||
grew = true
|
||||
}
|
||||
}
|
||||
return same
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// TestAWalkLeavesOutAModuleWhoseProgramDidNotChange is novox/hq issue 721: mesh-controller #335 changed one sentence
|
||||
// that mesh-builder never reaches; its check found build-agent the same as the build the mesh holds, so the walk moves
|
||||
// mesh-controller alone and says build-agent is unchanged.
|
||||
func TestAWalkLeavesOutAModuleWhoseProgramDidNotChange(t *testing.T) {
|
||||
pull := link.PullRef{Owner: "novox", Repo: "mesh-controller", Number: 335}
|
||||
head := "34050dc6aaaabbbbccccdddd"
|
||||
aPlan := func() *inventory.Plan {
|
||||
return &inventory.Plan{Tiers: [][]string{{"mesh-controller", "build-agent"}},
|
||||
Modules: map[string]*inventory.PlanModule{"mesh-controller": {}, "build-agent": {}},
|
||||
Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{{Repository: "novox/mesh-controller",
|
||||
Commit: "c2ec0a60", Number: 335, Head: head}}}}
|
||||
}
|
||||
held := map[string]string{"build-agent/agent": "store/build-agent@sha256:aaaa",
|
||||
"mesh-controller/controller": "store/mesh-controller@sha256:cccc"}
|
||||
builds := []link.CheckedBuild{
|
||||
{Pull: pull, Head: head[:12], Built: link.CheckBuilt{Module: "build-agent", Key: "k1",
|
||||
Made: []link.MadeArtifact{{Name: "agent", Kind: "image", Reference: "scratch/build-agent@sha256:bbbb"}},
|
||||
Same: "unchanged: its program is the same as running build agent aaaa",
|
||||
SameAs: map[string]string{"agent": "10.0.0.1:5000/build-agent@sha256:aaaa"}}},
|
||||
{Pull: pull, Head: head[:12], Built: link.CheckBuilt{Module: "mesh-controller", Key: "k2",
|
||||
Made: []link.MadeArtifact{{Name: "controller", Kind: "image", Reference: "scratch/mesh-controller@sha256:dddd"}}}},
|
||||
}
|
||||
|
||||
p := aPlan()
|
||||
same := sameAsRunning(p, builds, held)
|
||||
if len(same) != 1 || same["build-agent"] == "" {
|
||||
t.Fatalf("the walk leaves out %v, want build-agent alone", same)
|
||||
}
|
||||
withhold(p, []string{"build-agent"}, nil)
|
||||
if len(p.Tiers) != 1 || len(p.Tiers[0]) != 1 || p.Tiers[0][0] != "mesh-controller" {
|
||||
t.Fatalf("the walk moves %v, want mesh-controller alone", p.Tiers)
|
||||
}
|
||||
|
||||
// **The held build moved since the check**: what it was compared with no longer runs.
|
||||
moved := map[string]string{"build-agent/agent": "store/build-agent@sha256:eeee"}
|
||||
if same := sameAsRunning(aPlan(), builds, moved); len(same) != 0 {
|
||||
t.Fatalf("a module compared with a build the mesh no longer holds is left out: %v", same)
|
||||
}
|
||||
// **A check of another head** says nothing of this merge.
|
||||
other := aPlan()
|
||||
other.Delivery.Merges[0].Head = "ffffffffffffffff"
|
||||
if same := sameAsRunning(other, builds, held); len(same) != 0 {
|
||||
t.Fatalf("a check of another head left out %v", same)
|
||||
}
|
||||
// **Another pull request the walk answers** that did not find it the same leaves it in.
|
||||
two := aPlan()
|
||||
two.Delivery.Merges = append(two.Delivery.Merges, inventory.PlanMerge{Repository: "novox/mesh-catalog",
|
||||
Commit: "1234", Number: 9, Head: "abcdefabcdefabcdef"})
|
||||
if same := sameAsRunning(two, builds, held); len(same) != 0 {
|
||||
t.Fatalf("a module another merge moves is left out: %v", same)
|
||||
}
|
||||
// **A merge with no pull request** leaves the walk as it is.
|
||||
direct := aPlan()
|
||||
direct.Delivery.Merges[0].Number = 0
|
||||
if same := sameAsRunning(direct, builds, held); len(same) != 0 {
|
||||
t.Fatalf("a walk answering a merge without a pull request left out %v", same)
|
||||
}
|
||||
}
|
||||
|
||||
// TestADependentOfABaseThatCameOutTheSameIsLeftOut is ADR 0267 rule 3 (novox/hq issue 607): a dependent is in a walk
|
||||
// for its base; when the base comes out the same as the build the mesh holds, the dependent stands on that build
|
||||
// still, and leaves the walk with it — and so does a dependent of that dependent. One a merge moves itself stays.
|
||||
func TestADependentOfABaseThatCameOutTheSameIsLeftOut(t *testing.T) {
|
||||
edges := []inventory.Edge{{From: "app", To: "base", Kind: inventory.EdgeDeclared},
|
||||
{From: "app2", To: "app", Kind: inventory.EdgeDeclared}, {From: "own", To: "base", Kind: inventory.EdgeDeclared}}
|
||||
plan := &inventory.Plan{Modules: map[string]*inventory.PlanModule{"base": {}, "app": {}, "app2": {}, "own": {}}}
|
||||
same := withDependentsOf(map[string]string{"base": "unchanged: its program is the same"}, plan,
|
||||
[]string{"base", "own"}, edges)
|
||||
if !strings.Contains(same["app"], "only for base") || !strings.Contains(same["app2"], "only for app") {
|
||||
t.Fatalf("the dependents of a base that came out the same are not left out: %v", same)
|
||||
}
|
||||
if _, left := same["own"]; left {
|
||||
t.Fatalf("a module its merge moves itself is left out for its base: %v", same)
|
||||
}
|
||||
// A dependent of a base that still moves stays.
|
||||
edges = append(edges, inventory.Edge{From: "app", To: "own", Kind: inventory.EdgeDeclared})
|
||||
same = withDependentsOf(map[string]string{"base": "unchanged"}, plan, []string{"base", "own"}, edges)
|
||||
if _, left := same["app"]; left {
|
||||
t.Fatalf("a dependent of a base that still moves is left out: %v", same)
|
||||
}
|
||||
}
|
||||
|
||||
// Against the store, the path a real merge takes (novox/hq issues 721 and 611): a merge whose check built its one
|
||||
// module the same as the build the mesh holds is cut into a walk done with nothing to walk, whose record names the
|
||||
// module unchanged and why — the walk mesh-delivery reads as delivered, never stranded.
|
||||
func TestAWalkWhoseEveryBuildCameOutTheSameIsDone(t *testing.T) {
|
||||
open := windowed(t)
|
||||
endsItsLine(t)
|
||||
asksRecorded(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-one", Repository: "novox/one", Ref: "main",
|
||||
Module: "one", Commit: "c0", On: "builder", Asked: t0.Add(-time.Hour),
|
||||
Made: []inventory.Artifact{{Name: "one", Kind: "image", Reference: "store/one@sha256:aaaa"}},
|
||||
Sources: []inventory.BuildSource{{Paths: []string{"main.go", "go.mod"}}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// What runs: the module registered with that build's image (RegisteredHeld reads the module table).
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "one", Version: "1", Resources: []map[string]any{
|
||||
{"id": "app", "type": "container", "name": "one", "image": "store/one@sha256:aaaa"}}},
|
||||
inventory.Source{Repository: "novox/one", Ref: "main", BuiltFrom: "c0", Asked: time.Now()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pull := link.PullRef{Owner: "novox", Repo: "one", Number: 7}
|
||||
was := checkedBuildsOf
|
||||
checkedBuildsOf = func(context.Context, []link.PullRef) ([]link.CheckedBuild, error) {
|
||||
return []link.CheckedBuild{{Pull: pull, Head: "headaaaaaaaa", Built: link.CheckBuilt{Module: "one", Key: "k",
|
||||
Made: []link.MadeArtifact{{Name: "one", Kind: "image", Reference: "scratch/one@sha256:bbbb"}},
|
||||
Same: "unchanged: its program is the same as running build one aaaa",
|
||||
SameAs: map[string]string{"one": "10.0.0.1:5000/one@sha256:aaaa"}}}}, nil
|
||||
}
|
||||
t.Cleanup(func() { checkedBuildsOf = was })
|
||||
code := repoMerge("one", "one1aaaaaaaa", t0)
|
||||
code.Number, code.HeadSHA = 7, "headaaaaaaaa1111"
|
||||
hear(t, open, code, t0.Add(time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 {
|
||||
t.Fatalf("walks: %+v", ws)
|
||||
}
|
||||
w := ws[0]
|
||||
if w.State != inventory.PlanDone || len(w.Tiers) != 0 || len(w.Modules) != 0 {
|
||||
t.Fatalf("the walk reads %s, tiers %v, modules %v; want done with nothing to walk", w.State, w.Tiers, w.Modules)
|
||||
}
|
||||
if w.Delivery == nil || !strings.Contains(w.Delivery.Unchanged["one"], "its program is the same as running build") {
|
||||
t.Fatalf("the walk does not say one matched what runs: %+v", w.Delivery)
|
||||
}
|
||||
}
|
||||
@@ -43,6 +43,9 @@ func TestMain(m *testing.M) {
|
||||
}
|
||||
if os.Getenv(echoEnvironment) != "" {
|
||||
fmt.Printf("verb=%q caller=%q terminal=%v\n", os.Getenv("MESH_VERB"), os.Getenv("MESH_CALLER"), startedAtTheTerminal())
|
||||
if os.Getenv(echoFails) != "" {
|
||||
os.Exit(3)
|
||||
}
|
||||
os.Exit(0)
|
||||
}
|
||||
conditions.Unsayable = func(o conditions.Observation, field string, r outward.Refusal) {
|
||||
|
||||
@@ -457,6 +457,28 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
return nil, err
|
||||
}
|
||||
return []string{"forget-checks", str("owner"), str("repo"), str("number")}, nil
|
||||
case "try":
|
||||
// A branch tried before its pull request (novox/hq issue 686): asked, or read by the id it answered.
|
||||
if id := str("id"); id != "" {
|
||||
return []string{"try", "--id", id}, nil
|
||||
}
|
||||
if a.given["repository"] == "" && a.given["ref"] == "" {
|
||||
// Neither shape named: the command says its usage, which names both, as `build` does.
|
||||
return []string{"try"}, nil
|
||||
}
|
||||
if err := need("repository", "ref"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argv := []string{"try", str("repository"), str("ref")}
|
||||
for _, flag := range []string{"base", "tests"} {
|
||||
if v := str(flag); v != "" {
|
||||
argv = append(argv, "--"+flag, v)
|
||||
}
|
||||
}
|
||||
if on("store-only") {
|
||||
argv = append(argv, "--store-only")
|
||||
}
|
||||
return argv, nil
|
||||
case "cancel", "kill":
|
||||
if err := need("id"); err != nil {
|
||||
return nil, err
|
||||
@@ -708,6 +730,38 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
return nil, fmt.Errorf("%w: letting go of artifacts is a hand act, which says why. Nothing was done", err)
|
||||
}
|
||||
return append(argv, "--confirm", "--why", why), nil
|
||||
// The move of placement out of the settings layers (issue 736), a dry run unless confirmed.
|
||||
case "move-placement":
|
||||
argv := []string{"move-placement", "--json"}
|
||||
why := str("why")
|
||||
if !on("confirm") {
|
||||
if why != "" {
|
||||
return nil, errors.New("move-placement takes why only with confirm: without confirm nothing is " +
|
||||
"moved, and a reason for nothing would be recorded nowhere. Nothing was done")
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
if err := need("why"); err != nil {
|
||||
return nil, fmt.Errorf("%w: moving the settings layers' placement is an act by hand, which says why. "+
|
||||
"Nothing was done", err)
|
||||
}
|
||||
return append(argv, "--confirm", "--why", why), nil
|
||||
// The move of config into mesh-config (issue 664), a dry run unless confirmed.
|
||||
case "move-config":
|
||||
argv := []string{"move-config", "--json"}
|
||||
why := str("why")
|
||||
if !on("confirm") {
|
||||
if why != "" {
|
||||
return nil, errors.New("move-config takes why only with confirm: without confirm nothing is set, " +
|
||||
"and a reason for nothing would be recorded nowhere. Nothing was done")
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
if err := need("why"); err != nil {
|
||||
return nil, fmt.Errorf("%w: moving the settings layers into mesh-config is an act by hand, which says why. "+
|
||||
"Nothing was done", err)
|
||||
}
|
||||
return append(argv, "--confirm", "--why", why), nil
|
||||
case "images":
|
||||
if err := need("node"); err != nil {
|
||||
return nil, err
|
||||
@@ -815,6 +869,13 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
return nil, err
|
||||
}
|
||||
return []string{"secret", "ask", str("node"), str("module"), str("secret"), "--at", str("at")}, nil
|
||||
case "secret-move":
|
||||
// The move of the controller's secrets into the vault (design 57's V3, issue 676): no value in the arguments
|
||||
// or the answer, a machine's name at most.
|
||||
if n := str("node"); n != "" {
|
||||
return []string{"secret", "move", n}, nil
|
||||
}
|
||||
return []string{"secret", "move"}, nil
|
||||
case "issued":
|
||||
// The credential a provider's software issued, taken from the provider (novox/hq ADR 0305): no value in
|
||||
// the arguments, the provision alone.
|
||||
@@ -844,6 +905,13 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
if m := str("module"); m != "" {
|
||||
argv = append(argv, "--module", m)
|
||||
}
|
||||
// Why, which the vault asks of a rotation of a credential it holds (issue 767); a cause only beside it.
|
||||
if w := str("why"); w != "" {
|
||||
argv = append(argv, "--why", w)
|
||||
if c := str("cause"); c != "" {
|
||||
argv = append(argv, "--cause", c)
|
||||
}
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
_, node := a.given["node"]
|
||||
@@ -1014,10 +1082,16 @@ var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "co
|
||||
"hand-acts": true, "durations": true, "conditions": true, "doctor": true, "retire": true, "cleanup": true, "data": true,
|
||||
// What the records say the registries may keep (novox/hq ADR 0251).
|
||||
"artifacts": true, "collect": true, "images": true, "mirrors": true,
|
||||
// The move of placement says each layer it moved (issue 736).
|
||||
"move-placement": true,
|
||||
// The move of config says each value it set and where from (issue 664).
|
||||
"move-config": true,
|
||||
// The delivery's owner's verbs answer JSON where they read (plan, order, check, walks) — novox/hq ADR 0239.
|
||||
"delivery": true,
|
||||
// The build queue, each ask with its position among the waiting (novox/hq issue 455).
|
||||
"queue": true,
|
||||
// A branch tried before its pull request (novox/hq issue 686).
|
||||
"try": true,
|
||||
// A plan followed (novox/hq issue 229): `plans wait` answers JSON; its other forms say text, and so answer no data.
|
||||
"plans": true}
|
||||
|
||||
@@ -1057,6 +1131,10 @@ func repairingCommand(argv []string) string {
|
||||
return "collect"
|
||||
case argv[0] == "mirrors" && slices.Contains(argv, "--confirm"):
|
||||
return "mirrors"
|
||||
case argv[0] == "move-placement" && slices.Contains(argv, "--confirm"):
|
||||
return "move-placement"
|
||||
case argv[0] == "move-config" && slices.Contains(argv, "--confirm"):
|
||||
return "move-config"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -1308,6 +1386,10 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
||||
link.AnswersWithin(ctx, bound+planWaitStart)
|
||||
}
|
||||
}
|
||||
if verb == "try" && len(argv) > 1 && argv[1] == "--id" {
|
||||
// Reading a try holds its answer for the verdict, up to its bound (novox/hq issue 686, 229).
|
||||
link.AnswersWithin(ctx, tryWait+planWaitStart)
|
||||
}
|
||||
if answer, read := readHere(ctx, argv); read {
|
||||
return answer, nil
|
||||
}
|
||||
@@ -1704,6 +1786,16 @@ func takenFromTheProvider(argv []string) bool {
|
||||
!strings.HasPrefix(argv[2], "-")
|
||||
}
|
||||
|
||||
// movedByTheirMachines is exactly the line the `secret-move` verb composes: `secret move`, or `secret move <node>`
|
||||
// (design 57's V3, issue 676). It carries no value and answers none: each machine opens what was sealed to it and
|
||||
// hands it to the vault, which never replaces a value it holds.
|
||||
func movedByTheirMachines(argv []string) bool {
|
||||
if len(argv) < 2 || argv[0] != "secret" || argv[1] != "move" {
|
||||
return false
|
||||
}
|
||||
return len(argv) == 2 || (len(argv) == 3 && argv[2] != "" && !strings.HasPrefix(argv[2], "-"))
|
||||
}
|
||||
|
||||
// terminalOnly refuses, through any verb, a command that is the operator's at the controller's terminal
|
||||
// alone (novox/hq ADR 0266). **Every `node` subcommand that is not a read**: `node account` and
|
||||
// `node agent-account` above all. Whoever may call a verb includes agents, and an agent that named itself
|
||||
@@ -1720,7 +1812,8 @@ func terminalOnly(argv []string) error {
|
||||
// Of a secret's commands only rotation, which seals the new value to the machine that uses it, and the
|
||||
// `give` verb's own line: an own secret typed by the operator into the desk's hidden prompt, sealed to this
|
||||
// call and then to the module's machine, so no value travels in the verb or its answer (hq ADR 0259 §10).
|
||||
if argv[0] == "secret" && (len(argv) < 2 || argv[1] != "rotate") && !givenAtTheDesk(argv) && !takenFromTheProvider(argv) {
|
||||
if argv[0] == "secret" && (len(argv) < 2 || argv[1] != "rotate") && !givenAtTheDesk(argv) && !takenFromTheProvider(argv) &&
|
||||
!movedByTheirMachines(argv) {
|
||||
return terminalRefusal("secret %s is run at the controller's terminal only, never through a verb: accepting, "+
|
||||
"recovering or exporting a secret hands it to whoever asks, and that includes agents (novox/hq ADR "+
|
||||
"0266). Nothing was done", strings.Join(argv[1:], " "))
|
||||
|
||||
@@ -285,6 +285,8 @@ var accountedFlags = map[string]map[string]string{
|
||||
"collect": {"json": "set by the verb: the answer is data"},
|
||||
"images": {"json": "set by the verb: the answer is data"},
|
||||
"mirrors": {"json": "set by the verb: the answer is data"},
|
||||
"move-placement": {"json": "set by the verb: the answer is data"},
|
||||
"move-config": {"json": "set by the verb: the answer is data"},
|
||||
"conditions history": {"json": "set by the verb: the answer is data"},
|
||||
"conditions show": {"json": "set by the verb: the answer is data"},
|
||||
"healers": {"json": "set by the verb: the answer is data"},
|
||||
|
||||
@@ -12,8 +12,11 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/secrets"
|
||||
)
|
||||
|
||||
@@ -50,6 +53,9 @@ func secretCommand(ctx context.Context, args []string) error {
|
||||
return secretRecover(ctx, args[1:])
|
||||
case "export":
|
||||
return secretExport(ctx, args[1:])
|
||||
case "move":
|
||||
// The controller's secrets moved into the vault, each by the machine it was sealed to (design 57's V3).
|
||||
return secretMove(ctx, args[1:])
|
||||
default:
|
||||
return errors.New(secretUsage)
|
||||
}
|
||||
@@ -108,7 +114,12 @@ func secretCommand(ctx context.Context, args []string) error {
|
||||
if value == "" {
|
||||
return errors.New("there is nothing to seal")
|
||||
}
|
||||
return acceptForProvision(ctx, node, module, name, value)
|
||||
// The provider's module decides the path: filled the new way, the vault keeps the provision's credential.
|
||||
return bySwitch(module, filledTheNewWay,
|
||||
func() error { return acceptForProvision(ctx, node, module, name, value) },
|
||||
func() error {
|
||||
return said(acceptIntoVault(theVault(ctx), provisionArgs(node, name), value, actingAs()))
|
||||
})
|
||||
}
|
||||
if *desk != "" {
|
||||
if *from != "" || *provider != "" {
|
||||
@@ -125,23 +136,112 @@ func secretCommand(ctx context.Context, args []string) error {
|
||||
if value == "" {
|
||||
return errors.New("there is nothing to seal")
|
||||
}
|
||||
// The module whose secrets hold the value decides the path: with --provider, the value is the provider's one
|
||||
// credential for the provision, so the provider's module decides, as for an outside offer above.
|
||||
decides := module
|
||||
if *provider != "" {
|
||||
if decides, err = offeringModuleOn(ctx, *provider, name); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return bySwitch(decides, filledTheNewWay,
|
||||
func() error { return acceptKept(ctx, node, module, name, *provider, *local, value) },
|
||||
func() error {
|
||||
// Sealed here to the vault's key for this one call; only the sealed box leaves this process.
|
||||
target := ownSecretArgs(node, module, name)
|
||||
if *provider != "" {
|
||||
if *local != "" {
|
||||
return errors.New("--local names a credential inside the module; the vault keeps one credential " +
|
||||
"per provider and provision, so it takes no --local: nothing was sealed")
|
||||
}
|
||||
target = provisionArgs(*provider, name)
|
||||
}
|
||||
if *provider != "" {
|
||||
return said(acceptIntoVault(theVault(ctx), target, value, actingAs()))
|
||||
}
|
||||
return giveIntoVault(ctx, node, module, name, target, value)
|
||||
})
|
||||
}
|
||||
|
||||
// announceTheGiven is the announcement of a secret given at the terminal (announceSecretGiven, the step acceptKept
|
||||
// runs); a variable so a test can make it fail.
|
||||
var announceTheGiven = announceSecretGiven
|
||||
|
||||
// giveIntoVault is `secret accept` of a module's own secret for a module filled the new way, under the same
|
||||
// N1-give rule as acceptKept, through the same steps (keepGiven, announceSecretGiven): a module that runs as an
|
||||
// account of its own is announced on the operator's channels before the vault is asked, and when the announcement
|
||||
// fails nothing is sealed and the vault is never called; any other is sealed first and announced after.
|
||||
func giveIntoVault(ctx context.Context, node, module, name string, target map[string]any, value string) error {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
trusted, err := open.inventory.RunsAsItsOwnAccount(ctx, module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var words string
|
||||
askedBy := link.Caller()
|
||||
_, unannounced, err := keepGiven(trusted,
|
||||
func() error {
|
||||
return announceTheGiven(ctx, node, module, name, "at the controller's terminal", askedBy)
|
||||
},
|
||||
func() (bool, error) {
|
||||
var err error
|
||||
words, err = acceptIntoVault(theVault(ctx), target, value, actingAs())
|
||||
return false, err
|
||||
})
|
||||
if err != nil {
|
||||
if trusted && unannounced != nil {
|
||||
return fmt.Errorf("%s runs as an account of its own, and the change of its %s could not be announced on "+
|
||||
"your channels first, so nothing was sealed and the vault was not asked: %w", module, name, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
fmt.Println(words)
|
||||
if unannounced != nil {
|
||||
fmt.Printf(" this change could NOT be announced on the operator's channels: %v\n", unannounced)
|
||||
}
|
||||
// The giving is recorded, never the value (novox/hq ADR 0380 §6), as acceptKept records its own.
|
||||
act := terminalGiveHandAct(node, module, name, askedBy)
|
||||
if err := onTheBus(func(conn *nats.Conn) error {
|
||||
_, err := link.RecordHandAct(ctx, conn, act)
|
||||
return err
|
||||
}); err != nil {
|
||||
fmt.Printf(" this act could NOT be recorded in the hand-act log, and is done anyway: %v\n", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// said prints what was done, or answers why not.
|
||||
func said(words string, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(words)
|
||||
return nil
|
||||
}
|
||||
|
||||
// acceptKept is today's `secret accept`: the value sealed into the controller's store, for a module whose secrets
|
||||
// are filled the old way.
|
||||
func acceptKept(ctx context.Context, node, module, name, provider, local, value string) error {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
|
||||
if *provider != "" {
|
||||
if provider != "" {
|
||||
// Into the pair, not into the module's own secrets: what the provider is asked to create
|
||||
// and what the consumer reads are the same value, and neither end can be told a different
|
||||
// one later without the other (novox/hq 04-ISSUES/070).
|
||||
if err := open.inventory.AcceptSecretForPair(ctx, name, node, module, *provider, *local, value); err != nil {
|
||||
if err := open.inventory.AcceptSecretForPair(ctx, name, node, module, provider, local, value); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s on %s now holds %q from %s%s, sealed to both machines.\n", module, node, name, *provider, asLocal(*local))
|
||||
fmt.Printf("%s on %s now holds %q from %s%s, sealed to both machines.\n", module, node, name, provider, asLocal(local))
|
||||
fmt.Printf(" the mesh cannot read it back, will not replace it with one of its own, and will not rotate it\n")
|
||||
fmt.Printf(" run `push %s` and `push %s` to send it\n", *provider, node)
|
||||
fmt.Printf(" run `push %s` and `push %s` to send it\n", provider, node)
|
||||
return nil
|
||||
}
|
||||
// A trusted party's secret is announced before it is kept (the confirmation review of 2026-10-09, N1-give):
|
||||
@@ -151,8 +251,11 @@ func secretCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
askedBy := link.Caller()
|
||||
untilStart, unannounced, err := keepGiven(trusted,
|
||||
func() error { return announceSecretGiven(ctx, node, module, name, "at the controller's terminal") },
|
||||
func() error {
|
||||
return announceSecretGiven(ctx, node, module, name, "at the controller's terminal", askedBy)
|
||||
},
|
||||
func() (bool, error) { return open.inventory.AcceptGivenSecret(ctx, node, module, name, value) })
|
||||
if err != nil {
|
||||
if trusted && unannounced != nil {
|
||||
@@ -164,6 +267,15 @@ func secretCommand(ctx context.Context, args []string) error {
|
||||
if unannounced != nil {
|
||||
fmt.Printf(" this change could NOT be announced on the operator's channels: %v\n", unannounced)
|
||||
}
|
||||
// The giving is recorded, never the value (novox/hq ADR 0380 §6): the desk already logs its own act; the
|
||||
// terminal's own secret accept did not until now.
|
||||
act := terminalGiveHandAct(node, module, name, askedBy)
|
||||
if err := onTheBus(func(conn *nats.Conn) error {
|
||||
_, err := link.RecordHandAct(ctx, conn, act)
|
||||
return err
|
||||
}); err != nil {
|
||||
fmt.Printf(" this act could NOT be recorded in the hand-act log, and is done anyway: %v\n", err)
|
||||
}
|
||||
// Not printed back, and there is nowhere it could be printed from: it is sealed to that
|
||||
// machine and the mesh cannot read it again.
|
||||
fmt.Printf("%s on %s now holds %q, sealed to that machine.\n", module, node, name)
|
||||
@@ -206,7 +318,8 @@ const secretUsage = "secret rotate <node> <module> <name> [--why <text> [--cause
|
||||
"secret accept <node> <module> <name> [--from <file> | --at-desk <machine>] [--provider <node> [--local <name>]]\n" +
|
||||
"secret accept <provider's node> <provider module> <provision> [--from <file>] (a credential issued outside the mesh, for every consumer)\n" +
|
||||
"secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>] [--provider <node>]\n" +
|
||||
"secret export [--out <file>]"
|
||||
"secret export [--out <file>]\n" +
|
||||
"secret move [<node>] (every secret the controller holds, moved into the vault by the machine it was sealed to)"
|
||||
|
||||
// secretRecover is break-glass: a secret opened with the operator's key, written to a file.
|
||||
//
|
||||
@@ -485,6 +598,26 @@ func secretRotate(ctx context.Context, args []string) error {
|
||||
return errors.New(secretUsage)
|
||||
}
|
||||
node, module, name := rest[0], rest[1], rest[2]
|
||||
return bySwitch(module, filledTheNewWay,
|
||||
func() error { return rotateKept(ctx, node, module, name, why) },
|
||||
func() error {
|
||||
reason := ""
|
||||
if why.why != nil {
|
||||
reason = strings.TrimSpace(*why.why)
|
||||
}
|
||||
words, err := rotateInVault(theVault(ctx), ownSecretArgs(node, module, name), actingAs(), reason)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
why.record(ctx, "secret rotate", []string{node, module, name})
|
||||
fmt.Println(words)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// rotateKept is today's `secret rotate`: made anew in the controller's store and sent, for a module whose
|
||||
// secrets are filled the old way.
|
||||
func rotateKept(ctx context.Context, node, module, name string, why handActFlags) error {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -541,6 +674,16 @@ func whoAsked() string {
|
||||
return "the mesh"
|
||||
}
|
||||
|
||||
// terminalGiveHandAct is the hand-act log's entry for a secret accepted at the controller's terminal
|
||||
// (novox/hq ADR 0380 §6): who asked, where and that it happened — never the value, which the act never
|
||||
// carries a field for.
|
||||
func terminalGiveHandAct(node, module, name, askedBy string) link.HandAct {
|
||||
return link.HandAct{Verb: "secret accept", Args: []string{node, module, name},
|
||||
Why: fmt.Sprintf("the operator gave %s for %s on %s at the controller's terminal, asked by %s", name, module,
|
||||
node, askedByName(askedBy)),
|
||||
Cause: "given-at-the-terminal"}
|
||||
}
|
||||
|
||||
// keepGiven keeps a value given at the controller's terminal, and announces it on the operator's channels
|
||||
// (the confirmation review of 2026-10-09, N1-give). **A trusted party's — a module running as an account of its
|
||||
// own: the router, a verified channel — is announced before it is kept, and not kept when the announcement
|
||||
|
||||
@@ -0,0 +1,336 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// `secret move` — the controller's secrets move into the vault (novox/hq ADR 0349 clause 7; design 57's V3, issue 676;
|
||||
// the operator's decision of 2026-10-14 on that issue). Run once, after the feature's delivery.
|
||||
//
|
||||
// **The controller cannot open what it holds, so each machine moves its own.** For every machine — or the ones named —
|
||||
// the controller reads every row sealed to it, as the store holds it, and asks that machine's node-engine to move each
|
||||
// one, signed (link.AskMoveSecret): the engine opens it with its sealing key and hands it to the vault's move-in, sealed
|
||||
// to the call's key, keeping its origin and generation. The vault takes it from that engine alone, never over a value
|
||||
// it holds, and reads back what it kept to the engine, which compares; the engine answers only then. What moved is
|
||||
// recorded here, never a value. Nothing is rotated, and nothing points at the vault because of this: a module's
|
||||
// secrets are filled from the vault only once it moves (V4, issue 647).
|
||||
//
|
||||
// **A machine asleep is moved when it is next heard from**: once the move has started, a machine that reports is moved
|
||||
// in the background (moveWhenHeard), and what is left of it is said on the next `secret move`.
|
||||
//
|
||||
// The controller's record of asks at a desk, which holds no value, is copied as it is, once (`move-asks`).
|
||||
|
||||
const secretMoveUsage = "secret move [<node>] — move every secret the controller holds into the vault, as it is: each " +
|
||||
"machine's node-engine opens what was sealed to it and hands it to the vault, which reads it back; no value is " +
|
||||
"rotated, and a machine not answering is moved when it is next heard from"
|
||||
|
||||
// moveAsk is the ask of one row of a machine's engine.
|
||||
type moveAsk func(row inventory.SecretToMove, ask link.MoveSecretAsk) (link.MoveSecretAnswer, error)
|
||||
|
||||
// nodeMove is what the move did on one machine.
|
||||
type nodeMove struct {
|
||||
Node string
|
||||
Moved int
|
||||
Already int
|
||||
// Refused are the rows not moved, each with why; Changed the rows changed in the store after they moved.
|
||||
Refused []string
|
||||
Changed []string
|
||||
// Unreached is why the machine's engine could not be asked: it is moved when next heard from.
|
||||
Unreached error
|
||||
// Left is how many rows were not asked because the machine was not reached.
|
||||
Left int
|
||||
// Failed is why its rows could not be read.
|
||||
Failed error
|
||||
}
|
||||
|
||||
// done says nothing is left to move on the machine.
|
||||
func (m nodeMove) done() bool {
|
||||
return m.Failed == nil && m.Unreached == nil && len(m.Refused) == 0
|
||||
}
|
||||
|
||||
func (m nodeMove) String() string {
|
||||
var b strings.Builder
|
||||
if m.Failed != nil {
|
||||
fmt.Fprintf(&b, "%s: its rows could not be read, nothing was moved: %v\n", m.Node, m.Failed)
|
||||
return b.String()
|
||||
}
|
||||
fmt.Fprintf(&b, "%s: %d moved into the vault and read back the same, %d there already", m.Node, m.Moved, m.Already)
|
||||
if n := len(m.Refused); n > 0 {
|
||||
fmt.Fprintf(&b, ", %d not moved", n)
|
||||
}
|
||||
b.WriteString("\n")
|
||||
for _, r := range m.Refused {
|
||||
fmt.Fprintf(&b, " not moved: %s\n", r)
|
||||
}
|
||||
for _, c := range m.Changed {
|
||||
fmt.Fprintf(&b, " %s was changed in the controller's store after it moved; the vault keeps what it took, "+
|
||||
"and the move never replaces it\n", c)
|
||||
}
|
||||
if m.Unreached != nil {
|
||||
fmt.Fprintf(&b, " %d left: %s was not reached (%v); it is moved when it is next heard from\n", m.Left, m.Node,
|
||||
m.Unreached)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// moveRowAsk is a row as its machine's engine is asked to move it.
|
||||
func moveRowAsk(r inventory.SecretToMove) link.MoveSecretAsk {
|
||||
a := link.MoveSecretAsk{Origin: r.Origin, Generation: r.Generation, MadeAt: r.MadeAt.UTC(), Sealed: r.Sealed,
|
||||
SealedTo: r.SealedTo}
|
||||
if r.Own() {
|
||||
a.Module, a.Name = r.Module, r.Name
|
||||
} else {
|
||||
a.Provider, a.Provision, a.Consumer, a.ConsumerModule, a.Local = r.Provider, r.Provision, r.Consumer,
|
||||
r.ConsumerModule, r.Local
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// moveNode moves every row sealed to one machine that has not moved, one ask each, and records what the vault kept.
|
||||
// The first ask the engine does not answer ends the machine's turn: the rest wait for it to be heard from again.
|
||||
func moveNode(node string, rows func(string) ([]inventory.SecretToMove, error), ask moveAsk,
|
||||
record func(inventory.SecretToMove, int) error) nodeMove {
|
||||
m := nodeMove{Node: node}
|
||||
all, err := rows(node)
|
||||
if err != nil {
|
||||
m.Failed = err
|
||||
return m
|
||||
}
|
||||
for n, r := range all {
|
||||
switch {
|
||||
case r.Moved != nil && r.Moved.Digest == r.Digest:
|
||||
m.Already++
|
||||
continue
|
||||
case r.Moved != nil:
|
||||
m.Changed = append(m.Changed, r.Secret)
|
||||
continue
|
||||
case !r.Current:
|
||||
m.Refused = append(m.Refused, fmt.Sprintf("%s is sealed to an earlier sealing key of %s, which no longer "+
|
||||
"opens it: nothing can move it; it moves once it is made or accepted again", r.Secret, node))
|
||||
continue
|
||||
}
|
||||
answer, err := ask(r, moveRowAsk(r))
|
||||
if err != nil {
|
||||
m.Unreached, m.Left = err, len(all)-n
|
||||
return m
|
||||
}
|
||||
if answer.Refused != "" {
|
||||
m.Refused = append(m.Refused, fmt.Sprintf("%s: %s", r.Secret, answer.Refused))
|
||||
continue
|
||||
}
|
||||
if answer.Secret != r.Secret {
|
||||
m.Refused = append(m.Refused, fmt.Sprintf("%s: the vault kept it as %s, not under the name the move "+
|
||||
"gives it", r.Secret, answer.Secret))
|
||||
continue
|
||||
}
|
||||
if err := record(r, answer.Generation); err != nil {
|
||||
m.Refused = append(m.Refused, fmt.Sprintf("%s is in the vault, but that it moved could not be recorded: %v",
|
||||
r.Secret, err))
|
||||
continue
|
||||
}
|
||||
if answer.Unchanged {
|
||||
m.Already++
|
||||
} else {
|
||||
m.Moved++
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// movedAsksAtOnce is how many rows of the record of asks one call to the vault carries (mesh-vault MovedAsksAtOnce).
|
||||
const movedAsksAtOnce = 500
|
||||
|
||||
// moveAsks copies the controller's record of asks into the vault, in parts, and answers how many it copied.
|
||||
func moveAsks(asks []inventory.SecretAsk, vault vaultAsker) (int, error) {
|
||||
copied := 0
|
||||
for start := 0; start < len(asks); start += movedAsksAtOnce {
|
||||
end := min(start+movedAsksAtOnce, len(asks))
|
||||
var rows []map[string]any
|
||||
for _, a := range asks[start:end] {
|
||||
row := map[string]any{"id": a.ID, "node": a.Node, "module": a.Module, "name": a.Name, "desk": a.Desk,
|
||||
"asked_by": a.AskedBy, "opened_at": a.OpenedAt.UTC().Format(time.RFC3339Nano)}
|
||||
if a.EndedAt != nil {
|
||||
row["ended_at"], row["outcome"] = a.EndedAt.UTC().Format(time.RFC3339Nano), a.Outcome
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
raw, err := vault("move-asks", map[string]any{"asks": rows})
|
||||
if err != nil {
|
||||
return copied, fmt.Errorf("the vault did not take the record of asks after %d rows: %w", copied, err)
|
||||
}
|
||||
var said struct {
|
||||
Copied int `json:"copied"`
|
||||
Already int `json:"already"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &said)
|
||||
copied += said.Copied + said.Already
|
||||
}
|
||||
return copied, nil
|
||||
}
|
||||
|
||||
// moveOnTheBus is the move's ask over the bus, signed with the mesh's key.
|
||||
func moveOnTheBus(ctx context.Context, open *stores, conn *nats.Conn, by string) (moveAsk, error) {
|
||||
ident, err := open.Identity(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the mesh's signing key cannot be read, so nothing was asked: %w", err)
|
||||
}
|
||||
return func(r inventory.SecretToMove, row link.MoveSecretAsk) (link.MoveSecretAnswer, error) {
|
||||
return link.AskMoveSecret(ctx, conn, ident, r.Node, by, row, link.MoveSecretWithin)
|
||||
}, nil
|
||||
}
|
||||
|
||||
// secretMove is `secret move [<node>]`: the record of asks copied once, then every machine's rows — or one's — and
|
||||
// what moved said per machine. Any row not moved fails the command, said by name.
|
||||
func secretMove(ctx context.Context, args []string) error {
|
||||
if len(args) > 1 || (len(args) == 1 && (args[0] == "" || strings.HasPrefix(args[0], "-"))) {
|
||||
return errors.New(secretMoveUsage)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
by := actingAs()
|
||||
if by == "" {
|
||||
by = handOverBy()
|
||||
}
|
||||
move, err := inv.StartSecretMove(ctx, by)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var nodes []string
|
||||
if len(args) == 1 {
|
||||
if _, err := inv.NodeByName(ctx, args[0]); err != nil {
|
||||
return fmt.Errorf("nothing was moved: %w", err)
|
||||
}
|
||||
nodes = args
|
||||
} else {
|
||||
all, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, n := range all {
|
||||
nodes = append(nodes, n.Name)
|
||||
}
|
||||
}
|
||||
fmt.Printf("the move of the secrets into the vault, started %s by %s\n", move.StartedAt.Local().Format("2006-01-02 15:04"),
|
||||
move.StartedBy)
|
||||
if move.AsksMovedAt == nil {
|
||||
asks, err := inv.SecretAsks(ctx, time.Time{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
copied, err := moveAsks(asks, theVault(ctx))
|
||||
if err != nil {
|
||||
return fmt.Errorf("the record of asks was not copied, and no secret was moved: %w", err)
|
||||
}
|
||||
if err := inv.RecordAsksMoved(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("the record of asks at a desk: %d rows copied into the vault as they are\n", copied)
|
||||
}
|
||||
failed := 0
|
||||
err = onTheBus(func(conn *nats.Conn) error {
|
||||
ask, err := moveOnTheBus(ctx, open, conn, "the move of the secrets, by "+by)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, node := range nodes {
|
||||
m := moveNode(node, func(n string) ([]inventory.SecretToMove, error) { return inv.SecretsToMoveFrom(ctx, n) },
|
||||
ask, func(r inventory.SecretToMove, g int) error { return inv.RecordSecretMoved(ctx, r, g) })
|
||||
fmt.Print(m.String())
|
||||
if m.Failed != nil || len(m.Refused) > 0 {
|
||||
failed++
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if failed > 0 {
|
||||
return fmt.Errorf("on %d machine(s) a secret was not moved, said above by name", failed)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// moveHeard is the machines being moved in the background now, and when each was last moved, so a machine that reports
|
||||
// often is not asked again before moveHeardEvery has passed.
|
||||
var moveHeard = struct {
|
||||
sync.Mutex
|
||||
running map[string]bool
|
||||
last map[string]time.Time
|
||||
}{running: map[string]bool{}, last: map[string]time.Time{}}
|
||||
|
||||
// moveHeardEvery is how often a machine heard from is moved at most.
|
||||
const moveHeardEvery = 10 * time.Minute
|
||||
|
||||
// moveWhenHeard moves what is left of a machine that was just heard from, once the move has started: a machine asleep
|
||||
// when it started is moved when it next reports. What it did is said on the controller's output, and recorded for the
|
||||
// next `secret move`.
|
||||
func moveWhenHeard(ctx context.Context, open *stores, conn *nats.Conn, node string) {
|
||||
if open == nil || conn == nil || node == "" {
|
||||
return
|
||||
}
|
||||
moveHeard.Lock()
|
||||
if moveHeard.running[node] || time.Since(moveHeard.last[node]) < moveHeardEvery {
|
||||
moveHeard.Unlock()
|
||||
return
|
||||
}
|
||||
moveHeard.running[node], moveHeard.last[node] = true, time.Now()
|
||||
moveHeard.Unlock()
|
||||
defer func() {
|
||||
moveHeard.Lock()
|
||||
delete(moveHeard.running, node)
|
||||
moveHeard.Unlock()
|
||||
}()
|
||||
inv := open.inventory
|
||||
move, started, err := inv.SecretMoveStarted(ctx)
|
||||
if err != nil || !started {
|
||||
return
|
||||
}
|
||||
rows, err := inv.SecretsToMoveFrom(ctx, node)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
left := false
|
||||
for _, r := range rows {
|
||||
left = left || (r.Moved == nil && r.Current)
|
||||
}
|
||||
if !left {
|
||||
return
|
||||
}
|
||||
ask, err := moveOnTheBus(ctx, open, conn, "the move of the secrets, by "+move.StartedBy+", on hearing from "+node)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: %s was heard from and its secrets not moved: %v\n", node, err)
|
||||
return
|
||||
}
|
||||
m := moveNode(node, func(string) ([]inventory.SecretToMove, error) { return rows, nil }, ask,
|
||||
func(r inventory.SecretToMove, g int) error { return inv.RecordSecretMoved(ctx, r, g) })
|
||||
if m.done() {
|
||||
fmt.Printf("mesh-controller: %s was heard from; the move of its secrets is done: %s", node, m.String())
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: %s was heard from; the move of its secrets is not done: %s", node, m.String())
|
||||
}
|
||||
|
||||
// moveHeardOn is the connection a machine heard from is moved over: the serving controller's bus.
|
||||
func moveHeardOn() *nats.Conn {
|
||||
if js := servingBus.Load(); js != nil {
|
||||
return js.Conn()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user