Merge remote-tracking branch 'origin/main' into fix/403-fix-time
mesh/merge-gate head daf52b3aeb79 merged into trunk 4dbd4066ca54 — pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every …
mesh/repo-check head daf52b3aeb79 merged into trunk 4dbd4066ca54 — pass: its merge-check.sh passed
mesh/delivery failed: its walk failed, carrying its own commit: a gate on a first machine (what it carried put back), a build, a machine
mesh/delivery-group group fix/403-fix-time failed: a member failed

# Conflicts:
#	internal/link/events.go
This commit is contained in:
jochen
2026-10-12 21:28:52 +02:00
68 changed files with 6816 additions and 111 deletions
+2 -3
View File
@@ -16,8 +16,7 @@ RUN CGO_ENABLED=0 go build -trimpath -ldflags '-s -w' -o /mesh-builder ./cmd/mes
FROM ${ALPINE_BASE}
# git to clone what it is asked to build, and the docker client to build and push it. The daemon
# is the machine's, reached through its socket — a build machine shares the runtime it was given
# rather than running one inside itself. xz to read an upstream release an archive is pinned to
# (novox/hq ADR 0316): Go's standard library reads gzip and bzip2 and not xz.
RUN apk add --no-cache git docker-cli xz
# rather than running one inside itself.
RUN apk add --no-cache git docker-cli
COPY --from=build /mesh-builder /usr/local/bin/mesh-builder
ENTRYPOINT ["/usr/local/bin/mesh-builder"]
+8 -2
View File
@@ -256,7 +256,13 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
v, err = builder.Check(building, builder.Command, checkSpecOf(request), workspace, registry, forgeFrom(), say)
if err == nil {
result.Check = &link.CheckOutcome{Verdict: v.Verdict, Summary: v.Summary, Report: v.Report,
Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo)}
Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo),
Head: v.Head, Trunk: v.Trunk, Beside: v.Beside}
// What each of a group's heads was judged as (novox/hq ADR 0323 §6).
for _, m := range v.Members {
result.Check.Members = append(result.Check.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo,
Commit: m.Head, Trunk: m.Trunk})
}
}
} else if err == nil {
// The package-registry credential is a build input, so it is resolved before the clone: a
@@ -354,7 +360,7 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
}
for _, m := range c.Members {
spec.Group = append(spec.Group, builder.GroupHead{Owner: m.Owner, Repo: m.Repo, Repository: m.Repository,
Ref: m.Ref, Paths: m.Paths})
Ref: m.Ref, Paths: m.Paths, Base: m.Base})
}
return spec
}
+118
View File
@@ -0,0 +1,118 @@
package main
import (
"fmt"
"sort"
"strings"
"time"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
)
// kindApplyFailing is S21's kind: a machine whose declarations have been refused, or applied only in part,
// for longer than one apply interval (novox/hq issue 233).
const kindApplyFailing = "apply-failing"
// applyEvery is how often a node-engine applies what it was last told when nothing new arrives: its
// ReconcileEvery. A failure still there after one more of them is not one the next apply clears.
const applyEvery = 5 * time.Minute
// packageManagerKey is how a node-engine names a package manager that does not answer among what failed
// (mesh-host's probe.go): "the package manager (pacman)".
const packageManagerKey = "the package manager ("
// watchApplyFailing is S21: **a machine that does not apply what it was sent is said by the mesh, not only
// to someone who asks for `status`** (novox/hq issue 233, issue 187's family). A workstation refused every
// declaration for an hour and a half because its package manager did not answer, and the mesh's only word
// on it was a line in `status`. Raised once the machine's first report of this failure is older than one apply
// interval **while the machine is heard** — judged by time, not by how many reports said it: a node-engine
// says a re-apply only when how it went changed, so the same failure is one report however long it lasts.
// Names the refusal's or the failure's words; cleared by the clean apply the node-engine then says. Not while
// the machine is silent, asleep or not known: S1 says silent, and asleep is not lost.
func watchApplyFailing(f *signalFacts) []conditions.Observation {
heard := map[string]bool{}
for _, m := range f.machines {
if !m.asleep() && !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) <= heartbeatBound(m.every) {
heard[m.name] = true
}
}
var out []conditions.Observation
for _, d := range f.doing {
if !d.Wrong() || d.Since == nil || f.now.Sub(*d.Since) <= applyEvery || !heard[d.Node] {
continue
}
out = append(out, applyFailing(d, f.now))
}
return out
}
// applyFailing is one machine's S21 condition.
func applyFailing(d inventory.Doing, now time.Time) conditions.Observation {
m := d.Node
since := d.Since.UTC().Format("2006-01-02 15:04 MST")
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m,
Severity: conditions.Warning, Said: doingWords(d),
Resolved: m + " applies what it is sent again"}
how := "refused its declaration whole"
if d.Outcome != inventory.OutcomeRefused {
how = "applied its declaration only in part"
}
o.Summary = fmt.Sprintf("%s has %s since %s (bound %s): %s", m, how, since, ago(applyEvery), failedNames(d))
o.Headline = m + " is not applying its update"
o.Explanation = fmt.Sprintf("%s has %s for %s, and trying again has not helped. The details say what fails.",
m, plainHow(d), humanDuration(now.Sub(*d.Since)))
if tool := unansweringTool(d); tool != "" {
o.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent "+
"is applied, since %s (bound %s)", m, tool, since, ago(applyEvery))
o.Headline = fmt.Sprintf("%s cannot install packages", m)
o.Explanation = fmt.Sprintf("The package manager on %s (%s) has not answered for %s, so no package reaches "+
"it. Everything else is still applied, so an update that repairs it can land; if none does, it needs a "+
"look on the machine.", m, tool, humanDuration(now.Sub(*d.Since)))
}
return o
}
// plainHow is how a machine failed, as the operator reads it.
func plainHow(d inventory.Doing) string {
if d.Outcome == inventory.OutcomeRefused {
return "refused every update"
}
return "applied its updates only in part"
}
// unansweringTool is the package manager a machine said does not answer, or "".
func unansweringTool(d inventory.Doing) string {
for _, r := range d.Failed {
if rest, ok := strings.CutPrefix(r.ID, packageManagerKey); ok {
return strings.TrimSuffix(rest, ")")
}
}
return ""
}
// failedNames is what failed, by name: the refusal, or the failed resources' ids.
func failedNames(d inventory.Doing) string {
if d.Outcome == inventory.OutcomeRefused {
return "refused"
}
ids := make([]string, 0, len(d.Failed))
for _, r := range d.Failed {
ids = append(ids, r.ID)
}
sort.Strings(ids)
return "failed " + strings.Join(ids, ", ")
}
// doingWords is the machine's own words on what failed: the evidence, kept inside the mesh.
func doingWords(d inventory.Doing) string {
if d.Outcome == inventory.OutcomeRefused {
return "refused: " + oneLine(d.Refused)
}
var parts []string
for _, r := range d.Failed {
parts = append(parts, r.ID+": "+oneLine(r.Error))
}
sort.Strings(parts)
return strings.Join(parts, "; ")
}
+123
View File
@@ -0,0 +1,123 @@
package main
import (
"context"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
)
// unansweringPackageManager is anchor's last report as the workstation's read on 2026-10-04, once its
// node-engine applies the rest (novox/hq issue 233): the package failed and the package manager did not answer,
// the same failure on times reports since for before now.
func unansweringPackageManager(now time.Time, forHowLong time.Duration, times int) inventory.Doing {
since := now.Add(-forHowLong)
return inventory.Doing{Node: "anchor", Outcome: inventory.OutcomeFailed, At: now.Add(-time.Second),
Since: &since, Times: times, Applied: 41, Failed: []inventory.FailedResource{
{ID: "apply", Error: `applying "pacman.base": nothing can be said about "base": this is the arch host and ` +
"pacman does not answer here. Either this machine is not Arch, or its package database is broken: " +
"pacman exited 1: error: config file /etc/pacman.conf could not be read: No such file or directory"},
{ID: "the package manager (pacman)", Error: "pacman does not answer here, so every package in this " +
"declaration fails and the rest of it is applied (novox/hq issue 233): pacman exited 1: error: config " +
"file /etc/pacman.conf could not be read: No such file or directory"}}}
}
// **A machine whose package manager does not answer is raised by the mesh itself, naming the tool and the
// probe's words** (novox/hq issue 233): the words in the evidence, the path kept out of the summary.
func TestAPackageManagerThatDoesNotAnswerIsRaisedNamingItsWords(t *testing.T) {
now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
f := calm(now)
f.doing = []inventory.Doing{unansweringPackageManager(now, 90*time.Minute, 18)}
got := watchApplyFailing(f)
if len(got) != 1 {
t.Fatalf("raised %d conditions, want one: %+v", len(got), got)
}
o := got[0]
if o.Key() != "machine.anchor.apply-failing" || o.Machine != "anchor" {
t.Errorf("the condition is not the machine's: %s", o.Key())
}
if !strings.Contains(o.Summary, "pacman does not answer") || !strings.Contains(o.Summary, "since 2026-10-04 12:30") {
t.Errorf("the summary does not name the tool and since when: %s", o.Summary)
}
if strings.Contains(o.Summary, "/etc/") {
t.Errorf("the summary carries a path, which belongs in the evidence: %s", o.Summary)
}
if !strings.Contains(o.Said, "could not be read") {
t.Errorf("the evidence does not carry the probe's words: %s", o.Said)
}
if !strings.Contains(o.Headline, "cannot install packages") || o.Explanation == "" || o.Resolved == "" {
t.Errorf("the plain words do not say it: %+v", o)
}
}
// **A whole refusal is raised too**, with the node-engine's words: what the workstation's report said before the
// node-engine applied the rest.
func TestARefusalThatGoesOnIsRaised(t *testing.T) {
now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
since := now.Add(-time.Hour)
f := calm(now)
f.doing = []inventory.Doing{{Node: "anchor", Outcome: inventory.OutcomeRefused, Since: &since, Times: 12,
Refused: "this is the arch host and pacman does not answer here", At: now.Add(-time.Minute)}}
got := watchApplyFailing(f)
if len(got) != 1 || !strings.Contains(got[0].Summary, "refused its declaration whole") ||
!strings.Contains(got[0].Said, "pacman does not answer") {
t.Fatalf("a refusal that went on for an hour was not raised with its words: %+v", got)
}
}
// **Judged by time while the machine is heard, not by how many reports said it** (review of novox/hq issue 233):
// a node-engine says the same failure once, so one report an apply interval old is a fault that goes on; one
// younger is not yet; and a silent or sleeping machine is S1's.
func TestOneReportOlderThanAnApplyIsRaisedAndAQuietMachineIsNot(t *testing.T) {
now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
f := calm(now)
f.doing = []inventory.Doing{unansweringPackageManager(now, 6*time.Minute, 1)}
if got := watchApplyFailing(f); len(got) != 1 {
t.Errorf("one report of a failure six minutes old was not raised: %+v", got)
}
f.doing = []inventory.Doing{unansweringPackageManager(now, 4*time.Minute, 1)}
if got := watchApplyFailing(f); len(got) != 0 {
t.Errorf("a failure four minutes old was raised before one apply could clear it: %+v", got)
}
f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 1)}
f.machines[0].lastHeard = now.Add(-time.Hour)
if got := watchApplyFailing(f); len(got) != 0 {
t.Errorf("a silent machine was raised for its last report: %+v", got)
}
f = calm(now)
f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 1)}
f.doing[0].Node = "a-machine-the-mesh-does-not-know"
if got := watchApplyFailing(f); len(got) != 0 {
t.Errorf("a machine not heard of was raised: %+v", got)
}
}
// **A failed re-apply followed by a clean one clears S21** (review of novox/hq issue 233): the node-engine says
// the clean re-apply, the store keeps it as applied with no failure since, and the condition the failure raised
// is cleared through the keeper on the next tick.
func TestACleanReapplyAfterAFailedOneClearsTheCondition(t *testing.T) {
now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
store := conditions.NewInMemory()
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
defer k.Close(context.Background())
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
failing := calm(now)
failing.doing = []inventory.Doing{unansweringPackageManager(now, 20*time.Minute, 1)}
w.see(t.Context(), failing)
open, err := k.Open(t.Context())
if err != nil || len(open) != 1 || open[0].Kind != kindApplyFailing {
t.Fatalf("the failed re-apply raised %+v (%v), want apply-failing", open, err)
}
clean := calm(now.Add(5 * time.Minute))
clean.doing = nil // the clean re-apply recorded: NotDoingWhatTheyWereTold no longer lists the machine
w.see(t.Context(), clean)
if open, _ := k.Open(t.Context()); len(open) != 0 {
t.Fatalf("the condition stayed open after the clean re-apply: %+v", open)
}
}
+50 -7
View File
@@ -161,9 +161,7 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
return u, err == nil
})
fmt.Print(planText(plan))
direct := link.Checked{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Commit: p.Commit,
ID: link.NewBuildID(time.Now()), Verdict: "pass", Summary: noModuleTouched,
Gate: &link.CheckLayer{Verdict: "pass", Summary: noModuleTouched}, Plan: &plan}
direct := directVerdict(p, plan)
switch {
case !scope.gated() && !scope.Mesh:
fmt.Printf("%s/%s#%d (%.8s): %s, and the repository is not the mesh's: said, nothing run\n",
@@ -201,6 +199,24 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
return nil
}
// directVerdict is the verdict of a change no build judges — it touches no module, and either the repository is not
// the mesh's or the head holds no merge-check.sh. **It names the trunk commit it holds for** (novox/hq ADR 0323 §3):
// the base branch's head the forge announced with the head, so the merge tool can weigh it against the trunk's head
// and does not refuse it for ever (a verdict naming no trunk is never the trunk's head's). Nothing was built to
// judge: the trunk commit is the one the forge said the change was announced against.
func directVerdict(p link.PullUpdated, plan link.ChangePlan) link.Checked {
trunk := p.BaseSHA
if len(trunk) > 12 {
trunk = trunk[:12]
}
if trunk != "" {
plan.Trunk = trunk
}
return link.Checked{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Commit: p.Commit,
ID: link.NewBuildID(time.Now()), Verdict: "pass", Summary: noModuleTouched,
Gate: &link.CheckLayer{Verdict: "pass", Summary: noModuleTouched}, Plan: &plan, Trunk: trunk}
}
// checkRequestFor is the ask for one pull request's head: the repository as the mesh clones it, the head,
// and what is read beside it.
func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scope checkScope,
@@ -363,6 +379,9 @@ func checkedOf(result link.BuildResult) link.Checked {
case result.Check != nil:
c.Verdict, c.Summary, c.Report = result.Check.Verdict, result.Check.Summary, result.Check.Report
c.Gate, c.RepoCheck = result.Check.Gate, result.Check.RepoCheck
// What was judged (novox/hq ADR 0323 §3): the head merged into the trunk commit, and the commits of the
// repositories read beside it.
c.Trunk, c.Beside, c.Took = result.Check.Trunk, result.Check.Beside, result.Check.Took
if c.Gate == nil {
// A build seat from before the layers: its verdict is the gate's.
c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
@@ -388,14 +407,20 @@ func checkedOf(result link.BuildResult) link.Checked {
}
if result.Checked != nil {
c.Plan = result.Checked.Plan
if c.Plan != nil && c.Trunk != "" {
// The kept delivery plan records the trunk commit with its other inputs (ADR 0323 §3).
plan := *c.Plan
plan.Trunk = c.Trunk
c.Plan = &plan
}
// A delivery group's composed check (novox/hq ADR 0239): every head it judged, this one first.
if g := result.Checked; g.Group != "" {
c.Group = g.Group
c.Members = append(c.Members, link.CheckedMember{Owner: g.Owner, Repo: g.Repo, Number: g.Number,
Commit: result.Ref})
Commit: result.Ref, Trunk: c.Trunk})
for _, m := range g.Members {
c.Members = append(c.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo, Number: m.Number,
Commit: m.Ref})
Commit: m.Ref, Trunk: memberTrunk(result.Check, m)})
}
}
}
@@ -410,6 +435,20 @@ func checkedOf(result link.BuildResult) link.Checked {
return c
}
// memberTrunk is the trunk commit the build seat merged one of a group's heads into (novox/hq ADR 0323 §6), empty
// when it said none.
func memberTrunk(outcome *link.CheckOutcome, m link.GroupMember) string {
if outcome == nil {
return ""
}
for _, j := range outcome.Members {
if strings.EqualFold(j.Owner, m.Owner) && strings.EqualFold(j.Repo, m.Repo) && strings.HasPrefix(m.Ref, j.Commit) {
return j.Trunk
}
}
return ""
}
func prefixedAll(prefix string, items []string) []string {
out := make([]string, 0, len(items))
for _, i := range items {
@@ -424,8 +463,12 @@ func sayChecked(ctx context.Context, c link.Checked) {
if c.RepoCheck != nil {
repo = strings.ToUpper(c.RepoCheck.Verdict) + " — " + c.RepoCheck.Summary
}
fmt.Printf("%s: %s/%s#%d at %.8s checked on %s: gate %s — %s; repository %s\n", c.ID, c.Owner, c.Repo, c.Number,
c.Commit, orSomewhere(c.On), strings.ToUpper(c.Verdict), c.Summary, repo)
judged := ""
if c.Trunk != "" {
judged = " merged into trunk " + c.Trunk
}
fmt.Printf("%s: %s/%s#%d at %.8s%s checked on %s: gate %s — %s; repository %s\n", c.ID, c.Owner, c.Repo, c.Number,
c.Commit, judged, orSomewhere(c.On), strings.ToUpper(c.Verdict), c.Summary, repo)
if checkEvents == nil {
return
}
+78
View File
@@ -210,3 +210,81 @@ func TestABuildOffTheTrunkIsNeverPublished(t *testing.T) {
t.Errorf("a build seat that said nothing was refused: %v", err)
}
}
// **A verdict names the trunk commit it judged** (novox/hq ADR 0323 §3, issue 341): the controller's `checked`
// carries the head and the trunk commit the build seat merged it into, the commits of the repositories read beside
// it, how long it took, and the kept delivery plan records the trunk commit with its other inputs. A group's
// composed check names, per member, the trunk commit that head was merged into (§6).
func TestAVerdictNamesTheTrunkCommitItJudged(t *testing.T) {
plan := &link.ChangePlan{Repository: "novox/mesh-host", Base: "main", Head: "dfb1415aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
Summary: "builds mesh-host"}
asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-host", Number: 66, Base: "main", Plan: plan}
c := checkedOf(link.BuildResult{ID: "b", Ref: "dfb1415aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Checked: asked,
Check: &link.CheckOutcome{Verdict: "pass", Took: "4m12s", Head: "dfb1415aaaaa", Trunk: "d28ca95bbbbb",
Gate: &link.CheckLayer{Verdict: "pass"}, RepoCheck: &link.CheckLayer{Verdict: "pass"},
Beside: map[string]string{"mesh-catalog": "0123456789ab", "mesh-lab": "ba9876543210"}}})
if c.Trunk != "d28ca95bbbbb" || c.Commit != "dfb1415aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" {
t.Fatalf("the verdict names head %s and trunk %q", c.Commit, c.Trunk)
}
if c.Beside["mesh-catalog"] != "0123456789ab" || c.Beside["mesh-lab"] != "ba9876543210" {
t.Fatalf("the verdict names the repositories read beside it as %v", c.Beside)
}
if c.Took != "4m12s" {
t.Fatalf("the verdict says it took %q", c.Took)
}
if c.Plan == nil || c.Plan.Trunk != "d28ca95bbbbb" || c.Plan.Head != plan.Head {
t.Fatalf("the kept plan records %+v", c.Plan)
}
if plan.Trunk != "" {
t.Fatal("the plan the controller asked with was changed: the verdict's is a copy")
}
group := &link.CheckRequest{Owner: "novox", Repo: "mesh-controller", Number: 7, Base: "main", Group: "feat/x",
Members: []link.GroupMember{{Owner: "novox", Repo: "mesh-host", Number: 8, Ref: "cccccccccccc0000", Base: "main"}}}
c = checkedOf(link.BuildResult{ID: "g", Ref: "aaaaaaaaaaaa0000", Checked: group, Check: &link.CheckOutcome{
Verdict: "pass", Head: "aaaaaaaaaaaa", Trunk: "111111111111", Gate: &link.CheckLayer{Verdict: "pass"},
Members: []link.CheckedMember{{Owner: "novox", Repo: "mesh-host", Commit: "cccccccccccc", Trunk: "222222222222"}}}})
if len(c.Members) != 2 || c.Members[0].Trunk != "111111111111" || c.Members[1].Trunk != "222222222222" ||
c.Members[1].Commit != "cccccccccccc0000" {
t.Fatalf("the composed check names its members %+v", c.Members)
}
}
// **The merge made for a check is never registered** (novox/hq ADR 0323 §2): a build of it — a commit on no branch
// of the forge — is refused, naming the trunk. The rule of ADR 0238 §1 stands unchanged; this holds it for the
// merge commit a check makes.
func TestABuildOfACheckMergeIsRefusedNamingTheTrunk(t *testing.T) {
merge := link.BuildResult{ID: "b", Commit: "4b5c6d7e8f90aabbccddeeff0011223344556677", Trunk: "main"}
err := publishable(merge, "")
if !errors.Is(err, errOffTheTrunk) || !strings.Contains(err.Error(), "not on main") {
t.Fatalf("a build of a check's merge was answered %v", err)
}
}
// **A verdict no build judges names its trunk commit too** (novox/hq ADR 0323 §3): a change touching no module of a
// repository with no merge-check.sh (novox/mesh-tools-go) is answered by the controller itself. Naming no trunk, the
// merge tool refused it, asked again, got the same verdict, and refused for ever.
func TestADirectVerdictNamesTheTrunkTheForgeAnnounced(t *testing.T) {
p := link.PullUpdated{Owner: "novox", Repo: "mesh-tools-go", Number: 3, Base: "main", Commit: "1a2b3c4d5e6f7a8b9c0d",
BaseSHA: "7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b"}
c := directVerdict(p, link.ChangePlan{Repository: "novox/mesh-tools-go", Base: "main", Head: p.Commit})
if c.Trunk != "7a8b9c0d1e2f" || c.Plan == nil || c.Plan.Trunk != "7a8b9c0d1e2f" {
t.Fatalf("the direct verdict names trunk %q, its plan %+v", c.Trunk, c.Plan)
}
if c.Verdict != "pass" || c.Commit != p.Commit {
t.Fatalf("the direct verdict is %+v", c)
}
if c := directVerdict(link.PullUpdated{Owner: "novox", Repo: "x", Commit: "abc"}, link.ChangePlan{}); c.Trunk != "" {
t.Fatalf("a forge that said no base head gave the trunk %q", c.Trunk)
}
}
// A head checked again through mesh-delivery (`delivery check --members`, a recheck) reaches the same direct verdict:
// it names the base head the forge last announced, kept on the member, or the merge tool would refuse it for ever.
func TestARecheckedHeadsDirectVerdictNamesItsTrunk(t *testing.T) {
m := orderMember{ID: "novox/mesh-tools-go@1a2b3c4d5e6f", Repository: "novox/mesh-tools-go", Base: "main",
Head: "1a2b3c4d5e6f7a8b9c0d", Number: 3, BaseSHA: "7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b"}
if c := directVerdict(m.pull(), link.ChangePlan{}); c.Trunk != "7a8b9c0d1e2f" {
t.Fatalf("a rechecked head's direct verdict names trunk %q", c.Trunk)
}
}
+215
View File
@@ -0,0 +1,215 @@
package main
import (
"context"
"fmt"
"sort"
"time"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
)
// **No send but a person's `push <node>` and a gate's own put-back moves the mesh's own core to a build older
// than the one its machine was last sent** (novox/hq issue 201). Every send made through sendToEach keeps it — a
// plan's, a release's, a healer's, a rotation's, and the bus step too, which is a person's act but composes under
// sendKeeps like the rest. On 2026-10-02 two merges a minute apart made two plans;
// the controller's plan rolled the new controller, which widened its seat's row, and the node-engine's plan then
// sent the control node the controller build before it. The older binary read the row it could not serve and the
// mesh lost its voice for ten minutes — the thing that would have corrected it was the thing that was down. ADR
// 0185 took the outage out of it (a controller behind its row serves what it can); the race is closed here,
// without needing its trigger: whatever made the mesh's record older, the send keeps the build.
//
// **Kept, not refused.** The send composes the controller or the node-engine at the build the machine was last
// sent, as a recorded module is kept (ADR 0245), and carries everything else: no other module, secret, healer or
// walk waits on it, and nothing is retried. The kept move is no move for a gate either: ungatedIn and gatedSend
// leave it out (withoutKept), so a gate judges the other moves of its send. A plan whose own build is the older one finds its gate superseded by
// the build kept there (issue 352) and ends. The mesh's record stays older, and a warning condition says so until
// a send carries the record — a person's `push <node>` accepts the older build — or the record moves forward: a
// newer merge, or `rebuild mesh-controller`.
//
// Older is by the builds' registration order — when each was asked (novox/hq issue 219) — never by the commits'
// dates. So a retry or a rebuild of an old commit, asked anew, counts as newer: it is registered as the module's
// current build by the same order, and the guard does not stop that move. A build that failed its gate is never
// ahead: the machine was sent it and the record was put back before it (gateFailed), and if the put-back's own
// send was refused, the next send completes it rather than keeping the failed build there. A gate's own put-back
// carries its modules back whatever the order (sendScope.puttingBack); the controller's is held to the store's
// schema already (issue 352).
// kindCoreDowngrade is a core module kept at the build its machine was last sent, over an older record.
const kindCoreDowngrade = "core-downgrade-kept"
// coreDowngradeSource is the condition's source: the sends, which raise and clear it.
const coreDowngradeSource = "sends"
// downgradeGuarded is the core components a send may not move backwards: the controller, which reads the
// store and its seat's row, and the node-engine, which reads what the controller declares.
var downgradeGuarded = []string{lease.ComponentController, lease.ComponentEngine}
// askedBuild is what the order needs of one build of a module: when it was asked, and whether it failed its gate.
type askedBuild struct {
asked time.Time
failedGate bool
}
// buildAsked is the newest build of a module from a commit that worked — when it was asked (or recorded, when
// the request time is not known), and whether it failed its gate — and false when the records hold none. A
// variable, so a test reads its own order.
var buildAsked = func(ctx context.Context, inv *inventory.Inventory, module, commit string) (askedBuild, bool, error) {
builds, err := inv.Builds(ctx, module, askedLookBack)
if err != nil {
return askedBuild{}, false, err
}
for _, b := range builds {
if b.Worked() && sameCommit(b.Commit, commit) {
failed, err := inv.GateFailed(ctx, b.ID)
if err != nil {
return askedBuild{}, false, err
}
return askedBuild{asked: b.AskedOrAt(), failedGate: failed}, true, nil
}
}
return askedBuild{}, false, nil
}
// askedLookBack is how many of a module's newest builds are read for when a build was asked.
const askedLookBack = 50
// coreDowngrade is one core module a machine was last sent a build of that is newer than the mesh's record.
type coreDowngrade struct {
module, node, component string
// sent is the build the machine was last sent, record the one the mesh holds.
sent, record string
sentAsked, recordAsked time.Time
}
func (d coreDowngrade) String() string {
return fmt.Sprintf("%s was last sent %s %s, and the mesh holds the older %s (asked %s, before %s): a send there "+
"keeps %s; `push %s` sends the older build, and a newer merge or `rebuild %s` moves the record forward "+
"(novox/hq issue 201)", d.node, d.module, short(d.sent), short(d.record),
d.recordAsked.UTC().Format(time.RFC3339), d.sentAsked.UTC().Format(time.RFC3339), short(d.sent), d.node,
d.module)
}
// coreDowngrades is, for one machine, each guarded core module whose record is older than the build the machine
// was last sent, leaving out what a gate's own put-back puts back and a sent build that failed its gate. A move
// whose either build the records do not hold is not proved older and is left to the gate that judges it. Pure
// but for asked.
func coreDowngrades(node string, sent map[string]string, record map[string]string, scope sendScope,
asked func(module, commit string) (askedBuild, bool, error)) ([]coreDowngrade, error) {
var out []coreDowngrade
for module, was := range sent {
component := coreComponent(module)
now := record[module]
if !guarded(component) || was == "" || now == "" || sameCommit(was, now) {
continue
}
if scope.puttingBack && scope.modules[module] {
continue
}
from, known, err := asked(module, was)
if err != nil {
return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", module, short(was), err)
}
if !known || from.failedGate {
continue
}
to, known, err := asked(module, now)
if err != nil {
return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", module, short(now), err)
}
if known && to.asked.Before(from.asked) {
out = append(out, coreDowngrade{module: module, node: node, component: component, sent: was, record: now,
sentAsked: from.asked, recordAsked: to.asked})
}
}
sort.Slice(out, func(i, j int) bool { return out[i].module < out[j].module })
return out, nil
}
func guarded(component string) bool {
for _, c := range downgradeGuarded {
if c == component {
return true
}
}
return false
}
// coreKept is, for a send that keeps (sendKeeps), each guarded core module of a machine to keep at the build it
// was last sent, module → commit, each said and raised as a condition.
func coreKept(ctx context.Context, inv *inventory.Inventory, node string, sent map[string]string,
current map[string]inventory.CurrentBuild) (map[string]string, error) {
record := map[string]string{}
for m, c := range current {
record[m] = c.Commit
}
found, err := coreDowngrades(node, sent, record, scopeOf(ctx), func(module, commit string) (askedBuild, bool, error) {
return buildAsked(ctx, inv, module, commit)
})
if err != nil || len(found) == 0 {
return nil, err
}
out := map[string]string{}
for _, d := range found {
out[d.module] = d.sent
sayCoreDowngrade(ctx, d)
}
return out, nil
}
// coreDowngradeObservation is a kept core build as a condition: `core.<component>.<machine>.<kind>`.
func coreDowngradeObservation(component, node, summary string) conditions.Observation {
return conditions.Observation{Scope: conditions.ScopeCore, ID: component + "." + node, Kind: kindCoreDowngrade,
Token: kindCoreDowngrade, Machine: node, Severity: conditions.Warning, Source: coreDowngradeSource,
Summary: summary}
}
func sayCoreDowngrade(ctx context.Context, d coreDowngrade) {
fmt.Printf("KEPT: %s\n", d)
keeper := conditionsFrom
if keeper == nil {
return
}
if _, err := keeper.Observe(ctx, coreDowngradeObservation(d.component, d.node, d.String())); err != nil {
fmt.Printf("the condition for %s kept on %s could not be raised: %v\n", d.module, d.node, err)
}
}
// settleCoreDowngrades clears, after a send to a machine is away, the condition of each guarded core module the
// send carried at the mesh's record — a person's push that accepted the older build, or a send after the record
// moved forward — or did not carry at all. A send that kept the newer build leaves it open.
func settleCoreDowngrades(ctx context.Context, inv *inventory.Inventory, node string, carried map[string]string) {
keeper := conditionsFrom
if keeper == nil || carried == nil {
return
}
var open []string
for _, c := range downgradeGuarded {
if _, found, err := keeper.Get(ctx, coreDowngradeObservation(c, node, "").Key()); err == nil && found {
open = append(open, c)
}
}
if len(open) == 0 {
return
}
current, err := inv.CurrentBuilds(ctx)
if err != nil {
fmt.Printf("whether %s still keeps a core build over the mesh's record cannot be read: %v\n", node, err)
return
}
for module, now := range current {
component := coreComponent(module)
if !guarded(component) {
continue
}
if was, has := carried[module]; has && !sameCommit(was, now.Commit) {
continue
}
key := coreDowngradeObservation(component, node, "").Key()
if _, err := keeper.Clear(ctx, key, node+" was sent the "+component+" the mesh holds"); err != nil {
fmt.Printf("the condition %s could not be cleared: %v\n", key, err)
}
}
}
+5 -2
View File
@@ -580,6 +580,9 @@ type orderMember struct {
CloneURL string `json:"clone_url,omitempty"`
// After are the repositories its pull request says it goes after (`after: <repository>`).
After []string `json:"after,omitempty"`
// BaseSHA is the base branch's head the forge last announced the head with (novox/hq ADR 0323): the trunk commit
// a verdict no build judges names when the head is checked again through mesh-delivery.
BaseSHA string `json:"base_sha,omitempty"`
}
// pull is the member as the forge announced it.
@@ -589,7 +592,7 @@ func (m orderMember) pull() link.PullUpdated {
if base == "" {
base = "main"
}
return link.PullUpdated{Owner: owner, Repo: repo, Number: m.Number, Base: base, Commit: m.Head,
return link.PullUpdated{Owner: owner, Repo: repo, Number: m.Number, Base: base, Commit: m.Head, BaseSHA: m.BaseSHA,
CloneURL: m.CloneURL, Paths: m.Paths, PathsTruncated: m.PathsTruncated, Removed: m.Removed,
ModuleDirs: m.ModuleDirs, ModuleDirsSaid: m.ModuleDirsSaid}
}
@@ -1185,7 +1188,7 @@ func askGroupCheck(ctx context.Context, open *stores, group string, members []or
return "", fmt.Errorf("%s cannot be cloned for the group's check: %w", m.ID, err)
}
request.Check.Members = append(request.Check.Members, link.GroupMember{Owner: mp.Owner, Repo: mp.Repo,
Number: mp.Number, Repository: url, Ref: mp.Commit, Paths: mp.Paths})
Number: mp.Number, Repository: url, Ref: mp.Commit, Paths: mp.Paths, Base: mp.Base})
}
seat := buildSeatHeld(ctx)
ask, err := askOverOn(seat)
+2 -2
View File
@@ -1039,7 +1039,7 @@ func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module str
}
return
}
sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}}), open, g.Machines)
sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}, puttingBack: true}), open, g.Machines)
if err != nil {
notBack(fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+
"sends it", short(previous.Commit), strings.Join(g.Machines, ", "), err, g.Machines[0]))
@@ -1132,7 +1132,7 @@ func sendRollbacks(ctx context.Context, open *stores, p *inventory.Plan, b *roll
}
inv := open.inventory
sort.Strings(b.machines)
sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules}), open, b.machines)
sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules, puttingBack: true}), open, b.machines)
var back []string
for _, r := range b.pending {
if err != nil {
+259
View File
@@ -0,0 +1,259 @@
package main
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
)
// novox/hq issue 201: on 2026-10-02 a plan sent the control node a controller older than the one running there,
// whose seat row the newer one had already widened; the older binary refused the row and the mesh went quiet for
// ten minutes. No send but a person's push and a gate's own put-back moves the controller or the node-engine back
// behind the build its machine was last sent: the send keeps it there, and carries the rest.
// askedIn answers a build from a fixed table: module@commit → when it was asked; failed names those that failed
// their gate.
func askedIn(table map[string]time.Time, failed ...string) func(module, commit string) (askedBuild, bool, error) {
return func(module, commit string) (askedBuild, bool, error) {
at, known := table[module+"@"+commit]
b := askedBuild{asked: at}
for _, f := range failed {
if f == module+"@"+commit {
b.failedGate = true
}
}
return b, known, nil
}
}
func TestACoreModuleIsKeptOverAnOlderRecord(t *testing.T) {
old, now := t0, t0.Add(time.Minute)
table := map[string]time.Time{
"mesh-controller@c1": old, "mesh-controller@c2": now,
"mesh-host@h1": old, "mesh-host@h2": now,
"app@a1": old, "app@a2": now,
}
sent := map[string]string{"mesh-controller": "c2", "mesh-host": "h2", "app": "a2"}
record := map[string]string{"mesh-controller": "c1", "mesh-host": "h1", "app": "a1"}
found, err := coreDowngrades("anchor", sent, record, sendScope{judged: map[string]bool{"anchor": true}}, askedIn(table))
if err != nil {
t.Fatal(err)
}
if len(found) != 2 || found[0].module != "mesh-controller" || found[0].component != lease.ComponentController ||
found[0].sent != "c2" || found[1].module != "mesh-host" || found[1].component != lease.ComponentEngine {
t.Fatalf("the controller and the node-engine over an older record are kept, and nothing else: %+v", found)
}
if said := found[0].String(); !strings.Contains(said, "anchor was last sent mesh-controller c2") ||
!strings.Contains(said, "`push anchor`") || !strings.Contains(said, "`rebuild mesh-controller`") {
t.Fatalf("it says the machine, the build kept and the two ways out: %s", said)
}
// The record forward, the same build, or a build the records do not hold: nothing kept.
for _, c := range [][2]string{{"c1", "c2"}, {"c2", "c2"}, {"c2", "c9"}, {"c9", "c1"}} {
found, err := coreDowngrades("anchor", map[string]string{"mesh-controller": c[0]},
map[string]string{"mesh-controller": c[1]}, sendScope{}, askedIn(table))
if err != nil || len(found) != 0 {
t.Fatalf("sent %s, record %s: no downgrade proved, nothing kept: %+v %v", c[0], c[1], found, err)
}
}
// A sent build that failed its gate is never ahead: the record put back before it is sent.
if found, _ := coreDowngrades("anchor", sent, record, sendScope{}, askedIn(table, "mesh-controller@c2")); len(found) != 1 ||
found[0].module != "mesh-host" {
t.Fatalf("a controller that failed its gate was kept over the build put back: %+v", found)
}
// A gate's own put-back carries its own modules back, and nothing else.
putBack := sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true}
if found, _ := coreDowngrades("anchor", sent, record, putBack, askedIn(table)); len(found) != 1 || found[0].module != "mesh-host" {
t.Fatalf("a put-back of the controller carries it back, and not the node-engine: %+v", found)
}
if found, _ := coreDowngrades("anchor", sent, record, sendScope{modules: map[string]bool{"mesh-controller": true}},
askedIn(table)); len(found) != 2 {
t.Fatalf("a send for the controller that is no put-back carries it back: %+v", found)
}
}
// core201 is a mesh whose control node, anchor, was sent the newer controller c2 while the mesh's record of the
// controller is the build before it, c1, which passed its gate when it rolled out.
func core201(t *testing.T) *stores {
t.Helper()
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
record := func(commit string, asked time.Time) inventory.Build {
manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: "1"})
b := inventory.Build{ID: "build-mesh-controller-" + commit, Module: "mesh-controller", Commit: commit,
Repository: "novox/mesh-controller", Manifest: manifest, Asked: asked, At: asked,
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + commit + "0000000000000000"}}}
if err := inv.RecordBuild(ctx, b); err != nil {
t.Fatal(err)
}
return b
}
before := record("c1aaaaaaaaaa", old)
if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: before.ID, Module: "mesh-controller", Commit: before.Commit,
Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil {
t.Fatal(err)
}
record("c2bbbbbbbbbb", now)
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{
Repository: "novox/mesh-controller", BuiltFrom: "c2bbbbbbbbbb", Head: "c2bbbbbbbbbb", Asked: now}); err != nil {
t.Fatal(err)
}
if _, err := inv.Assign(ctx, "anchor", "mesh-controller"); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}); err != nil {
t.Fatal(err)
}
if err := inv.RestoreModule(ctx, before); err != nil {
t.Fatal(err)
}
wasMoves := machineMoves
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
modules, err := open.inventory.Assigned(ctx, node)
if err != nil {
return nil, err
}
sent, known, err := open.inventory.SentBuilds(ctx, node)
if err != nil {
return nil, err
}
return f.moves(node, modules, sent, known, all), nil
}
t.Cleanup(func() { machineMoves = wasMoves })
return open
}
// TestReplay201 replays the race on a store: the control node runs the newer controller and the mesh's record is
// the build before it. A plan's send there — the node-engine's plan of 2026-10-02 — goes, keeps the newer
// controller, and says so as a condition; the gate's own put-back carries it back; a send that carries the record,
// as a person's push does, clears the condition.
func TestReplay201(t *testing.T) {
open := core201(t)
ctx := t.Context()
k, _ := withConditionsInMemory(t)
judged := withScope(ctx, sendScope{judged: map[string]bool{"anchor": true}})
if kept, err := ungatedIn(judged, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 {
t.Fatalf("a plan's send to the machine is made, the rest of it with it: %v %v", kept, err)
}
keeps, err := recordedKept(keepingRecorded(judged), open, "anchor")
if err != nil || keeps["mesh-controller"] != "c2bbbbbbbbbb" {
t.Fatalf("the send keeps the newer controller there: %v %v", keeps, err)
}
shelf := map[string]catalogue.Manifest{"mesh-controller": {Module: "mesh-controller", Version: "old"}}
if _, err := keepRecorded(keepingRecorded(judged), open, "anchor", shelf); err != nil || shelf["mesh-controller"].Version != "1" {
t.Fatalf("the send composes the kept build's manifest: %+v %v", shelf, err)
}
key := coreDowngradeObservation(lease.ComponentController, "anchor", "").Key()
if c, found, err := k.Get(ctx, key); err != nil || !found || !strings.Contains(c.Summary, "c2bbbbbb") {
t.Fatalf("the condition %s is raised, saying the build kept: %+v %v %v", key, c, found, err)
}
putBack := keepingRecorded(withScope(ctx, sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true}))
if keeps, err := recordedKept(putBack, open, "anchor"); err != nil || len(keeps) != 0 {
t.Fatalf("the gate's own put-back of the controller carries it back: %v %v", keeps, err)
}
settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"})
if _, found, _ := k.Get(ctx, key); !found {
t.Fatal("a send that kept the newer controller cleared the condition")
}
settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c1aaaaaaaaaa"})
if _, found, _ := k.Get(ctx, key); found {
t.Fatal("a send carrying the mesh's record — a person's push — leaves the condition open")
}
}
// The review of #273: a gate failed the newer controller, put the record back, and the put-back's own send was
// refused. The next automatic send — a healer's, which judges nothing — completes the put-back: a build that
// failed its gate is never kept.
func TestARollbackWhoseSendWasRefusedIsCompletedByTheNextSend(t *testing.T) {
open := core201(t)
ctx := t.Context()
k, _ := withConditionsInMemory(t)
if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: "build-mesh-controller-c2bbbbbbbbbb",
Module: "mesh-controller", Commit: "c2bbbbbbbbbb", Previous: "c1aaaaaaaaaa", Machines: []string{"anchor"},
Verdict: inventory.GateFailed, Rollback: inventory.NotRolledBack}); err != nil {
t.Fatal(err)
}
healer := withScope(ctx, sendScope{})
if kept, err := ungatedIn(healer, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 {
t.Fatalf("the healer's send is made: %v %v", kept, err)
}
if keeps, err := recordedKept(keepingRecorded(healer), open, "anchor"); err != nil || len(keeps) != 0 {
t.Fatalf("the failed controller was kept over the build put back: %v %v", keeps, err)
}
if open, _ := k.Open(ctx); len(open) != 0 {
t.Fatalf("a completed put-back raised a condition: %+v", open)
}
}
// Review 91 of #273: a gated send to a machine whose controller is kept over an older record leaves the kept
// controller out of what its gate judges. Judged as moving back to the record, the gate would read the send's own
// kept controller as another build sent there after it (issue 352) and be superseded, judging nothing.
func TestAGateJudgesTheOtherMovesBesideAKeptCore(t *testing.T) {
open := core201(t)
ctx := t.Context()
withConditionsInMemory(t)
inv := open.inventory
old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
for _, b := range []struct {
commit string
at time.Time
}{{"a1aaaaaaaaaa", old}, {"a2bbbbbbbbbb", now}} {
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-" + b.commit, Module: "app", Commit: b.commit,
Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: b.at, At: b.at,
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-" + b.commit}}}); err != nil {
t.Fatal(err)
}
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: b.commit, Head: b.commit,
Asked: b.at}); err != nil {
t.Fatal(err)
}
}
if _, err := inv.Assign(ctx, "anchor", "app"); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-2",
map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a1aaaaaaaaaa"}); err != nil {
t.Fatal(err)
}
// The send as it is made: the controller kept at c2, the app moved to the record.
wasSend := sendRollout
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
return names, open.inventory.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-3",
map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a2bbbbbbbbbb"})
}
t.Cleanup(func() { sendRollout = wasSend })
moves, sent, _, err := gatedSend(ctx, open, "anchor", nil)
if err != nil || len(sent) != 1 {
t.Fatalf("the gated send is made: %v %v", sent, err)
}
if len(moves) != 1 || moves[0].Module != "app" || moves[0].To != "a2bbbbbbbbbb" {
t.Fatalf("the gate judges the app's move and not the kept controller: %+v", moves)
}
builds, _, err := inv.SentBuilds(ctx, "anchor")
if err != nil {
t.Fatal(err)
}
g := &inventory.PlanGate{Carried: moves}
f := gateFacts{commits: judgedCommits(g, nil), sentBuilds: map[string]map[string]string{"anchor": builds}}
for _, m := range []string{"app", "mesh-controller"} {
if why, superseded := f.supersededOn(m, "anchor"); superseded {
t.Fatalf("the gate is superseded by its own send: %s", why)
}
}
}
@@ -0,0 +1,120 @@
package main
import (
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// An own secret issued outside the mesh that nobody gave is never made (novox/hq issue 401, ADR 0228). Seen on
// 2026-10-10: the send made a random value for the mounts module's smb-credentials and placed it, and the module read
// it as a credential given and wrong. The send now makes none, places none, and places a value only once a person
// gave one — and a value the mesh made for it before is not placed either.
// anAssistant is a module with a secret the mesh makes and one only an outside party issues.
func anAssistant() catalogue.Manifest {
return catalogue.Manifest{Module: "assistant", Version: "1",
OwnSecrets: catalogue.OwnSecrets{
"password": {Path: "/var/lib/mesh/assistant/password", Taken: catalogue.TakenAtStart},
"vendor-key": {Path: "/var/lib/mesh/assistant/vendor-key", Taken: catalogue.TakenAtStart, IssuedBy: catalogue.IssuedOutside},
},
Resources: []map[string]any{
{"id": "state", "type": "directory", "path": "/var/lib/mesh/assistant", "mode": "0700"},
}}
}
// sentTo composes what a push sends to a machine, on the send path.
func sentTo(t *testing.T, open *stores, node string) []map[string]any {
t.Helper()
plan, settings, err := planFor(t.Context(), open, node)
if err != nil {
t.Fatal(err)
}
declared, err := declarationFor(t.Context(), open, node, plan, settings)
if err != nil {
t.Fatalf("the send to %s did not compose: %v", node, err)
}
return declared.Resources
}
func placedAt(resources []map[string]any, path string) map[string]any {
for _, r := range resources {
if r["path"] == path && r["sealed"] != nil {
return r
}
}
return nil
}
func TestTheSendMakesNoValueForAnOutsideSecretNobodyGave(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, anAssistant())
if _, err := assign(ctx, open, "laptop", "assistant"); err != nil {
t.Fatal(err)
}
sent := sentTo(t, open, "laptop")
if placedAt(sent, "/var/lib/mesh/assistant/password") == nil {
t.Fatal("the secret the mesh makes was not placed")
}
if got := placedAt(sent, "/var/lib/mesh/assistant/vendor-key"); got != nil {
t.Fatalf("a secret nobody gave was placed: %v", got)
}
if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "assistant", "vendor-key"); err != nil || held {
t.Fatalf("the send made a value for a secret issued outside the mesh (held %v, %v)", held, err)
}
// A value the mesh made before this rule is not one a person gave: it is not placed.
if _, err := open.inventory.SecretForModule(ctx, "laptop", "assistant", "vendor-key"); err != nil {
t.Fatal(err)
}
if got := placedAt(sentTo(t, open, "laptop"), "/var/lib/mesh/assistant/vendor-key"); got != nil {
t.Fatalf("a value the mesh made for a secret issued outside was placed: %v", got)
}
// Given, it is placed.
if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "assistant", "vendor-key", "the vendor's key"); err != nil {
t.Fatal(err)
}
given, ok, err := open.inventory.GivenOwnSecret(ctx, "laptop", "assistant", "vendor-key")
if err != nil || !ok || !given.Current {
t.Fatalf("the value given is not read as given: %+v %v %v", given, ok, err)
}
placed := placedAt(sentTo(t, open, "laptop"), "/var/lib/mesh/assistant/vendor-key")
if placed == nil || placed["sealed"] != given.Sealed {
t.Fatalf("the value given is placed as %v", placed)
}
}
// The read paths answer as the send does: a question about the machine composes without the secret nobody gave, and
// asking writes nothing.
func TestAQuestionComposesWithoutAnOutsideSecretNobodyGave(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, anAssistant())
if _, err := assign(ctx, open, "laptop", "assistant"); err != nil {
t.Fatal(err)
}
pushedBy(t, open, "laptop") // the mesh's own secret is made by the send
plan, settings, err := planFor(ctx, open, "laptop")
if err != nil {
t.Fatal(err)
}
gens, err := generators(ctx, open)
if err != nil {
t.Fatal(err)
}
for _, choosing := range []Choosing{Reading, Foreseeing} {
got, err := declarationWith(ctx, open, "laptop", plan, settings, gens, choosing)
if err != nil {
t.Fatalf("%v: %v", choosing, err)
}
if placedAt(got.Resources, "/var/lib/mesh/assistant/vendor-key") != nil || len(got.foreseen) != 0 {
t.Fatalf("%v: a secret nobody gave is composed (foreseen %v)", choosing, got.foreseen)
}
}
if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "assistant", "vendor-key"); err != nil || held {
t.Fatalf("a question made a value (held %v, %v)", held, err)
}
}
+21
View File
@@ -81,6 +81,13 @@ var plainWordings = map[string]func(conditions.Observation) words{
"It usually catches up by itself.", m, m),
Resolved: m + " applied the change"}
}),
kindApplyFailing: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: m + " is not applying its update",
Explanation: fmt.Sprintf("%s has refused or only partly applied what the mesh sent it, again and again. "+
"The details say what fails.", conditions.Capital(m)),
Resolved: m + " applies what it is sent again"}
}),
"tools-silent": worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: "Tools on " + m + " are not answering",
@@ -175,6 +182,20 @@ var plainWordings = map[string]func(conditions.Observation) words{
"register it, so it was not made.", module, m),
Resolved: "Resolved: " + module + " on " + m + " is settled"}
}),
// The controller or the node-engine kept at the build a machine was last sent, over an older record (novox/hq
// issue 201).
kindCoreDowngrade: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
component := idPart(o, 0)
if o.Scope != conditions.ScopeCore || !guarded(component) {
component = "core component"
}
return words{Headline: m + " keeps a newer " + component + " than the mesh holds",
Needs: fmt.Sprintf("push %s to take the older build, or a newer merge or rebuild moves the record forward.", m),
Explanation: fmt.Sprintf("The mesh's record of its own %s is older than the build %s was last sent, so "+
"the mesh's sends keep the newer one there rather than put it back.", component, m),
Resolved: "Resolved: " + m + " runs the " + component + " the mesh holds"}
}),
kindBindingKept: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: "A module's data source is held on " + m,
+33 -5
View File
@@ -590,6 +590,13 @@ func busCredentialIssued(ctx context.Context, inv *inventory.Inventory, node, mo
return nil
}
// givenUnderAnOldKey is the refusal for a value a person gave — a family's member, or an own secret issued outside
// the mesh — sealed to a key the machine no longer holds: the mesh cannot make another, so it is given again.
func givenUnderAnOldKey(module, node, name string) error {
return fmt.Errorf("%s on %s holds %q, which was given to the mesh rather than made by it, and %s has "+
"since generated a new sealing key. The mesh cannot make another; give it again", module, node, name, node)
}
// renderingFor is everything a node's declaration is composed with, and the node's record.
func renderingFor(ctx context.Context, open *stores, node string,
plan catalogue.Resolution, settings catalogue.SettingsBy,
@@ -708,10 +715,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
}
if !g.Current {
if choosing == Allocating {
return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf(
"%s on %s holds %q, which was given to the mesh rather than made by it, and %s has "+
"since generated a new sealing key. The mesh cannot make another; give it again",
m.Module, node, g.Name, node)
return catalogue.Rendering{}, inventory.Node{}, givenUnderAnOldKey(m.Module, node, g.Name)
}
continue
}
@@ -721,7 +725,31 @@ func renderingFor(ctx context.Context, open *stores, node string,
needed[m.Module][g.Name] = g.Sealed
}
}
for name := range m.OwnSecrets.Plain() {
for name, own := range m.OwnSecrets.Plain() {
// **An own secret issued outside the mesh is never made** (novox/hq issue 401, ADR 0228; ADR 0283 for
// a family): only a value a person gave is placed. One not given is nothing — no file, and an empty
// value where a file of the module names it — so the module can tell it was not given, where a value
// the mesh made looked given and wrong. A value the mesh made for it before is not one either.
if own.IssuedBy == catalogue.IssuedOutside {
g, given, err := inv.GivenOwnSecret(ctx, node, m.Module, name)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
if !given {
continue
}
if !g.Current {
if choosing == Allocating {
return catalogue.Rendering{}, inventory.Node{}, givenUnderAnOldKey(m.Module, node, name)
}
continue
}
if needed[m.Module] == nil {
needed[m.Module] = map[string]string{}
}
needed[m.Module][name] = g.Sealed
continue
}
// Minted on the send path and only read on every other. Making one is an insert, and
// a question that writes is a question that can block against the machine it is about.
var sealed string
+26
View File
@@ -0,0 +1,26 @@
package main
import (
"fmt"
"regexp"
)
// planID is every id the mesh gives a plan: a merge's walk or batch (plan-<nanoseconds>) or a release
// (release-<nanoseconds>).
var planID = regexp.MustCompile(`^(plan|release)-[0-9]+$`)
// planIDArguments are the arguments of plans that name a plan. wait is among them for when it is declared
// (novox/hq issue 229); an argument not declared is refused before this is asked.
var planIDArguments = []string{"id", "wait", "stop", "close", "retry", "go"}
// plainPlanIDs refuses a plans call naming a plan by anything but a plan's id (novox/hq issue 561): checked
// once, where the call becomes a command line, so nothing else a caller sent reaches a command or the store.
func plainPlanIDs(given map[string]string) error {
for _, arg := range planIDArguments {
if id, named := given[arg]; named && !planID.MatchString(id) {
return fmt.Errorf("plans %s: %q is not a plan's id (plan-<digits> or release-<digits>, as `plans` "+
"lists them); nothing was done", arg, id)
}
}
return nil
}
+37
View File
@@ -0,0 +1,37 @@
package main
import (
"strings"
"testing"
)
// **A plan's id is checked once, where a call becomes a command line** (novox/hq issue 561): every argument of
// plans that names a plan is refused unless it is an id the mesh gives a plan, so a quote never reaches a
// command, nor the store behind it.
func TestAPlanIdThatIsNoPlansIdIsRefusedBeforeACommandStarts(t *testing.T) {
for _, arg := range []string{"id", "stop", "close", "retry", "go"} {
for _, bad := range []string{"x' or '1'='1", "plan-1'", "plan-", "plan-1 2", "batch-1", "PLAN-1"} {
args := map[string]any{arg: bad}
if arg == "stop" || arg == "close" || arg == "go" {
args["why"] = "a test"
}
argv, err := argvFor("plans", args)
if err == nil {
t.Errorf("plans %s %q was composed: %v", arg, bad, argv)
continue
}
if !strings.Contains(err.Error(), "is not a plan's id") {
t.Errorf("plans %s %q was refused for another reason: %v", arg, bad, err)
}
}
for _, good := range []string{"plan-1791813626034291775", "release-1791811938267404408"} {
args := map[string]any{arg: good}
if arg == "stop" || arg == "close" || arg == "go" {
args["why"] = "a test"
}
if _, err := argvFor("plans", args); err != nil {
t.Errorf("plans %s %q was refused: %v", arg, good, err)
}
}
}
}
+158
View File
@@ -0,0 +1,158 @@
package main
import (
"context"
"encoding/json"
"fmt"
"os"
"strconv"
"strings"
"time"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// Following a plan (novox/hq issue 229): `plans wait <id>` holds its answer until the plan moves, reaches its
// next tier or ends, or until its bound passes, and says which. Before it, an agent following a rollout had
// `plans` answer once, in prose, and wrote a loop of its own round the mesh's tools to be told the plan moved.
//
// **No wait is longer than a call is answered within** (link.LongestAnswer, inside the console's thirty
// seconds): a longer one would answer "still running" and leave its caller to ask `calls` — the loop again.
// So a caller following a plan to its end asks again; each call answers the moment the plan moves.
// maxPlanWait is the longest bound a wait takes, and the bound when none is given: what a call is answered
// within, less the time a verb's command takes to start and read the store.
var maxPlanWait = link.LongestAnswer - planWaitStart
// planWaitStart is what the verb's window allows beyond the bound: starting the command and opening the stores.
// A store slow enough to push the read and the answer past it degrades as any long call does: the caller is
// answered that the call is still running, and `calls` with its id says what came of it.
const planWaitStart = 5 * time.Second
// planWaitEvery is how often a wait reads the plan again.
const planWaitEvery = time.Second
// The things a wait waits until.
var planWaitUntil = []string{"moved", "tier", "done"}
// planWaitAsked checks what a wait is asked for: until (moved when empty) and a bound (a duration, or a number
// of seconds; maxPlanWait when empty).
func planWaitAsked(until, bound string) (string, time.Duration, error) {
if until == "" {
until = "moved"
}
known := false
for _, u := range planWaitUntil {
known = known || u == until
}
if !known {
return "", 0, fmt.Errorf("a wait is until %s, not %q", strings.Join(planWaitUntil, ", "), until)
}
d := maxPlanWait
if bound = strings.TrimSpace(bound); bound != "" {
var err error
if secs, isNumber := strconv.Atoi(bound); isNumber == nil {
d = time.Duration(secs) * time.Second
} else if d, err = time.ParseDuration(bound); err != nil {
return "", 0, fmt.Errorf("the bound %q is not a duration (such as 20s) or a number of seconds", bound)
}
if d <= 0 || d > maxPlanWait {
return "", 0, fmt.Errorf("the bound is at most %s, the longest a call is answered within; %s was asked "+
"— to follow a plan longer, ask again: each call answers the moment the plan moves", maxPlanWait, d)
}
}
return until, d, nil
}
// planEnded is a plan that will not move again: done, failed or superseded. A batch being assembled is not one.
func planEnded(p inventory.Plan) bool { return !p.Open() && !p.Batch() }
// waitOnPlan reads the plan until what it waits for happens or the bound passes, and answers the plan as last
// read and what ended the wait: moved (any save of it), tier (its tier changed), done (it ended — which ends
// every wait, since nothing comes after it), or bound. Moved and tier are judged against the plan as first read.
func waitOnPlan(ctx context.Context, read func(context.Context) (inventory.Plan, error), until string,
bound, every time.Duration) (inventory.Plan, string, error) {
first, err := read(ctx)
if err != nil {
return inventory.Plan{}, "", err
}
if planEnded(first) {
return first, "done", nil
}
deadline := time.Now().Add(bound)
tick := time.NewTicker(every)
defer tick.Stop()
last := first
for {
select {
case <-ctx.Done():
return last, "", ctx.Err()
case <-tick.C:
}
if !time.Now().Before(deadline) {
return last, "bound", nil
}
p, err := read(ctx)
if err != nil {
return last, "", err
}
last = p
switch {
case planEnded(p):
return p, "done", nil
case until == "moved" && p.Revision != first.Revision:
return p, "moved", nil
case until == "tier" && p.Tier != first.Tier:
return p, "tier", nil
}
}
}
// planWaited is what `plans wait` answers: the plan, what ended the wait, and the line a person reads (`text`,
// which the verb answers as its output and not again in the data).
type planWaited struct {
// Came is what ended the wait: moved, tier, done, or bound when nothing did.
Came string `json:"came"`
Until string `json:"until"`
Waited string `json:"waited"`
Plan planStatus `json:"plan"`
// Revision is the plan's as read: a caller asking again with the same plan sees whether it moved since.
Revision int64 `json:"revision"`
Text string `json:"text"`
}
func planWaitedDoc(p inventory.Plan, st planStatus, until, came string, waited time.Duration, line string) planWaited {
said := map[string]string{
"moved": "moved", "tier": "reached another tier", "done": "ended",
"bound": "did not " + map[string]string{"moved": "move", "tier": "reach another tier", "done": "end"}[until],
}[came]
text := fmt.Sprintf("%s %s in %s (waited until %s)\n%s — %s\n", p.ID, said, waited.Round(time.Second), until,
p.ID, line)
return planWaited{Came: came, Until: until, Waited: waited.Round(time.Millisecond).String(), Plan: st,
Revision: p.Revision, Text: text}
}
// planWaitCommand is `plans wait <id>`: the plan followed until it moves, or the bound passes.
func planWaitCommand(ctx context.Context, inv *inventory.Inventory, id, untilAsked, boundAsked string) error {
until, bound, err := planWaitAsked(untilAsked, boundAsked)
if err != nil {
return err
}
began := time.Now()
p, came, err := waitOnPlan(ctx, func(ctx context.Context) (inventory.Plan, error) {
return inv.PlanByID(ctx, id)
}, until, bound, planWaitEvery)
if err != nil {
return err
}
// Said as `plans <id>` says its first line, and as `status` says the plan.
now, one := time.Now(), []inventory.Plan{p}
bounds, pause := readTierBounds(ctx, inv, now), buildSeatPause(ctx, inv, one)
line := planLineSaid(p, now, pause, bounds.of(p.Repository), func(module string) []string {
on, _ := inv.Running(ctx, module)
return on
}, waitsOf(ctx, inv, one))
doc := planWaitedDoc(p, planStatuses(one, now, pause, bounds)[0], until, came, now.Sub(began), line)
return json.NewEncoder(os.Stdout).Encode(doc)
}
+196
View File
@@ -0,0 +1,196 @@
package main
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
// A plan as the store would answer it at each read: the reads are counted, and each answers the next.
type planReads struct {
plans []inventory.Plan
n int
}
func (r *planReads) read(context.Context) (inventory.Plan, error) {
p := r.plans[min(r.n, len(r.plans)-1)]
r.n++
return p, nil
}
func followedPlan(rev int64, state string, tier int) inventory.Plan {
return inventory.Plan{ID: "plan-1", Repository: "novox/x", Commit: "abc", State: state, Tier: tier,
Tiers: [][]string{{"a"}, {"b"}}, Revision: rev}
}
// **`plans` waits for a plan, and says what ended the wait** (novox/hq issue 229): an agent following a
// rollout is told when the plan moved, reached its next tier or ended, or that the bound passed — in one
// call, without a loop of its own.
func TestAWaitForAPlanEndsWhenWhatItWaitsForHappens(t *testing.T) {
cases := []struct {
name string
until string
reads []inventory.Plan
came string
times int
}{
{"moved: any save", "moved", []inventory.Plan{
followedPlan(1, inventory.PlanBuilding, 0), followedPlan(1, inventory.PlanBuilding, 0),
followedPlan(2, inventory.PlanBuilding, 0)}, "moved", 3},
{"tier: a save inside the tier is not enough", "tier", []inventory.Plan{
followedPlan(1, inventory.PlanBuilding, 0), followedPlan(2, inventory.PlanBuilding, 0),
followedPlan(3, inventory.PlanRolling, 1)}, "tier", 3},
{"tier: a plan that ended has no next tier to wait for", "tier", []inventory.Plan{
followedPlan(1, inventory.PlanBuilding, 0), followedPlan(2, inventory.PlanFailed, 0)}, "done", 2},
{"done: a new tier is not the end", "done", []inventory.Plan{
followedPlan(1, inventory.PlanBuilding, 0), followedPlan(2, inventory.PlanRolling, 1),
followedPlan(3, inventory.PlanDone, 1)}, "done", 3},
{"done: a plan already ended answers at once", "done", []inventory.Plan{
followedPlan(4, inventory.PlanSuperseded, 1)}, "done", 1},
{"the bound passes", "moved", []inventory.Plan{followedPlan(1, inventory.PlanBuilding, 0)}, "bound", -1},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
r := &planReads{plans: c.reads}
p, came, err := waitOnPlan(context.Background(), r.read, c.until, 50*time.Millisecond, time.Millisecond)
if err != nil {
t.Fatal(err)
}
if came != c.came {
t.Fatalf("the wait ended as %q, want %q", came, c.came)
}
if c.times > 0 && r.n != c.times {
t.Errorf("read the plan %d times, want %d: it answered before or after what it waited for", r.n, c.times)
}
if want := c.reads[min(r.n, len(c.reads))-1]; p.Revision != want.Revision {
t.Errorf("answered revision %d, not the plan as it was last read (%d)", p.Revision, want.Revision)
}
})
}
}
// What a wait is asked for is checked before anything is read: what it waits until, and a bound no longer than
// a call is answered within.
func TestAWaitIsRefusedWhatItCannotDo(t *testing.T) {
for _, bad := range []struct{ until, bound string }{{"landed", ""}, {"", "1h"}, {"", "-3s"}, {"", "soon"}} {
if _, _, err := planWaitAsked(bad.until, bad.bound); err == nil {
t.Errorf("until %q, bound %q was taken", bad.until, bad.bound)
}
}
until, bound, err := planWaitAsked("", "")
if err != nil || until != "moved" || bound != maxPlanWait {
t.Fatalf("the defaults are %q %s %v, want moved and %s", until, bound, err, maxPlanWait)
}
if _, bound, _ := planWaitAsked("tier", "12"); bound != 12*time.Second {
t.Errorf("a bare number is seconds; read %s", bound)
}
}
// **The answer is data, with its prose as a field beside it** (novox/hq issue 229): the verb's `answer` carries
// the plan and what ended the wait, and its output is the line a person reads — never JSON inside a string.
func TestAWaitForAPlanAnswersAsData(t *testing.T) {
p := followedPlan(7, inventory.PlanRolling, 1)
st := planStatuses([]inventory.Plan{p}, time.Now(), pauseView{}, tierBounds{})[0]
doc := planWaitedDoc(p, st, "tier", "tier", 3*time.Second, "rolling, tier 2 of 2")
stdout, _ := json.Marshal(doc)
a := answerOf([]string{"plans", "wait", "plan-1"}, stdout, "", true)
data, ok := a.Answer.(map[string]any)
if !ok {
t.Fatalf("the answer is not data: %v", a.Answer)
}
if data["came"] != "tier" || data["revision"] != float64(7) {
t.Errorf("the data does not say what ended the wait: %v", data)
}
if plan, _ := data["plan"].(map[string]any); plan["state"] != "rolling" || plan["tier"] != float64(1) {
t.Errorf("the data does not carry the plan: %v", data["plan"])
}
if _, text := data["text"]; text {
t.Errorf("the prose is in the data too: %v", data)
}
if strings.Contains(a.Output, "{") || !strings.Contains(a.Output, "plan-1") || !strings.Contains(a.Output, "tier") {
t.Errorf("the output is not the line a person reads: %q", a.Output)
}
}
// The verb composes the wait, and asks for a window as long as its bound.
func TestThePlansVerbComposesAWait(t *testing.T) {
argv, err := argvFor("plans", map[string]any{"wait": "plan-1", "until": "done", "bound": "15s"})
if err != nil {
t.Fatal(err)
}
if got := strings.Join(argv, " "); got != "plans wait plan-1 --until done --bound 15s" {
t.Fatalf("composed %q", got)
}
if _, err := argvFor("plans", map[string]any{"until": "done"}); err == nil {
t.Error("until without wait was taken")
}
if _, err := argvFor("plans", map[string]any{"wait": "plan-1", "bound": "1h"}); err == nil {
t.Error("a bound past what a call can wait was taken")
}
}
// Live, with the store: `plans wait` run as the verb runs it answers when the plan reaches its next tier, as
// data, and says the bound passed when nothing moved (novox/hq issue 229).
func TestAPlanFollowedFromTheStoreAnswersWhenItMoves(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
now := time.Now().UTC()
walk := inventory.Plan{ID: "plan-229", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c229c229",
Created: now, State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
Modules: map[string]*inventory.PlanModule{"a": {}, "b": {}}}
if err := open.inventory.SavePlan(ctx, &walk); err != nil {
t.Fatal(err)
}
moved := make(chan error, 1)
go func() {
time.Sleep(1500 * time.Millisecond)
p, err := open.inventory.PlanByID(ctx, walk.ID)
if err == nil {
p.Tier, p.State = 1, inventory.PlanRolling
err = open.inventory.SavePlan(ctx, &p)
}
moved <- err
}()
began := time.Now()
var got planWaited
out := printed(t, func() error {
return plansCommand(ctx, []string{"wait", walk.ID, "--until", "tier", "--bound", "15s"})
})
if err := <-moved; err != nil {
t.Fatal(err)
}
if err := json.Unmarshal([]byte(out), &got); err != nil {
t.Fatalf("plans wait did not answer data: %v: %q", err, out)
}
if got.Came != "tier" || got.Plan.Tier != 1 || got.Plan.State != inventory.PlanRolling {
t.Fatalf("the wait answered %+v", got)
}
if waited := time.Since(began); waited > 10*time.Second {
t.Errorf("it answered after %s, not when the plan moved", waited)
}
out = printed(t, func() error {
return plansCommand(ctx, []string{"wait", walk.ID, "--bound", "2"})
})
if err := json.Unmarshal([]byte(out), &got); err != nil || got.Came != "bound" || got.Until != "moved" {
t.Fatalf("a plan that did not move answered %+v (%v)", got, err)
}
}
// The verb's schema says the longest bound in words, and the words are the value (review 75 on #266): the
// catalogue cannot read maxPlanWait, so this holds them together.
func TestThePlansSchemaSaysTheLongestBoundThereIs(t *testing.T) {
v, ok := controllerVerb("plans")
if !ok {
t.Fatal("no plans verb")
}
props, _ := v.Input["properties"].(map[string]any)
bound, _ := props["bound"].(map[string]any)
said, _ := bound["description"].(string)
if !strings.Contains(said, maxPlanWait.String()+", the default, is the longest") {
t.Fatalf("the schema says %q; the longest bound is %s", said, maxPlanWait)
}
}
+3
View File
@@ -1027,6 +1027,9 @@ func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (stri
if err != nil {
return "", err
}
// A send that carried the mesh's record of the controller and the node-engine — a person's push included —
// ends the condition of one kept over it (novox/hq issue 201).
settleCoreDowngrades(ctx, b.open.inventory, s.node, s.declared.Builds)
if len(s.declared.Bindings) > 0 {
// And where it bound each consumer of a provision that keeps its data (novox/hq ADR 0232):
// what the next resolution keeps it at. On the same outliving context as the send's record.
+17
View File
@@ -135,6 +135,17 @@ func recordedKept(ctx context.Context, open *stores, node string) (map[string]st
}
out[m] = was
}
// **And the controller and the node-engine at the build the machine was last sent, over an older record**
// (novox/hq issue 201, core_downgrade.go). Under keepingEveryBuild they are kept already.
if !every {
core, err := coreKept(ctx, inv, node, sent, current)
if err != nil {
return nil, err
}
for m, was := range core {
out[m] = was
}
}
return out, nil
}
@@ -159,6 +170,12 @@ func keepRecorded(ctx context.Context, open *stores, node string, shelf map[stri
return nil, fmt.Errorf("%s runs %s's build %s, which the build records no longer hold: the bus's user "+
"list cannot be sent there alone with it kept (novox/hq issue 490)", node, m, short(kept[m]))
}
if !found && guarded(coreComponent(m)) {
return nil, fmt.Errorf("%s was last sent %s's build %s, newer than the one the mesh holds, and the build records "+
"no longer hold it: this send cannot keep it and does not move it back — `push %s` sends the older build on "+
"a person's word, and a newer merge or `rebuild %s` moves the record forward (novox/hq issue 201)",
node, m, short(kept[m]), node, m)
}
if !found {
return nil, fmt.Errorf("%s records rather than rolls out, and %s runs its build %s, which the build "+
"records no longer hold: this send cannot keep it and does not move it — `push %s` sends the new "+
+36 -10
View File
@@ -51,6 +51,9 @@ type sendScope struct {
modules map[string]bool
// person is a person's act: it carries what it carries.
person bool
// puttingBack is a gate's own put-back of its modules: the one send that may carry a core module to a build
// older than the one its machine runs (novox/hq issue 201).
puttingBack bool
}
type sendScopeKey struct{}
@@ -237,6 +240,32 @@ var machineMoves = func(ctx context.Context, open *stores, f moveFacts, node str
return f.moves(node, modules, sent, known, all), nil
}
// withoutKept is a machine's moves less those its send keeps at the build the machine was last sent, and so does
// not make: the grants step keeps every module (novox/hq issue 490), and every send keeps the controller and the
// node-engine over an older record (issue 201). Read as the send composes it (sendKeeps, set after the checks that
// call this), so a move it could not keep is still counted.
func withoutKept(ctx context.Context, open *stores, node string, moves []inventory.CarriedMove) ([]inventory.CarriedMove, error) {
if len(moves) == 0 {
return moves, nil
}
keeping := ctx
if _, on := keptExcept(ctx); !on {
keeping = keepingRecorded(ctx)
}
keeps, err := recordedKept(keeping, open, node)
if err != nil || len(keeps) == 0 {
return moves, err
}
var out []inventory.CarriedMove
for _, mv := range moves {
if was, kept := keeps[mv.Module]; kept && sameCommit(was, mv.From) {
continue
}
out = append(out, mv)
}
return out, nil
}
// ungatedIn refuses a send whose machines would move a build no gate has seen, outside its scope: the
// machines named, and why; the machine holding the bus, added only for its user list, is left instead.
func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder string) ([]string, error) {
@@ -254,19 +283,11 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
if err != nil {
return nil, err
}
// The grants step composes every module at the build its machine was last sent (novox/hq issue 490):
// a move it keeps is not made. Read, not assumed, so a move it could not keep is still refused here.
var keeps map[string]string
if everyBuildKept(ctx, n) {
if keeps, err = recordedKept(ctx, open, n); err != nil {
return nil, err
}
if moves, err = withoutKept(ctx, open, n, moves); err != nil {
return nil, err
}
var waiting []string
for _, mv := range moves {
if was, kept := keeps[mv.Module]; kept && sameCommit(was, mv.From) {
continue
}
if !scope.judged[n] && !scope.modules[mv.Module] {
waiting = append(waiting, fmt.Sprintf("%s %s → %s", mv.Module, short(mv.From), short(mv.To)))
}
@@ -308,6 +329,11 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
if err != nil {
return nil, nil, nil, err
}
// A move the send keeps is not made, and is no move for the gate to judge (novox/hq issue 201): a core module
// kept over an older record judged as moving back to it would read the gate as superseded by its own send.
if moves, err = withoutKept(ctx, open, node, moves); err != nil {
return nil, nil, nil, err
}
own := func(module string) bool {
return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module })
}
+7
View File
@@ -1463,6 +1463,10 @@ func plansCommand(ctx context.Context, args []string) error {
modules := set.String("modules", "", "or the modules it would change, comma-separated")
moduleDirs := set.String("module-dirs", "", "with --paths: the directories holding a module.json at the commit, "+
"comma-separated, as the forge's announcer says them (novox/hq issue 278)")
// Following a plan (novox/hq issue 229): `plans wait <id>`.
until := set.String("until", "", "with wait: what to wait for — moved (any change), tier (its next tier) or done (its end)")
bound := set.String("bound", "", "with wait: how long to wait at most, as 15s or seconds (default and at most the "+
"longest a call is answered within)")
// Ending a plan by hand is a repair, and says why (novox/hq to-be 45 §7).
why := addHandActFlags(set)
positionals, err := parseAround(set, args)
@@ -1482,6 +1486,9 @@ func plansCommand(ctx context.Context, args []string) error {
defer open.Close()
inv := open.inventory
now := time.Now()
if len(positionals) == 2 && positionals[0] == "wait" {
return planWaitCommand(ctx, inv, positionals[1], *until, *bound)
}
if len(positionals) == 1 {
p, err := inv.PlanByID(ctx, positionals[0])
if err != nil {
+83
View File
@@ -0,0 +1,83 @@
package main
import (
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// TestReplay401 replays novox/hq issue 401 (2026-10-10): the mounts module's smb-credentials, an own secret
// only an outside party issues, was given by nobody, and the send made a random value for it and placed it on
// the machine; the module read it as a credential given and wrong. The outcome asserted (ADR 0228): the send
// makes no value for an own secret issued outside the mesh that nobody gave and places none, and a value the
// mesh made for it before is not placed either; once a person gives one, that value is placed. Written with
// only what the controller had before its fix (mesh-controller #262) — register, assign, planFor,
// declarationFor and the inventory's secret reads — so it is laid over the commit before. Needs the store
// (MESH_TEST_POSTGRES), and is named in testdata/store-tests.
func TestReplay401(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
const made, outside = "/var/lib/mesh/mounts/password", "/var/lib/mesh/mounts/smb-credentials"
register(t, open, catalogue.Manifest{Module: "mounts", Version: "1",
OwnSecrets: catalogue.OwnSecrets{
"password": {Path: made, Taken: catalogue.TakenAtStart},
"smb-credentials": {Path: outside, Taken: catalogue.TakenAtStart, IssuedBy: catalogue.IssuedOutside},
},
Resources: []map[string]any{
{"id": "state", "type": "directory", "path": "/var/lib/mesh/mounts", "mode": "0700"},
}})
if _, err := assign(ctx, open, "laptop", "mounts"); err != nil {
t.Fatal(err)
}
sent := func() []map[string]any {
t.Helper()
plan, settings, err := planFor(ctx, open, "laptop")
if err != nil {
t.Fatal(err)
}
declared, err := declarationFor(ctx, open, "laptop", plan, settings)
if err != nil {
t.Fatalf("the send to laptop did not compose: %v", err)
}
return declared.Resources
}
placed := func(resources []map[string]any, path string) map[string]any {
for _, r := range resources {
if r["path"] == path && r["sealed"] != nil {
return r
}
}
return nil
}
first := sent()
if placed(first, made) == nil {
t.Fatal("the secret the mesh makes was not placed")
}
if got := placed(first, outside); got != nil {
t.Errorf("the send placed a value for smb-credentials, which nobody gave: %v", got)
}
if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "mounts", "smb-credentials"); err != nil || held {
t.Fatalf("the send made a value for a secret issued outside the mesh (held %v, %v)", held, err)
}
// A value the mesh made for it before the rule is not one a person gave: it is not placed.
if _, err := open.inventory.SecretForModule(ctx, "laptop", "mounts", "smb-credentials"); err != nil {
t.Fatal(err)
}
if got := placed(sent(), outside); got != nil {
t.Fatalf("a value the mesh made for a secret issued outside was placed: %v", got)
}
// Given by a person, it is placed, as given.
if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "mounts", "smb-credentials", "the share's credentials"); err != nil {
t.Fatal(err)
}
given, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "mounts", "smb-credentials")
if err != nil || !held {
t.Fatalf("the value given is not held (%v)", err)
}
if got := placed(sent(), outside); got == nil || got["sealed"] != given {
t.Fatalf("the value given is placed as %v", got)
}
}
+32 -2
View File
@@ -336,6 +336,9 @@ func (a *verbArguments) commandLine() ([]string, error) {
}
return argv, nil
case "plans":
if err := plainPlanIDs(a.given); err != nil {
return nil, err
}
if r := str("repository"); r != "" {
argv := []string{"plans", "--what-if", r}
if p := str("paths"); p != "" {
@@ -365,6 +368,21 @@ func (a *verbArguments) commandLine() ([]string, error) {
return argv, nil
}
}
if id := str("wait"); id != "" {
// Following a plan (novox/hq issue 229): checked here too, so a call asking what no wait can do is
// refused before a command is started.
if _, _, err := planWaitAsked(a.given["until"], a.given["bound"]); err != nil {
return nil, err
}
argv := []string{"plans", "wait", id}
if u := str("until"); u != "" {
argv = append(argv, "--until", u)
}
if b := str("bound"); b != "" {
argv = append(argv, "--bound", b)
}
return argv, nil
}
if id := str("id"); id != "" {
return []string{"plans", id}, nil
}
@@ -985,12 +1003,14 @@ var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "co
// The delivery's owner's verbs answer JSON where they read (plan, order, check, walks) — novox/hq ADR 0239.
"delivery": true,
// The build queue, each ask with its position among the waiting (novox/hq issue 455).
"queue": true}
"queue": true,
// A plan followed (novox/hq issue 229): `plans wait` answers JSON; its other forms say text, and so answer no data.
"plans": true}
// tableBesideVerbs are the JSON verbs whose document carries the table a person reads as its `text`: the
// verb answers that table as its output, as it did before it answered data, and the rest as `answer` —
// both from the one read, so they never disagree (novox/hq issue 455).
var tableBesideVerbs = map[string]bool{"queue": true}
var tableBesideVerbs = map[string]bool{"queue": true, "plans": true}
// overviewVerbs are the JSON verbs whose answer is said once, as data, and not again as the text the
// command printed: the overviews, which grow with the mesh (novox/hq issue 314). Every reader of theirs
@@ -1264,6 +1284,16 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
// Whatever it did, `status` is composed again once it has.
defer statusFrom.nudge()
}
if verb == "plans" && len(argv) > 1 && argv[1] == "wait" {
// A wait holds its answer up to its bound (novox/hq issue 229), and is answered in full inside it.
asked := ""
if i := slices.Index(argv, "--bound"); i > 0 && i+1 < len(argv) {
asked = argv[i+1]
}
if _, bound, err := planWaitAsked("", asked); err == nil {
link.AnswersWithin(ctx, bound+planWaitStart)
}
}
if answer, read := readHere(ctx, argv); read {
return answer, nil
}
+10
View File
@@ -254,6 +254,16 @@ var signalsTable = []signalRow{
newest: func(f *signalFacts) time.Time {
return newestOf(f.bus.waits, func(w busWaitFacts) time.Time { return w.since })
}},
{Row: "S21", Signal: "a clean apply", Emitter: "node-engine", Trigger: "each report of a declaration refused or applied only in part (novox/hq issue 233)",
Bound: "one apply interval (5 min) past the first report of the same failure, judged by time while the machine " +
"is heard — the node-engine says a re-apply only when how it went changed; not while it is silent or " +
"asleep; naming the refusal, or the package manager that does not answer with its words; cleared by the " +
"clean apply it then says",
Kind: kindApplyFailing, Severity: conditions.Warning, Phase: 2,
needs: func(f *signalFacts) error { return f.doingErr }, watch: watchApplyFailing,
newest: func(f *signalFacts) time.Time {
return newestOf(f.doing, func(d inventory.Doing) time.Time { return d.At })
}},
}
// watchFacts is S14: the snapshot a merge check is fed is older than its bound, or none was kept since
+8
View File
@@ -186,6 +186,14 @@ var suppressions = map[string]suppression{
},
past: func(f *signalFacts) { f.refusedSends = []inventory.Send{refusedSend(f.now.Add(-time.Second))} },
},
// A machine whose package manager does not answer (novox/hq issue 233): said once its first report of the
// failure is older than one apply interval while it is heard. Inside: exactly five minutes old.
"S21": {
inside: func(f *signalFacts) { f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute, 1)} },
past: func(f *signalFacts) {
f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute+time.Second, 1)}
},
},
// Twice by hand within a fortnight is a healer wanted; once, or the first of two a day too old, is not.
"S15": {
inside: func(f *signalFacts) {
@@ -344,3 +344,58 @@ func TestAnyKeyOfAModuleWithATrustedMergeableFileIsRefusedThroughAVerb(t *testin
t.Fatalf("a file marked untrusted through the verb: %v", err)
}
}
// Who owns a module's directory is said at the terminal alone (novox/hq issue 279): a directory's owner may name
// the settings the module runs as, root chowns the folder to them at the next push, and through the settings verb
// any caller could have set puid to 0 or to its own uid. The key is refused through every verb route even where a
// file nothing trusts also asks for it, and taken at the terminal; the layer a refused call meets is unchanged.
func TestADirectoryOwnersKeyIsRefusedThroughAVerb(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, catalogue.Manifest{Module: "sonarr", Version: "1",
Resources: []map[string]any{
{"id": "recycle-bin", "type": "directory", "mode": "0755", "owner": "${setting:puid}:${setting:pgid}"},
{"id": "identity", "type": "file", "path": "/etc/sonarr.env", "mode": "0644", "trusted": false,
"content": "PUID=${setting:puid}\nPGID=${setting:pgid}\nX=${setting:x}\n"}}})
if err := atTheTerminal(t, "settings", "set", "sonarr", `{"puid":1001,"pgid":2000,"x":0}`,
"--node", "anchor"); err != nil {
t.Fatalf("the owner at the terminal: %v", err)
}
if _, err := assign(ctx, open, "anchor", "sonarr"); err != nil {
t.Fatal(err)
}
layer := func() string {
t.Helper()
values, _, err := open.inventory.Layer(ctx, "anchor", "sonarr")
if err != nil {
t.Fatal(err)
}
raw, _ := json.Marshal(values)
return string(raw)
}
kept := layer()
refused := func(what string, err error) {
t.Helper()
if err == nil || !strings.Contains(err.Error(), "controller's terminal") {
t.Fatalf("%s: %v", what, err)
}
}
refused("puid 0 through the verb", throughVerb(t, "settings", map[string]any{"module": "sonarr",
"node": "anchor", "values": `{"puid":0,"pgid":2000,"x":0}`}))
refused("pgid through the verb, mesh-wide", throughVerb(t, "settings", map[string]any{"module": "sonarr",
"values": `{"pgid":0}`}))
refused("cleared through the verb", throughVerb(t, "settings", map[string]any{"module": "sonarr",
"node": "anchor", "clear": "true"}))
if err := throughVerb(t, "command", map[string]any{
"command": `settings set sonarr '{"puid":0,"pgid":2000,"x":0}' --node anchor`}); err == nil {
t.Fatal("the command verb set the owner")
}
if got := layer(); got != kept {
t.Fatalf("a refused call changed the layer: %s, was %s", got, kept)
}
// Another key, the owner kept, is still the verb's.
if err := throughVerb(t, "settings", map[string]any{"module": "sonarr", "node": "anchor",
"values": `{"puid":1001,"pgid":2000,"x":1}`}); err != nil {
t.Fatalf("another key through the verb, the owner kept: %v", err)
}
}
+5
View File
@@ -59,6 +59,10 @@ type signalFacts struct {
// from, each naming its sender (S20, novox/hq issue 234).
refusedSends []inventory.Send
refusedSendsErr error
// doing is every machine whose last report was not a clean apply, with since when it has said the same
// failure (S21, novox/hq issue 233).
doing []inventory.Doing
doingErr error
loop loopFacts
loopErr error
@@ -369,6 +373,7 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
}
f.handActs, f.handActsErr = w.gatherHandActs(ctx, now)
f.refusedSends, f.refusedSendsErr = inv.RefusedSendsSince(ctx, now.Add(-generationRefusalsSaid))
f.doing, f.doingErr = inv.NotDoingWhatTheyWereTold(ctx)
f.facts.taken, _, f.facts.began, f.facts.err = exportedFacts.last()
return f
}