Bring F3 up to its release branch, with both sides' store tests (part 1 of 14)

The merge is too large for one commit through the forge's tool, so it lands in parts; the tree after the last part is the merge plus the declared-config mirror carrying every declaration field.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

cmd/mesh-builder/main.go

@@ -332,6 +332,12 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
 				result.Check.Members = append(result.Check.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo,
 					Commit: m.Head, Trunk: m.Trunk})
 			}
+			// A branch tried before its pull request reads its outcome from this log, whole (novox/hq issue 686).
+			if request.Check.Try {
+				if line, err := json.Marshal(triedOf(v)); err == nil {
+					say(link.StepTried, string(line))
+				}
+			}
 		}
 	} else if err == nil {
 		// The package-registry credential is a build input, so it is resolved before the clone: a
@@ -457,7 +463,8 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
 	spec := builder.CheckSpec{ID: request.ID, Repository: request.Repository, Ref: request.Ref,
 		Owner: c.Owner, Repo: c.Repo, Number: c.Number, Paths: c.Paths, Beside: map[string]builder.Beside{},
 		Modules: c.Modules, New: c.New, Manifests: c.Manifests, Judge: c.Judge, Base: c.Base,
-		Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held)}
+		Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held),
+		Tests: c.Tests, StoreOnly: c.StoreOnly}
 	for dir, b := range c.Beside {
 		spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref}
 	}
@@ -472,6 +479,23 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
 	return spec
 }
 
+// triedOf is a try's outcome as its log line says it (novox/hq issue 686): the repository's own check's verdict,
+// or the check's where that did not run — a head that does not merge — and the lines naming what failed.
+func triedOf(v builder.CheckVerdict) link.Tried {
+	t := link.Tried{Verdict: v.Verdict, Summary: v.Summary, Took: v.Took.Round(time.Second).String(),
+		Head: v.Head, Trunk: v.Trunk}
+	if v.Repo != nil && (v.Gate == nil || v.Gate.Verdict == "pass") {
+		t.Verdict, t.Summary = v.Repo.Verdict, v.Repo.Summary
+	}
+	if v.Repo != nil && v.Repo.Failed != "" {
+		t.Failed = strings.Split(v.Repo.Failed, "\n")
+	}
+	if t.Verdict == "" {
+		t.Verdict = "error"
+	}
+	return t
+}
+
 // checkBuiltOf is a check's builds for its merge as the outcome carries them (novox/hq ADR 0350 §1).
 func checkBuiltOf(built []builder.CheckBuilt) []link.CheckBuilt {
 	var out []link.CheckBuilt

cmd/mesh-builder/tried_test.go

@@ -0,0 +1,31 @@
+package main
+
+import (
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/builder"
+)
+
+// A try's outcome, as the build seat says it in the build's log (novox/hq issue 686): the repository's own
+// check's verdict, how long it took, and each line naming what failed.
+func TestATrysOutcomeNamesWhatFailed(t *testing.T) {
+	tried := triedOf(builder.CheckVerdict{Verdict: "pass", Summary: "the change touches no module", Took: 303 * time.Second,
+		Head: "c0ffee", Trunk: "ca3f43c", Gate: &builder.Layer{Verdict: "pass"},
+		Repo: &builder.Layer{Verdict: "fail", Summary: "its merge-check.sh failed: TestX",
+			Failed: "--- FAIL: TestX (0.01s)\n    x_test.go:12: got 1, want 2"}})
+	if tried.Verdict != "fail" || tried.Took != "5m3s" || len(tried.Failed) != 2 || tried.Trunk != "ca3f43c" ||
+		tried.Summary != "its merge-check.sh failed: TestX" {
+		t.Fatalf("a failed try: %+v", tried)
+	}
+	// A head that does not merge fails both layers: the check's own word.
+	conflict := triedOf(builder.CheckVerdict{Verdict: "fail", Summary: "does not merge", Gate: &builder.Layer{Verdict: "fail"},
+		Repo: &builder.Layer{Verdict: "fail", Summary: "does not merge"}})
+	if conflict.Verdict != "fail" || conflict.Summary != "does not merge" {
+		t.Fatalf("a try that does not merge: %+v", conflict)
+	}
+	// No verdict is an error, never a pass.
+	if none := triedOf(builder.CheckVerdict{}); none.Verdict != "error" {
+		t.Fatalf("a try with no verdict: %+v", none)
+	}
+}

cmd/mesh-controller/adoption.go

@@ -424,8 +424,9 @@ func comparisonOf(module string, c comparison, opts takeOptions) (preview string
 	}
 	// And its settings on this machine, composed against its definition (rule 1, rule 6).
 	for _, layer := range c.layers {
-		keys := make([]string, 0, len(layer.Values))
-		for k := range layer.Values {
+		said := layer.Whole()
+		keys := make([]string, 0, len(said))
+		for k := range said {
 			keys = append(keys, k)
 		}
 		sort.Strings(keys)

cmd/mesh-controller/check.go

@@ -55,6 +55,11 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
 			continue
 		}
 		m, err := catalogue.ParseManifest(raw)
+		if err == nil {
+			// Each template's text read from the tree beside the manifest, and judged as inline content is
+			// (novox/hq ADR 0349 §2): a missing template is refused by name.
+			m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(path))
+		}
 		if err != nil {
 			fmt.Fprintf(out, "%s: %v\n", path, err)
 			failed++

cmd/mesh-controller/facts.go

@@ -348,11 +348,11 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot
 				if layer.From == catalogue.MeshWideLayer {
 					if !meshWide[module] {
 						meshWide[module] = true
-						f.Settings = append(f.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Values)})
+						f.Settings = append(f.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Whole())})
 					}
 					continue
 				}
-				m.Settings = append(m.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Values)})
+				m.Settings = append(m.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Whole())})
 			}
 		}
 		m.Declaration = declarationFacts(ctx, open, n.Name, gens, gensErr, scrub)

cmd/mesh-controller/gate.go

@@ -1443,6 +1443,12 @@ func probeGates(ctx context.Context, d *doctor) ([]conditions.Observation, error
 	}
 	// A release held after a failed one, while builds still wait for a gate (ADR 0236).
 	out = append(out, backlogObservation()...)
+	// And a rest that went silent (novox/hq ADR 0353 §4).
+	silent, err := silentObservation(ctx, d.open.inventory)
+	if err != nil {
+		return nil, err
+	}
+	out = append(out, silent...)
 	return sortedFound(dedupeObservations(out)), nil
 }
 
@@ -1532,6 +1538,12 @@ func gateLine(g *inventory.PlanGate) string {
 	case inventory.GateFailed:
 		what = "FAILED"
 	}
+	switch {
+	case g.Verdict == "" && g.Pending:
+		what = "kept, not yet sent" // novox/hq ADR 0353 §8
+	case g.Verdict == "" && g.Silent:
+		what = "silent on its send, left" // ADR 0353 §4
+	}
 	if g.Verdict == "" && g.Waiting() {
 		what = "waits for a busy container on " + g.Busy.Machine
 	}

cmd/mesh-controller/held_back.go

@@ -151,7 +151,7 @@ func heldMachines(ctx context.Context, open *stores, names []string) (map[string
 		for _, mv := range f.moves(node, modules, same, known, false) {
 			if planStillToSend(plans, mv.Module, node) == "" {
 				why = append(why, fmt.Sprintf("%s would move from %s to %s, which has passed no gate yet — a release "+
-					"plan sends it, one machine at a time, judged", mv.Module, buildName(mv.From), buildName(mv.To)))
+					"plan sends it, one machine first and then the rest at once, judged", mv.Module, buildName(mv.From), buildName(mv.To)))
 			}
 		}
 		if len(why) > 0 {

cmd/mesh-controller/issue648_test.go

@@ -0,0 +1,87 @@
+package main
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// A tier whose module is no move — made from the source every machine running it runs, so nothing is
+// sent — is not waited on for a report after its marking (novox/hq issue 648). Its machines answered the
+// send that last carried that build before the walk got there, and nothing will make them report again:
+// the walk asks the next tier in the same step, rather than "waiting for app on anchor, laptop" for ever.
+func TestANoMoveTierWaitsForNoNewReport(t *testing.T) {
+	judgeLeavesAsTheCore(t)
+	g := aGateMesh(t)
+	ctx := t.Context()
+	inv := g.open.inventory
+
+	// app is the build machine, so a module of the next tier is built by it: the tier gates the next on app
+	// running on its machines.
+	for _, m := range []catalogue.Manifest{
+		{Module: "app", Version: "c2", Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}},
+		{Module: "web", Version: "w1"},
+	} {
+		if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
+			Path: "modules/" + m.Module, BuiltFrom: m.Version, Head: m.Version, Asked: time.Now().Add(-time.Minute)}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	// What anchor and laptop were last sent (c1) was made from the source the plan's build of c2 is.
+	for _, b := range []inventory.Build{
+		{ID: "build-1s", Module: "app", Commit: "c1", SourceFingerprint: "src1:app",
+			Asked: time.Now().Add(-30 * time.Second), At: time.Now().Add(-30 * time.Second)},
+		{ID: "build-2s", Module: "app", Commit: "c2", SourceFingerprint: "src1:app", Asked: time.Now(), At: time.Now()},
+	} {
+		b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
+		if err := inv.RecordBuild(ctx, b); err != nil {
+			t.Fatal(err)
+		}
+	}
+	// Each machine answered that send, applied, before the walk's tier was built.
+	for _, n := range []string{"anchor", "laptop"} {
+		if _, err := inv.RecordDoing(ctx, nodeID(t, g.open, n), inventory.Doing{Node: n,
+			Outcome: inventory.OutcomeApplied, Declared: "d-" + n + "-c1", Applied: 1,
+			At: time.Now().Add(-10 * time.Second)}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	built := time.Now().UTC()
+	p := g.plan(t)
+	p.Tiers = [][]string{{"app"}, {"web"}}
+	p.Modules["app"].Build, p.Modules["app"].BuiltAt = "build-2s", &built
+	if err := inv.SavePlan(ctx, &p); err != nil {
+		t.Fatal(err)
+	}
+
+	advancePlans(ctx, g.open)
+
+	if len(g.sent) != 0 {
+		t.Fatalf("a no-move build was sent: %v", g.sent)
+	}
+	p = g.plan(t)
+	if s := p.Modules["app"]; s == nil || !strings.HasPrefix(s.Why, "no move") {
+		t.Fatalf("the build was not marked no move: %+v", s)
+	}
+	if p.Tier < 1 || strings.Contains(p.Note, "waiting for app") {
+		t.Fatalf("the walk waits on a no-move tier for a report nothing will cause: tier %d, %s %q", p.Tier, p.State, p.Note)
+	}
+
+	// A machine that has not answered its last send is still waited on: no move is not "never sent".
+	p.Tier, p.State, p.Note = 0, inventory.PlanRolling, ""
+	if err := inv.RecordSent(ctx, nodeID(t, g.open, "laptop"), "d-laptop-later", map[string]string{"app": "c1"}); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.SavePlan(ctx, &p); err != nil {
+		t.Fatal(err)
+	}
+	advancePlans(ctx, g.open)
+	p = g.plan(t)
+	if p.Tier != 0 || !strings.Contains(p.Note, "waiting for app on laptop") {
+		t.Fatalf("a machine that has not answered its last send was not waited on: tier %d, %q", p.Tier, p.Note)
+	}
+}

cmd/mesh-controller/main.go

@@ -106,6 +106,8 @@ func run() error {
 		return killCommand(ctx, args[1:])
 	case "forget-checks":
 		return forgetChecksCommand(ctx, args[1:])
+	case "try":
+		return tryCommand(ctx, args[1:])
 	case "pause", "resume":
 		return pauseCommand(ctx, args[0], args[1:])
 	case "collection":
@@ -381,6 +383,13 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
 	// **A merge check builds nothing** (novox/hq to-be 45 §9): its verdict is said, and nothing of it is
 	// recorded or registered.
 	if result.Check != nil || result.Checked != nil {
+		if isTry(result) {
+			// A branch tried before its pull request (novox/hq issue 686): its asker reads the outcome from the
+			// build's log. Never said as `checked`, so the forge's holder sets no status.
+			fmt.Printf("%s: a try of %s at %.8s, read with `try --id`: said to nobody, nothing taken in\n", result.ID,
+				result.Repository, result.Ref)
+			return nil
+		}
 		checked(ctx, result)
 		return nil
 	}

cmd/mesh-controller/merge_gate.go

@@ -576,6 +576,11 @@ func shelfWithChange(base map[string]catalogue.Manifest, sources map[string]inve
 			return err
 		}
 		m, err := catalogue.ParseManifest(raw)
+		if err == nil {
+			// Each template's text from the tree beside the manifest, as the build reads it back from the archive it
+			// packs (novox/hq ADR 0349 §2): so a change to a template alone is a change to its module.
+			m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(path))
+		}
 		if err != nil {
 			if inScope(dir) {
 				failures = append(failures, fmt.Sprintf("%s: the controller judging this (%s) refuses it — %s",

cmd/mesh-controller/mesh_for_test.go

@@ -33,7 +33,21 @@ import (
 func aMesh(t *testing.T) *stores {
 	t.Helper()
 	inventory.ForTest(t) // raises the store, migrates it, and points the environment at it
-	licences.ForTest(t)  // planning reaches this one too, by name and never by connection
+	return aMeshOnTheStore(t)
+}
+
+// aMeshBefore is aMesh on a store migrated only below migration `number`, with the function that runs the
+// rest: for a test of what a migration that moves data does to what the mesh composes (novox/hq issue 651).
+func aMeshBefore(t *testing.T, number int) (*stores, func()) {
+	t.Helper()
+	_, migrate := inventory.ForTestBefore(t, number)
+	return aMeshOnTheStore(t), migrate
+}
+
+// aMeshOnTheStore is the mesh aMesh makes, on the store the environment points at.
+func aMeshOnTheStore(t *testing.T) *stores {
+	t.Helper()
+	licences.ForTest(t) // planning reaches this one too, by name and never by connection
 
 	open, err := openStores(t.Context())
 	if err != nil {

cmd/mesh-controller/modules.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"path/filepath"
 	"sort"
 	"strings"
 	"time"
@@ -127,6 +128,10 @@ func moduleCommand(ctx context.Context, args []string) error {
 		if err != nil {
 			return err
 		}
+		// A manifest handed over by hand takes each template's text from the tree beside it (novox/hq ADR 0349 §2).
+		if m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(positionals[0])); err != nil {
+			return err
+		}
 		from, err := whereItComesFrom(*repo, *ref, *commit, *path, *self)
 		if err != nil {
 			return err
@@ -537,6 +542,10 @@ func settingsCommand(ctx context.Context, args []string) error {
 				if p.SetBy != "" {
 					by = "; " + p.SetBy
 				}
+				if p.Why != "" {
+					// A layer whose placement moved out of it (migration 0097): still the layer, as it stands.
+					by += "; " + p.Why
+				}
 				fmt.Printf("%s on %s, until %s (%s%s):\n  %s\n", positionals[0], where,
 					p.ReplacedAt.Local().Format("2006-01-02 15:04:05"), p.ReplacedBy, by, shown)
 			}

cmd/mesh-controller/placement_moved_test.go

@@ -0,0 +1,113 @@
+package main
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// **Placement leaves the settings layers and no machine is told anything new** (novox/hq ADR 0348 clause
+// 11, issue 651). A mesh is seeded as the store holds it before migration 0097 — `places`, `accesses` and a
+// kept network among the settings, at the mesh's level and a machine's, one layer of placement only — then
+// migrated, and every machine's declaration is composed. What each machine is told of the seeded modules is
+// byte for byte what the controller before this change composed from the same seed: the golden files were
+// written by the integration branch's controller at bfc79c6 running this same test (MESH_WRITE_GOLDEN=1),
+// where there was no migration 0097 to run.
+func TestADeclarationIsTheSameAfterPlacementMoves(t *testing.T) {
+	open, migrate := aMeshBefore(t, 97)
+	ctx := t.Context()
+	inv := open.inventory
+	image := "registry.invalid:5000/library@sha256:" + strings.Repeat("a", 64)
+	register(t, open, catalogue.Manifest{Module: "library", Version: "1",
+		Accesses: []catalogue.Access{{ID: "series"}, {ID: "downloads", Mode: catalogue.AccessRead}},
+		Resources: []map[string]any{
+			{"id": "config", "type": "directory", "mode": "0750"},
+			{"id": "server", "type": "container", "image": image,
+				"volumes": []any{"${dir:config}:/config", "${access:series}:/series", "${access:downloads}:/downloads:ro"}},
+			// Today's form: a mergeable file naming no keys takes every key of the layer (ADR 0326 §3).
+			{"id": "runtime-config", "type": "file", "path": "${dir:config}/runtime.json", "mode": "0640",
+				"content": `{"log":"info"}`, "merge": "json"},
+		}})
+	register(t, open, catalogue.Manifest{Module: "found", Version: "1",
+		Resources: []map[string]any{{"id": "server", "type": "container", "image": image}}})
+	if err := inv.SetAdopted(ctx, "anchor", true); err != nil {
+		t.Fatal(err)
+	}
+	for _, l := range []struct{ node, module, values string }{
+		{"", "library", `{"accesses": {"downloads": "/storage/downloads"}, "log": "warn"}`},
+		{"anchor", "library", `{"places": {"config": {"path": "/var/lib/library/config", "owner": "1001:2000"}},
+			"accesses": {"series": "/storage/media/series"}, "workers": 4}`},
+		{"laptop", "library", `{"accesses": {"series": "/home/media/series"}}`},
+		{"anchor", "found", `{"networks": {"server": ["predecessor_default"]}}`},
+	} {
+		if l.node == "" {
+			inv.ExecForTest(t, `insert into settings (node, module, values) values (null, $1, $2)`, l.module, l.values)
+			continue
+		}
+		inv.ExecForTest(t, `insert into settings (node, module, values) select id, $2, $3 from node where name = $1`,
+			l.node, l.module, l.values)
+	}
+
+	migrate()
+	// Assigned once the store is migrated: an assignment reads the layers, as this controller holds them.
+	for _, a := range [][]string{{"anchor", "library", "found"}, {"laptop", "library"}} {
+		if _, err := assign(ctx, open, a[0], a[1:]...); err != nil {
+			t.Fatalf("assign %v: %v", a, err)
+		}
+	}
+
+	gens, err := generators(ctx, open)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, node := range []string{"anchor", "laptop"} {
+		plan, settings, err := planFor(ctx, open, node)
+		if err != nil {
+			t.Fatal(err)
+		}
+		with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Allocating)
+		if err != nil {
+			t.Fatal(err)
+		}
+		composed, err := plan.Compose(with)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(composed.LeftOut) > 0 {
+			t.Fatalf("%s leaves out %v", node, composed.LeftOut)
+		}
+		var told []map[string]any
+		for _, r := range composed.Resources {
+			if owner := composed.Owner[r["id"].(string)]; owner == "library" || owner == "found" {
+				told = append(told, r)
+			}
+		}
+		sort.Slice(told, func(i, j int) bool { return told[i]["id"].(string) < told[j]["id"].(string) })
+		got, err := json.MarshalIndent(told, "", "  ")
+		if err != nil {
+			t.Fatal(err)
+		}
+		golden := filepath.Join("testdata", "placement-moved", node+".json")
+		if os.Getenv("MESH_WRITE_GOLDEN") != "" {
+			if err := os.MkdirAll(filepath.Dir(golden), 0o755); err != nil {
+				t.Fatal(err)
+			}
+			if err := os.WriteFile(golden, append(got, '\n'), 0o644); err != nil {
+				t.Fatal(err)
+			}
+			continue
+		}
+		want, err := os.ReadFile(golden)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if string(want) != string(got)+"\n" {
+			t.Errorf("%s is told something else after placement moved:\n%s\nwas:\n%s", node, got, want)
+		}
+	}
+}

cmd/mesh-controller/plain_words.go

@@ -456,6 +456,9 @@ var plainWordings = map[string]func(conditions.Observation) words{
 	"release-held": worded(func(o conditions.Observation) words {
 		return releaseHeldWords(nil, o.Also)
 	}),
+	"release-silent": worded(func(o conditions.Observation) words {
+		return releaseSilentWords(o.Also)
+	}),
 	"facts-stale": worded(func(o conditions.Observation) words {
 		return words{Headline: "Merge checks use outdated facts",
 			Explanation: "The facts a merge check judges a change against are out of date; they are gathered again " +
@@ -841,6 +844,19 @@ func releaseHeldWords(modules, machines []string) words {
 		Resolved: "Resolved: the held updates are released"}
 }
 
+// releaseSilentWords are the plain words of machines that said nothing on an update they were sent (ADR 0353 §4).
+func releaseSilentWords(machines []string) words {
+	where := "Some machines"
+	if len(machines) > 0 {
+		where = conditions.Capital(namesWords(machines, 4))
+	}
+	return words{Headline: "Machines did not answer an update",
+		Needs: "check that they are awake and on the mesh; the next walk sends them what waits.",
+		Explanation: where + " were sent module updates and said nothing about them within fifteen minutes, so " +
+			"they were left as they were. What they run is not known to be the update.",
+		Resolved: "Resolved: the next walk reached them"}
+}
+
 // reloginNeeds is what an account waiting for its groups needs (ADR 0252).
 func reloginNeeds(node string) string {
 	return fmt.Sprintf("log out of %s completely and log in again, or restart it.", node)