Bring F3 up to K2 on its release branch (part 3 of 3)

testdata/store-tests holds both sides' entries; F3's own sit beside each other, so the list merges with the release branch's.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/catalogue/predefined_test.go

@@ -0,0 +1,175 @@
+package catalogue
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// The list of predefined keys (novox/hq ADR 0348 §7, design 57 package K2, issue 661): read strictly, and every
+// manifest naming a key judged against the list of the tree it is read in.
+
+// theFirstKeys is the catalogue's first list: font-size and agent-managed-settings.
+const theFirstKeys = `{
+  "font-size": {"type": "number", "description": "the size text is shown in, in points", "minimum": 6, "maximum": 72},
+  "agent-managed-settings": {"type": "object", "description": "the managed settings of every agent session, a JSON object", "trusted": true}
+}`
+
+func TestTheFirstListIsRead(t *testing.T) {
+	list, err := ParsePredefined([]byte(theFirstKeys))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := strings.Join(list.Keys(), ","); got != "agent-managed-settings,font-size" {
+		t.Fatalf("keys %s", got)
+	}
+	if k := list["font-size"]; k.Type != ConfigNumber || *k.Minimum != 6 || *k.Maximum != 72 || k.Trusted {
+		t.Fatalf("font-size read as %+v", k)
+	}
+	if k := list["agent-managed-settings"]; k.Type != ConfigObject || !k.Trusted {
+		t.Fatalf("agent-managed-settings read as %+v", k)
+	}
+	// One list is one text, however it was written.
+	a, _ := list.Canonical()
+	again, _ := ParsePredefined(a)
+	b, _ := again.Canonical()
+	if string(a) != string(b) {
+		t.Fatalf("canonical text moved:\n%s\n%s", a, b)
+	}
+}
+
+func TestAListIsRefusedWholeForAnyWrongKey(t *testing.T) {
+	for _, c := range []struct{ file, says string }{
+		{`[]`, "not a JSON object of keys"},
+		{`null`, "not a JSON object of keys"},
+		{`{"font-size": {"type": "number", "description": "points", "default": 10}}`, `unknown field "default"`},
+		{`{"font-size": {"description": "points"}}`, "no type"},
+		{`{"font-size": {"type": "decimal", "description": "points"}}`, `type "decimal"`},
+		{`{"font-size": {"type": "number"}}`, "no description"},
+		{`{"accent": {"type": "object", "description": "colours"}}`, "not trusted"},
+		{`{"font-size": {"type": "number", "description": "points", "minimum": 9, "maximum": 6}}`, "above its maximum"},
+		{`{"accent": {"type": "string", "description": "a name", "pattern": "("}}`, "not a regular expression"},
+		{`{"accent": {"type": "boolean", "description": "on or off", "minimum": 1}}`, "minimum or maximum is for a number"},
+		{`{"Font Size": {"type": "number", "description": "points"}}`, "not a usable key"},
+		{`{"places": {"type": "string", "description": "where"}}`, "placement"},
+		{`{"puid": {"type": "number", "description": "an id"}}`, "placement"},
+	} {
+		if _, err := ParsePredefined([]byte(c.file)); err == nil || !strings.Contains(err.Error(), c.says) {
+			t.Errorf("%s: wanted a refusal saying %q, got %v", c.file, c.says, err)
+		}
+	}
+}
+
+// fallingBackTo is inConfigForm with its font size falling back to a predefined key.
+func fallingBackTo(t *testing.T, key string) []byte {
+	t.Helper()
+	m := inConfigForm()
+	declaration(m, "font-size")["predefined"] = key
+	raw, err := json.Marshal(m)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return raw
+}
+
+// A manifest naming font-size registers against the list; one naming a key not on it, or of another type, is
+// refused — judged against the list given, whatever the mesh published.
+func TestAManifestIsJudgedAgainstTheListOfItsTree(t *testing.T) {
+	withPredefined(t, nil)
+	list, err := ParsePredefined([]byte(theFirstKeys))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ParseManifestWith(fallingBackTo(t, "font-size"), list); err != nil {
+		t.Fatalf("a value falling back to font-size is refused: %v", err)
+	}
+	if _, err := ParseManifestWith(fallingBackTo(t, "font-scale"), list); err == nil ||
+		!strings.Contains(err.Error(), `"font-scale", which is not on the list`) {
+		t.Fatalf("a key not on the list: %v", err)
+	}
+	m := inConfigForm()
+	declaration(m, "theme")["predefined"] = "font-size"
+	raw, _ := json.Marshal(m)
+	if _, err := ParseManifestWith(raw, list); err == nil || !strings.Contains(err.Error(), "which is a number") {
+		t.Fatalf("a key of another type: %v", err)
+	}
+	// Where no tree is read, the published list judges: none published refuses.
+	if _, err := ParseManifest(fallingBackTo(t, "font-size")); err == nil || !strings.Contains(err.Error(), "is not present") {
+		t.Fatalf("judged against no list: %v", err)
+	}
+	withPredefined(t, list)
+	if _, err := ParseManifest(fallingBackTo(t, "font-size")); err != nil {
+		t.Fatalf("judged against the published list: %v", err)
+	}
+}
+
+// A template's text filled from the tree keeps the list the manifest was judged against: read again, it is not
+// judged against the published one.
+func TestTemplatesFilledKeepTheListTheManifestWasJudgedAgainst(t *testing.T) {
+	withPredefined(t, nil)
+	list, _ := ParsePredefined([]byte(theFirstKeys))
+	dir := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(dir, TemplatesDir), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(dir, TemplatesDir, "rc"), []byte("font = ${config:font-size}\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	m := inConfigForm()
+	declaration(m, "font-size")["predefined"] = "font-size"
+	r := resourceOf(m, "configuration")
+	delete(r, "content")
+	r["template"] = TemplatesDir + "/rc"
+	m["build"] = map[string]any{"artifacts": []any{map[string]any{"name": "files", "kind": "archive", "from": "files"}}}
+	declaration(m, "width")["runtime"] = true
+	declaration(m, "sender")["runtime"] = true
+	raw, _ := json.Marshal(m)
+	parsed, err := ParseManifestWith(raw, list)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := WithTemplatesFrom(parsed, dir); err != nil {
+		t.Fatalf("filled from the tree, it was judged against another list: %v", err)
+	}
+}
+
+// module check reads the list at the root of the repository above the manifest; no repository, no list.
+func TestTheListIsFoundAtTheRepositoryRootAboveAManifest(t *testing.T) {
+	root := t.TempDir()
+	module := filepath.Join(root, "modules", "dunst")
+	if err := os.MkdirAll(module, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if list, err := PredefinedAbove(module); err != nil || list != nil {
+		t.Fatalf("outside a repository: %v %v", list, err)
+	}
+	if err := os.Mkdir(filepath.Join(root, ".git"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if list, err := PredefinedAbove(module); err != nil || list != nil {
+		t.Fatalf("a repository without the file: %v %v", list, err)
+	}
+	if err := os.WriteFile(filepath.Join(root, PredefinedFile), []byte(theFirstKeys), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	list, err := PredefinedAbove(module)
+	if err != nil || len(list) != 2 {
+		t.Fatalf("the list above: %v %v", list, err)
+	}
+	if err := os.WriteFile(filepath.Join(root, PredefinedFile), []byte(`{"font-size": {}}`), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := PredefinedAbove(module); err == nil {
+		t.Fatal("a wrong list above was read")
+	}
+	if err := os.Remove(filepath.Join(root, PredefinedFile)); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Symlink("/etc/hostname", filepath.Join(root, PredefinedFile)); err == nil {
+		if _, err := PredefinedAbove(module); err == nil || !strings.Contains(err.Error(), "not a plain file") {
+			t.Fatalf("a link read as the list: %v", err)
+		}
+	}
+}

internal/catalogue/template.go

@@ -312,7 +312,7 @@ func WithTemplatesFrom(m Manifest, dir string) (Manifest, error) {
 	if err != nil {
 		return Manifest{}, err
 	}
-	return ParseManifest(raw)
+	return parseManifest(raw, m.predefined)
 }
 
 // HeldTemplateProblem is why a manifest cannot be held by the mesh because it still names a template, or "": the

internal/inventory/migrations/0098-the-list-of-predefined-keys.sql

@@ -0,0 +1,20 @@
+-- The list of predefined keys (novox/hq ADR 0348 clause 7, design 57 package K2, issue 661).
+--
+-- Mesh config and node config hold values only for the keys in the catalogue's `predefined.json`. A build of
+-- the catalogue reads the file at the commit it builds and carries it; the controller keeps the newest trunk
+-- build's, which every manifest it reads again is judged against, and which mesh-config publishes from here
+-- (design 57 package K3). One list for the mesh: one row.
+--
+-- **The newest trunk build wins**, by the generation of the commit built — how many commits it stands on, which
+-- of two commits on one trunk is larger for the later — so a list heard late from an older build never replaces
+-- a newer one. Not a config value: a key, its type, description, validation and whether it is trusted, all of it
+-- public in the catalogue's repository.
+create table predefined_list (
+    one        boolean     primary key default true check (one),
+    keys       jsonb       not null,
+    repository text        not null,
+    commit     text        not null,
+    generation bigint      not null check (generation > 0),
+    build      text        not null,
+    kept_at    timestamptz not null default now()
+);

internal/inventory/predefined.go

@@ -0,0 +1,76 @@
+package inventory
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// The list of predefined keys the mesh publishes (novox/hq ADR 0348 §7, design 57 package K2): the newest trunk
+// build's of the catalogue, by the generation of the commit built.
+
+// PredefinedRecord is the list kept, and the build it came from.
+type PredefinedRecord struct {
+	Keys       catalogue.PredefinedList
+	Repository string
+	Commit     string
+	Generation int
+	Build      string
+	KeptAt     time.Time
+}
+
+// KeepPredefined keeps a trunk build's list when its commit is newer than the one kept — a larger generation —
+// and answers whether it did. An older build's list, or one of the same generation, heard later is not kept: the
+// newer one stands, whatever order the outcomes arrive in. One statement, so two builds heard at once cannot both
+// win.
+func (i *Inventory) KeepPredefined(ctx context.Context, r PredefinedRecord) (bool, error) {
+	if r.Keys == nil {
+		return false, errors.New("no list to keep")
+	}
+	if r.Generation <= 0 {
+		return false, fmt.Errorf("the build %s did not say its commit's generation, so whether its list is newer "+
+			"than the one kept cannot be said", r.Build)
+	}
+	if problems := catalogue.PredefinedProblems(r.Keys); len(problems) > 0 {
+		return false, fmt.Errorf("the list of %s is refused: %v", r.Build, problems)
+	}
+	text, err := r.Keys.Canonical()
+	if err != nil {
+		return false, err
+	}
+	tag, err := i.store.Pool().Exec(ctx, `
+		insert into predefined_list (keys, repository, commit, generation, build)
+		values ($1, $2, $3, $4, $5)
+		on conflict (one) do update set keys = excluded.keys, repository = excluded.repository,
+			commit = excluded.commit, generation = excluded.generation, build = excluded.build, kept_at = now()
+		where predefined_list.generation < excluded.generation`,
+		text, r.Repository, r.Commit, r.Generation, r.Build)
+	if err != nil {
+		return false, err
+	}
+	return tag.RowsAffected() == 1, nil
+}
+
+// PredefinedKept is the list kept, and false when none is yet.
+func (i *Inventory) PredefinedKept(ctx context.Context) (PredefinedRecord, bool, error) {
+	var r PredefinedRecord
+	var text []byte
+	err := i.store.Pool().QueryRow(ctx,
+		`select keys, repository, commit, generation, build, kept_at from predefined_list`).
+		Scan(&text, &r.Repository, &r.Commit, &r.Generation, &r.Build, &r.KeptAt)
+	if errors.Is(err, pgx.ErrNoRows) {
+		return PredefinedRecord{}, false, nil
+	}
+	if err != nil {
+		return PredefinedRecord{}, false, err
+	}
+	if r.Keys, err = catalogue.ParsePredefined(text); err != nil {
+		return PredefinedRecord{}, false, fmt.Errorf("the list kept from %s cannot be read: %w", r.Build, err)
+	}
+	return r, true, nil
+}

internal/inventory/predefined_test.go

@@ -0,0 +1,69 @@
+package inventory
+
+import (
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// The list of predefined keys the controller keeps (novox/hq ADR 0348 §7 and its check "a newer trunk build's
+// list replaces the published one, and an older build's list heard later does not"; design 57 package K2).
+
+func listOf(t *testing.T, text string) catalogue.PredefinedList {
+	t.Helper()
+	l, err := catalogue.ParsePredefined([]byte(text))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return l
+}
+
+func TestANewerBuildsListReplacesTheKeptOneAndAnOlderOneHeardLaterDoesNot(t *testing.T) {
+	inv := ForTest(t)
+	ctx := t.Context()
+	if _, has, err := inv.PredefinedKept(ctx); err != nil || has {
+		t.Fatalf("a fresh store keeps a list: %v %v", has, err)
+	}
+	older := PredefinedRecord{Keys: listOf(t, `{"font-size": {"type": "number", "description": "points"}}`),
+		Repository: "novox/mesh-catalog", Commit: "aaaa", Generation: 100, Build: "b-older"}
+	newer := PredefinedRecord{Keys: listOf(t, `{"font-size": {"type": "number", "description": "points"},
+		"agent-managed-settings": {"type": "object", "description": "a JSON object", "trusted": true}}`),
+		Repository: "novox/mesh-catalog", Commit: "bbbb", Generation: 101, Build: "b-newer"}
+
+	if kept, err := inv.KeepPredefined(ctx, older); err != nil || !kept {
+		t.Fatalf("the first list was not kept: %v %v", kept, err)
+	}
+	if kept, err := inv.KeepPredefined(ctx, newer); err != nil || !kept {
+		t.Fatalf("a newer build's list did not replace the kept one: %v %v", kept, err)
+	}
+	// The older build's outcome heard after the newer one's.
+	if kept, err := inv.KeepPredefined(ctx, older); err != nil || kept {
+		t.Fatalf("an older build's list heard later replaced the newer one: %v %v", kept, err)
+	}
+	// Nor one of the same generation: the one kept stands.
+	same := newer
+	same.Keys, same.Build = older.Keys, "b-again"
+	if kept, err := inv.KeepPredefined(ctx, same); err != nil || kept {
+		t.Fatalf("a list of the same generation replaced the kept one: %v %v", kept, err)
+	}
+	got, has, err := inv.PredefinedKept(ctx)
+	if err != nil || !has {
+		t.Fatal(has, err)
+	}
+	if got.Build != "b-newer" || got.Commit != "bbbb" || got.Generation != 101 || len(got.Keys) != 2 ||
+		!got.Keys["agent-managed-settings"].Trusted {
+		t.Fatalf("kept %+v", got)
+	}
+
+	// A build that could not say its generation, or carried a wrong list, is never kept.
+	unordered := newer
+	unordered.Generation = 0
+	if kept, err := inv.KeepPredefined(ctx, unordered); err == nil || kept {
+		t.Fatalf("a list without a generation was kept: %v", err)
+	}
+	wrong := newer
+	wrong.Generation, wrong.Keys = 200, catalogue.PredefinedList{"accent": {Type: "object", Description: "x"}}
+	if kept, err := inv.KeepPredefined(ctx, wrong); err == nil || kept {
+		t.Fatalf("a wrong list was kept: %v", err)
+	}
+}

internal/link/build.go

@@ -358,6 +358,12 @@ type BuildResult struct {
 	ReusedFrom *ReusedFrom `json:"reused-from,omitempty"`
 	BuiltWhy   string      `json:"built-why,omitempty"`
 
+	// Predefined is the list of predefined keys at the root of the repository at the commit built (novox/hq ADR
+	// 0348 §7), which the manifest was judged against: the controller publishes the catalogue's from a trunk
+	// build, the newest by Generation winning. Absent for a repository holding none, and from a builder that
+	// predates the list.
+	Predefined *PredefinedList `json:"predefined,omitempty"`
+
 	// Source is the request's, echoed: the seat form of the repository, for whoever registers.
 	Source *SourceOnSeat `json:"source,omitempty"`
 
@@ -370,6 +376,14 @@ type BuildResult struct {
 	Checked *CheckRequest `json:"check,omitempty"`
 }
 
+// PredefinedList is the list of predefined keys a build read (novox/hq ADR 0348 §7): Keys its canonical text, a
+// JSON object of keys, and Generation how many commits the commit built stands on — of two commits on one trunk
+// the later has more, which is what orders two builds' lists; 0 when the builder could not say.
+type PredefinedList struct {
+	Keys       json.RawMessage `json:"keys"`
+	Generation int             `json:"generation,omitempty"`
+}
+
 // ReadRepository is a repository a build read source from besides the module's own, at the branch,
 // tag or commit it read. Spelled here as well as in the catalogue and the inventory, for the reason
 // MadeArtifact is: one direction of dependency.

testdata/store-tests

@@ -50,6 +50,7 @@
 ./internal/inventory TestOneAcceptForAProviderIsSealedToEveryConsumer
 ./internal/inventory TestASharedOwnSecretIssuedOutsideIsNeverMade
 ./internal/recordmirror TestNatsAnAssignmentAndARegistrationEachChangeOneKeyAndRaiseOneEvent
+./internal/inventory TestAModuleInTheConfigFormIsMirroredWithItsWholeDeclaration
 ./cmd/mesh-controller TestOneAcceptForAProviderIsPlacedForEveryConsumer
 ./cmd/mesh-controller TestReplay164
 ./cmd/mesh-controller TestAnApprovedLayerOpensTheWalkThatSendsIt
@@ -62,6 +63,9 @@
 ./cmd/mesh-controller TestAProviderElsewhereServesItsOwnComposedName
 ./cmd/mesh-controller TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne
 ./internal/inventory TestAReportOfABusyContainerIsKeptAndReadBack
+./internal/inventory TestANewerBuildsListReplacesTheKeptOneAndAnOlderOneHeardLaterDoesNot
+./cmd/mesh-controller TestTheMergeGateJudgesEachManifestAgainstTheListOfTheTree
+./cmd/mesh-controller TestRegistrationPublishesTheNewestTrunkListAndJudgesEveryModuleAgainstIt
 ./internal/inventory TestPlacementMovesOutOfTheSettingsLayers
 ./internal/inventory TestASettingRoutesPlacementBesideTheLayer
 ./cmd/mesh-controller TestADeclarationIsTheSameAfterPlacementMoves
@@ -76,4 +80,3 @@
 ./cmd/mesh-controller TestANoMoveTierWaitsForNoNewReport
 ./cmd/mesh-controller TestOneProviderDownAndThreeConsumersFailingAreOneCondition
 ./cmd/mesh-controller TestAConsumerFailingBesideAHealthyProviderIsItsOwn
-./internal/inventory TestAModuleInTheConfigFormIsMirroredWithItsWholeDeclaration