Bring F3 up to K2 on its release branch (part 2 of 3)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/builder/predefined_test.go

@@ -0,0 +1,106 @@
+package builder
+
+import (
+	"context"
+	"slices"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// A build reads the list of predefined keys at the root of the repository at the commit it builds (novox/hq ADR
+// 0348 §7, design 57 package K2, issue 661): it refuses a list that is wrong, judges the manifest against it, and
+// carries it with the commit's generation for the controller to publish.
+
+const theFirstKeys = `{
+  "font-size": {"type": "number", "description": "the size text is shown in, in points", "minimum": 6, "maximum": 72},
+  "agent-managed-settings": {"type": "object", "description": "the managed settings of every agent session, a JSON object", "trusted": true}
+}`
+
+// fallingBack is a module of the catalogue whose font size falls back to the predefined key named.
+func fallingBack(key, kind string) string {
+	def := "10"
+	if kind == "string" {
+		def = `"10"`
+	}
+	return `{"module":"dunst","version":"1",
+  "build":{"artifacts":[{"name":"look","kind":"archive","from":"files"}]},
+  "config":{"font-size":{"type":"` + kind + `","description":"the notification's font size, in points",
+    "default":` + def + `,"predefined":"` + key + `"}},
+  "resources":[
+    {"id":"theme","type":"archive","path":"/opt/dunst","artifact":"look"},
+    {"id":"rc","type":"file","path":"/etc/xdg/dunst/dunstrc","content":"font = Inter ${config:font-size}\n"}]}`
+}
+
+func buildInTheCatalogue(t *testing.T, manifest string, list *string) (Result, error) {
+	t.Helper()
+	r := &recorded{contents: map[string]string{"modules/dunst/" + ManifestName: manifest,
+		"modules/dunst/files/theme.conf": "dark", "README.md": "the catalogue"}}
+	if list != nil {
+		r.contents[catalogue.PredefinedFile] = *list
+	}
+	return Build(context.Background(), onTrunk{recorded: r}.run, r, "https://forge.invalid/mesh-catalog.git",
+		"modules/dunst", "", t.TempDir(), nil, Npmrc{}, GitCredential{}, nil)
+}
+
+func TestABuildNamingAPredefinedKeyIsJudgedAgainstItsTreesListAndCarriesIt(t *testing.T) {
+	list := theFirstKeys
+	got, err := buildInTheCatalogue(t, fallingBack("font-size", "number"), &list)
+	if err != nil {
+		t.Fatalf("a module falling back to font-size did not build: %v", err)
+	}
+	if got.Predefined == nil || got.Predefined.Generation != 42 {
+		t.Fatalf("the list was not carried with its generation: %+v", got.Predefined)
+	}
+	carried, err := catalogue.ParsePredefined(got.Predefined.Keys)
+	if err != nil || len(carried) != 2 || carried["font-size"].Type != catalogue.ConfigNumber {
+		t.Fatalf("the list carried is %s (%v)", got.Predefined.Keys, err)
+	}
+	// A change to the list is a change to the module that was judged against it.
+	if len(got.Sources) == 0 || !slices.Contains(got.Sources[0].Paths, catalogue.PredefinedFile) {
+		t.Fatalf("its build source leaves the list out: %+v", got.Sources)
+	}
+
+	for _, c := range []struct{ manifest, says string }{
+		{fallingBack("font-scale", "number"), `"font-scale", which is not on the list`},
+		{fallingBack("font-size", "string"), "which is a number, and this value is a string"},
+	} {
+		if _, err := buildInTheCatalogue(t, c.manifest, &list); err == nil || !strings.Contains(err.Error(), c.says) {
+			t.Errorf("wanted the build refused saying %q, got %v", c.says, err)
+		}
+	}
+}
+
+func TestABuildRefusesAWrongList(t *testing.T) {
+	wrong := `{"font-size": {"type": "number", "description": "points", "unit": "pt"}}`
+	if _, err := buildInTheCatalogue(t, fallingBack("font-size", "number"), &wrong); err == nil ||
+		!strings.Contains(err.Error(), `unknown field "unit"`) {
+		t.Fatalf("a wrong list was built with: %v", err)
+	}
+}
+
+// A repository without the list builds as before, carrying none; a value naming a key there is refused.
+func TestABuildWithoutAListCarriesNone(t *testing.T) {
+	plain := strings.Replace(fallingBack("font-size", "number"), `,"predefined":"font-size"`, "", 1)
+	got, err := buildInTheCatalogue(t, plain, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got.Predefined != nil {
+		t.Fatalf("a list was carried from a tree holding none: %+v", got.Predefined)
+	}
+	if slices.Contains(got.Sources[0].Paths, catalogue.PredefinedFile) {
+		t.Fatalf("the build source names a list: %+v", got.Sources)
+	}
+	if _, err := buildInTheCatalogue(t, fallingBack("font-size", "number"), nil); err == nil ||
+		!strings.Contains(err.Error(), "is not present") {
+		t.Fatalf("a key named where no list is: %v", err)
+	}
+	// A list present and a module naming no key: carried, and not the module's build source.
+	list := theFirstKeys
+	got, err = buildInTheCatalogue(t, plain, &list)
+	if err != nil || got.Predefined == nil || slices.Contains(got.Sources[0].Paths, catalogue.PredefinedFile) {
+		t.Fatalf("a module naming no key: %+v %+v %v", got.Predefined, got.Sources, err)
+	}
+}

internal/builder/source.go

@@ -58,6 +58,9 @@ type sourceInputs struct {
 	// whole when an artifact's is not known (novox/hq ADR 0267).
 	own      map[string]bool
 	ownIsAll bool
+	// root is the module's build source in its own repository outside its directory, relative to the
+	// repository's root: the list of predefined keys a manifest naming one was judged against (ADR 0348 §7).
+	root map[string]bool
 	// read is, per context (repository and ref), the build source read there; nil for one read whole.
 	read   map[string]map[string]bool
 	readAs map[string]catalogue.ArtifactContext
@@ -86,6 +89,9 @@ func (s *sourceInputs) copy() *sourceInputs {
 	for k, v := range s.own {
 		c.own[k] = v
 	}
+	for k, v := range s.root {
+		c.root[k] = v
+	}
 	for k, v := range s.read {
 		if v == nil {
 			c.read[k] = nil
@@ -112,7 +118,7 @@ func (s *sourceInputs) contextOffHead() {
 
 func newSourceInputs(module string) *sourceInputs {
 	return &sourceInputs{module: module, contexts: map[string]string{}, toolchains: map[string]string{},
-		own: map[string]bool{}, read: map[string]map[string]bool{}, readAs: map[string]catalogue.ArtifactContext{},
+		own: map[string]bool{}, root: map[string]bool{}, read: map[string]map[string]bool{}, readAs: map[string]catalogue.ArtifactContext{},
 		contextsAtHead: true}
 }
 
@@ -126,6 +132,16 @@ func (s *sourceInputs) ownHas(entries ...string) {
 	}
 }
 
+// rootHas adds entries, relative to the repository's root, to the module's build source in its own repository.
+func (s *sourceInputs) rootHas(entries ...string) {
+	if s == nil {
+		return
+	}
+	for _, e := range entries {
+		s.root[e] = true
+	}
+}
+
 // ownWhole says an artifact's build source in the module's own repository is not known: the module's
 // whole directory is its source, as it was before.
 func (s *sourceInputs) ownWhole() {
@@ -178,6 +194,9 @@ func (s *sourceInputs) buildSources() []BuildSource {
 		for e := range s.own {
 			own = append(own, withPrefix(s.prefix, e))
 		}
+		for e := range s.root {
+			own = append(own, e)
+		}
 		sort.Strings(own)
 		out = append(out, BuildSource{Paths: compactSorted(own)})
 	}

internal/catalogue/config.go

@@ -61,18 +61,6 @@ const (
 // accountFacts are what the mesh places for a module's account, never config values (ADR 0348 §11).
 var accountFacts = map[string]bool{"puid": true, "pgid": true}
 
-// PredefinedKey is one key of the catalogue's list of predefined keys, as far as a module's value falling back
-// to it is judged here: its type, and whether it is trusted (ADR 0348 §7, §8).
-type PredefinedKey struct {
-	Type    string
-	Trusted bool
-}
-
-// predefinedKeys is the list of predefined keys, nil until it is present. The list (`predefined.json` at the
-// catalogue's root) arrives with design 57's package K2 (issue 661); until then a value naming one is refused
-// saying the list is not yet present.
-var predefinedKeys map[string]PredefinedKey
-
 // configForms is which of the two forms a manifest was written in, by the words of each it uses.
 type configForms struct {
 	config, settings []string
@@ -259,12 +247,13 @@ func ConfigProblems(m Manifest) []string {
 				"(novox/hq ADR 0348 §4, ADR 0112)", word)
 		}
 		if d.Predefined != "" {
-			p, listed := predefinedKeys[d.Predefined]
+			p, listed := m.predefined[d.Predefined]
 			switch {
-			case predefinedKeys == nil:
+			case m.predefined == nil:
-				say("falls back to the predefined key %q, and the list of predefined keys (the catalogue's "+
+				say("falls back to the predefined key %q, and the list of predefined keys (%s at the root of "+
-					"predefined.json, design 57 package K2) is not yet present: no value can fall back to one "+
+					"the catalogue's repository) is not present where it is judged: a value falls back only to a "+
-					"until it is; leave \"predefined\" out for now (novox/hq ADR 0348 §7)", d.Predefined)
+					"key on that list; judge it in the catalogue, or leave \"predefined\" out (novox/hq ADR 0348 §7)",
+					d.Predefined, PredefinedFile)
 			case !listed:
 				say("falls back to %q, which is not on the list of predefined keys: name a key on it, or leave "+
 					"\"predefined\" out (novox/hq ADR 0348 §7)", d.Predefined)

internal/catalogue/config_test.go

@@ -157,20 +157,20 @@ func TestConfigRefusesAnObjectOrListNotTrusted(t *testing.T) {
 	}
 }
 
-// withPredefined is the list of predefined keys as package K2 will give it, for the length of a test.
+// withPredefined is the list of predefined keys the mesh published, for the length of a test.
-func withPredefined(t *testing.T, keys map[string]PredefinedKey) {
+func withPredefined(t *testing.T, keys PredefinedList) {
 	t.Helper()
-	was := predefinedKeys
+	was := ReplacePublished(keys)
-	predefinedKeys = keys
+	t.Cleanup(func() { ReplacePublished(was) })
-	t.Cleanup(func() { predefinedKeys = was })
 }
 
-// Rule 5: a predefined key not on the list, or of another type. Until the list is present (package K2), a value
+// Rule 5: a predefined key not on the list, or of another type. Where no list is present, a value naming one is
-// naming one is refused saying so.
+// refused saying so.
 func TestConfigRefusesAPredefinedKeyNotOnTheListOrOfAnotherType(t *testing.T) {
+	withPredefined(t, nil)
 	m := inConfigForm()
 	declaration(m, "font-size")["predefined"] = "font-size"
-	refused(t, m, `"font-size"`, "the list of predefined keys", "not yet present")
+	refused(t, m, `"font-size"`, "the list of predefined keys", "is not present")
 
 	withPredefined(t, map[string]PredefinedKey{"font-size": {Type: "number"}, "accent": {Type: "string"}})
 	if _, err := parseRaw(t, m); err != nil {
@@ -186,9 +186,10 @@ func TestConfigRefusesAPredefinedKeyNotOnTheListOrOfAnotherType(t *testing.T) {
 
 // Rule 6: an untrusted value falling back to a trusted predefined key.
 func TestConfigRefusesAnUntrustedValueFallingBackToATrustedKey(t *testing.T) {
+	withPredefined(t, nil)
 	m := inConfigForm()
 	declaration(m, "theme")["predefined"] = "theme"
-	refused(t, m, `"theme"`, "not yet present")
+	refused(t, m, `"theme"`, "is not present")
 
 	withPredefined(t, map[string]PredefinedKey{"theme": {Type: "string", Trusted: true}})
 	refused(t, m, `"theme"`, "trusted", "is not")

internal/catalogue/manifest.go

@@ -541,6 +541,10 @@ type Manifest struct {
 	// machine's declaration by name until the controller is updated (LeftOut).
 	unknown string
 
+	// predefined is the list of predefined keys it was judged against (predefined.go, novox/hq ADR 0348 §7):
+	// read again — its templates filled from the tree — it is judged against the same list.
+	predefined PredefinedList
+
 	// bestEffort marks the view of a left-out module that only its backup lines are made from
 	// (backupView, novox/hq ADR 0262): a line of it that cannot be placed is said in the plan's list of
 	// what could not be placed, never an error that would cost the whole machine its declaration.
@@ -1600,6 +1604,17 @@ func SecretLocal(to, local string) string {
 func (m Manifest) UnknownField() string { return m.unknown }
 
 func ParseManifest(raw []byte) (Manifest, error) {
+	return parseManifest(raw, PublishedPredefined())
+}
+
+// ParseManifestWith is ParseManifest judging every value that falls back to a predefined key against the list
+// given — the one read from the tree the manifest is in (predefined.go, novox/hq ADR 0348 §7) — where
+// ParseManifest judges against the list the mesh published. Nil is no list: a value naming a key is refused.
+func ParseManifestWith(raw []byte, predefined PredefinedList) (Manifest, error) {
+	return parseManifest(raw, predefined)
+}
+
+func parseManifest(raw []byte, predefined PredefinedList) (Manifest, error) {
 	var m Manifest
 	// Strictly. **An unknown key is refused**, which is the discipline the host's declaration
 	// parser has and manifests lacked (novox/hq 04-ISSUES/003): a `scope:` key survived in five
@@ -1627,6 +1642,7 @@ func ParseManifest(raw []byte) (Manifest, error) {
 		}
 		return Manifest{}, fmt.Errorf("this is not a module manifest: %w", err)
 	}
+	m.predefined = predefined
 
 	var problems []string
 	if !name.MatchString(m.Module) {

internal/catalogue/predefined.go

@@ -0,0 +1,236 @@
+package catalogue
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+)
+
+// The list of predefined keys (novox/hq ADR 0348 §7, design 57 package K2, issue 661).
+//
+// **A fixed, reviewable list, owned by the build.** Mesh config and node config hold values only for the keys
+// in the catalogue's `predefined.json`, at the root of its repository: each key with its type, a description
+// including its unit, its validation (enum, minimum and maximum, pattern) and whether it is trusted. A module's
+// value may fall back to one of them, of the same type (ADR 0348 §2, §4).
+//
+// **Read where a manifest is judged, from the tree it is judged in.** A build reads the file at the root of the
+// repository at the commit it builds, `module check` at the root of the repository above each manifest, the
+// merge gate at the root of the tree under check; each refuses a file that is not exactly a list of keys, and
+// judges every manifest it reads against it (ParseManifestWith). Everywhere else — registration, a stored build
+// read again — a manifest is judged against the list the mesh published (PublishedPredefined): the newest trunk
+// build's of the catalogue, which the controller keeps and loads with its store.
+
+// PredefinedFile is the list's file at the root of the catalogue's repository.
+const PredefinedFile = "predefined.json"
+
+// CatalogueRepository is the name of the repository whose list the mesh publishes: the catalogue's, by the name
+// a check finds it beside under (internal/builder's check, ADR 0121).
+const CatalogueRepository = "mesh-catalog"
+
+// PredefinedKey is one key of the list: its type, its description, its validation and whether it is trusted
+// (ADR 0348 §7). It has no default: a module's value falling back to it has its own.
+type PredefinedKey struct {
+	Type        string   `json:"type"`
+	Description string   `json:"description"`
+	Enum        []any    `json:"enum,omitempty"`
+	Minimum     *float64 `json:"minimum,omitempty"`
+	Maximum     *float64 `json:"maximum,omitempty"`
+	Pattern     string   `json:"pattern,omitempty"`
+	Trusted     bool     `json:"trusted,omitempty"`
+}
+
+// PredefinedList is the list by key. Nil is no list at all, which is not the same as an empty one: a value
+// naming a key is refused then saying the list is not present.
+type PredefinedList map[string]PredefinedKey
+
+// ParsePredefined reads predefined.json strictly: one JSON object of keys, each with exactly the fields of a
+// PredefinedKey, and every key judged as a module's config value is (PredefinedProblems). Anything else is
+// refused whole, naming each problem — a list half read would let a module fall back to a key nobody reviewed.
+func ParsePredefined(raw []byte) (PredefinedList, error) {
+	var keys map[string]json.RawMessage
+	if err := json.Unmarshal(raw, &keys); err != nil {
+		return nil, fmt.Errorf("%s is not a JSON object of keys: %w", PredefinedFile, err)
+	}
+	if keys == nil {
+		return nil, fmt.Errorf("%s is not a JSON object of keys: it says null", PredefinedFile)
+	}
+	list := PredefinedList{}
+	var problems []string
+	for _, key := range sortedKeys(keys) {
+		decoder := json.NewDecoder(bytes.NewReader(keys[key]))
+		decoder.DisallowUnknownFields()
+		var k PredefinedKey
+		if err := decoder.Decode(&k); err != nil {
+			problems = append(problems, fmt.Sprintf("%q is not a predefined key's declaration (type, description, "+
+				"enum, minimum, maximum, pattern, trusted): %v", key, err))
+			continue
+		}
+		list[key] = k
+	}
+	problems = append(problems, PredefinedProblems(list)...)
+	if len(problems) > 0 {
+		return nil, fmt.Errorf("%s is refused: %s", PredefinedFile, strings.Join(problems, "; "))
+	}
+	return list, nil
+}
+
+// PredefinedProblems is every way a key of the list is declared wrongly, by the rules a module's config value
+// is held to (ADR 0348 §4, §7, §11): a usable name that is not placement, a type, a description, validation
+// that fits the type, and an object or a list only when trusted.
+func PredefinedProblems(list PredefinedList) []string {
+	var problems []string
+	for _, key := range sortedKeys(list) {
+		k := list[key]
+		say := func(format string, args ...any) {
+			problems = append(problems, fmt.Sprintf("the predefined key %q: ", key)+fmt.Sprintf(format, args...))
+		}
+		if !settingRef.MatchString("${setting:" + key + "}") {
+			say("not a usable key: lower-case letters, digits, dots, dashes and underscores")
+			continue
+		}
+		if meshWords[key] || accountFacts[key] {
+			say("%q is placement, which the mesh decides, never a config value (novox/hq ADR 0348 §11)", key)
+			continue
+		}
+		switch k.Type {
+		case ConfigNumber, ConfigString, ConfigBoolean:
+		case ConfigObject, ConfigList:
+			if !k.Trusted {
+				say("is %s and not trusted: an object or a list is trusted, \"trusted\": true (novox/hq ADR 0348 §4)", k.Type)
+			}
+		case "":
+			say("no type: say \"type\": number, string or boolean, or object or list when trusted (novox/hq ADR 0348 §4)")
+		default:
+			say("type %q is none of number, string, boolean, object and list (novox/hq ADR 0348 §4)", k.Type)
+		}
+		if strings.TrimSpace(k.Description) == "" {
+			say("no description: say in a sentence what the value is, including its unit (novox/hq ADR 0348 §7)")
+		}
+		for _, wrong := range validationProblems(SettingDeclaration{Type: k.Type, Enum: k.Enum, Minimum: k.Minimum,
+			Maximum: k.Maximum, Pattern: k.Pattern}) {
+			say("%s", wrong)
+		}
+	}
+	return problems
+}
+
+// ReadPredefined is the list in predefined.json at a repository's root, and nil with no error when the root
+// holds none: a repository other than the catalogue, or the catalogue before the file.
+func ReadPredefined(root string) (PredefinedList, error) {
+	path := filepath.Join(root, PredefinedFile)
+	info, err := os.Lstat(path)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	if !info.Mode().IsRegular() {
+		return nil, fmt.Errorf("%s is %s, not a plain file", path, kindOf(info.Mode()))
+	}
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	return ParsePredefined(raw)
+}
+
+// RepositoryRoot is the nearest directory at or above dir holding `.git`: the root of the repository a manifest
+// is in, where `module check` reads the list. Empty when none is: a manifest outside a repository is judged
+// against no list.
+func RepositoryRoot(dir string) string {
+	at, err := filepath.Abs(dir)
+	if err != nil {
+		return ""
+	}
+	for {
+		if _, err := os.Lstat(filepath.Join(at, ".git")); err == nil {
+			return at
+		}
+		up := filepath.Dir(at)
+		if up == at {
+			return ""
+		}
+		at = up
+	}
+}
+
+// PredefinedAbove is the list at the root of the repository above dir (RepositoryRoot), nil when there is no
+// repository or it holds no list.
+func PredefinedAbove(dir string) (PredefinedList, error) {
+	root := RepositoryRoot(dir)
+	if root == "" {
+		return nil, nil
+	}
+	return ReadPredefined(root)
+}
+
+// Canonical is the list as JSON with its keys sorted: what a build carries and the controller keeps, so one list
+// is one text.
+func (l PredefinedList) Canonical() ([]byte, error) {
+	if l == nil {
+		return nil, nil
+	}
+	return json.Marshal(map[string]PredefinedKey(l))
+}
+
+// Keys is the list's keys, sorted.
+func (l PredefinedList) Keys() []string {
+	out := make([]string, 0, len(l))
+	for k := range l {
+		out = append(out, k)
+	}
+	sort.Strings(out)
+	return out
+}
+
+// NamesPredefined is whether any value of the manifest falls back to a predefined key.
+func (m Manifest) NamesPredefined() bool {
+	for _, d := range m.Settings {
+		if d.Predefined != "" {
+			return true
+		}
+	}
+	return false
+}
+
+// publishedPredefined is the list the mesh published, which a manifest is judged against where no tree is read.
+var publishedPredefined struct {
+	sync.RWMutex
+	list PredefinedList
+}
+
+// UsePredefined makes list the one a manifest is judged against where no tree is read: what the controller
+// loads from its store, and what it keeps when a newer trunk build of the catalogue publishes one. Nil leaves
+// the list it has: a store that holds none yet never empties one already loaded.
+func UsePredefined(list PredefinedList) {
+	if list == nil {
+		return
+	}
+	publishedPredefined.Lock()
+	defer publishedPredefined.Unlock()
+	publishedPredefined.list = list
+}
+
+// PublishedPredefined is the list a manifest is judged against where no tree is read; nil when none is loaded.
+func PublishedPredefined() PredefinedList {
+	publishedPredefined.RLock()
+	defer publishedPredefined.RUnlock()
+	return publishedPredefined.list
+}
+
+// ReplacePublished sets the published list, nil included, and answers the one it replaced: for a test that
+// judges against a list of its own and puts the mesh's back.
+func ReplacePublished(list PredefinedList) (was PredefinedList) {
+	publishedPredefined.Lock()
+	defer publishedPredefined.Unlock()
+	was, publishedPredefined.list = publishedPredefined.list, list
+	return was
+}