Keep the list from a catalogue build, and carry it and mesh and node config through the facts and the gate (hq issue 630)

The merge gate must compose with the same config the mesh holds, and judge a catalogue change against the list it brings.

cmd/mesh-controller/build.go

@@ -696,6 +696,20 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
 			}
 		}
 	}
+	// **The list of predefined keys arrives with a build of the catalogue** (novox/hq issue 630): the
+	// builder reads predefined.json at the catalogue repository's root at the commit built, and it is kept
+	// before the module is registered, so a preference falling back to a key added in the same commit is
+	// judged against the list that adds it. Only from the catalogue: another repository names no list.
+	if len(result.Predefined) > 0 && isCatalogueRepository(result.Repository) {
+		list, err := catalogue.ParsePredefined(result.Predefined)
+		if err != nil {
+			return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On,
+				manifest.Module, short(result.Commit), err)
+		}
+		if err := inv.SetPredefined(ctx, list, result.Repository, result.Commit); err != nil {
+			return manifest, kept, err
+		}
+	}
 	if err := inv.RegisterModule(ctx, manifest, recorded); err != nil {
 		if errors.Is(err, inventory.ErrSuperseded) {
 			return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w",
@@ -1025,3 +1039,16 @@ func buildLog(ctx context.Context, id string) error {
 	}
 	return nil
 }
+
+// catalogueRepository is the repository the catalogue is: the one whose predefined.json is the list of
+// predefined keys (novox/hq issue 630).
+const catalogueRepository = "mesh-catalog"
+
+// isCatalogueRepository is whether a repository, as a path on the forge or a URL, is the catalogue.
+func isCatalogueRepository(repository string) bool {
+	trimmed := strings.TrimSuffix(strings.TrimSuffix(strings.TrimSpace(repository), "/"), ".git")
+	if i := strings.LastIndex(trimmed, "/"); i >= 0 {
+		trimmed = trimmed[i+1:]
+	}
+	return trimmed == catalogueRepository
+}

cmd/mesh-controller/check.go

@@ -47,6 +47,13 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
 	shelf := catalogue.Shelf{}
 	faulted := map[string]bool{}
 	failed := 0
+	// The list of predefined keys a preference may fall back to (novox/hq issue 630): the predefined.json
+	// at the root of the repository the manifests are in, found above the first of them.
+	list, listAt, listErr := predefinedAbove(paths[0])
+	if listErr != nil {
+		fmt.Fprintf(out, "%s: %v\n", listAt, listErr)
+		failed++
+	}
 	for _, path := range paths {
 		raw, err := os.ReadFile(path)
 		if err != nil {
@@ -92,6 +99,20 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
 			failed += len(named)
 			faulted[m.Module] = true
 		}
+		// A preference falls back only to a key on the list, with its type (novox/hq issue 630).
+		if named := catalogue.PredefinedNamed(m); len(named) > 0 {
+			if listAt == "" {
+				fmt.Fprintf(out, "note: %s: its preferences fall back to %s, and no %s was found above it; "+
+					"registration judges them against the list the mesh holds\n", path, strings.Join(named, ", "),
+					catalogue.PredefinedFile)
+			} else if wrong := catalogue.PredefinedProblems(m, list); len(wrong) > 0 {
+				for _, p := range wrong {
+					fmt.Fprintf(out, "%s: %s\n", path, p)
+				}
+				failed += len(wrong)
+				faulted[m.Module] = true
+			}
+		}
 		shelf[m.Module] = m
 	}
 
@@ -285,3 +306,27 @@ func manifestsUnder(dir string) ([]string, error) {
 	sort.Strings(found)
 	return found, err
 }
+
+// predefinedAbove reads the predefined.json found in the directory of a manifest or one above it, up to
+// the repository's root (the directory holding .git). The path it was found at is "" when there is none.
+func predefinedAbove(manifest string) (catalogue.Predefined, string, error) {
+	dir, err := filepath.Abs(filepath.Dir(manifest))
+	if err != nil {
+		return nil, "", nil
+	}
+	for {
+		at := filepath.Join(dir, catalogue.PredefinedFile)
+		if raw, err := os.ReadFile(at); err == nil {
+			list, err := catalogue.ParsePredefined(raw)
+			return list, at, err
+		}
+		if _, err := os.Stat(filepath.Join(dir, ".git")); err == nil {
+			return nil, "", nil
+		}
+		up := filepath.Dir(dir)
+		if up == dir {
+			return nil, "", nil
+		}
+		dir = up
+	}
+}

cmd/mesh-controller/facts.go

@@ -263,6 +263,37 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot
 		f.Controller.Commit = c.Commit
 	}
 
+	// The list of predefined keys, and mesh and node config (novox/hq issue 630).
+	predefined, err := inv.Predefined(ctx)
+	if err != nil {
+		return snapshot.Facts{}, err
+	}
+	for key, k := range predefined {
+		if f.Predefined == nil {
+			f.Predefined = map[string]snapshot.PredefinedKey{}
+		}
+		f.Predefined[key] = snapshot.PredefinedKey{Type: k.Type, Why: k.Why}
+	}
+	config, err := inv.Config(ctx)
+	if err != nil {
+		return snapshot.Facts{}, err
+	}
+	nodeConfig := map[string]map[string]any{}
+	meshConfig := map[string]any{}
+	for _, c := range config {
+		if c.Node == "" {
+			meshConfig[c.Key] = c.Value
+			continue
+		}
+		if nodeConfig[c.Node] == nil {
+			nodeConfig[c.Node] = map[string]any{}
+		}
+		nodeConfig[c.Node][c.Key] = c.Value
+	}
+	if len(meshConfig) > 0 {
+		f.Config = scrub.Values(meshConfig)
+	}
+
 	gens, gensErr := generators(ctx, open)
 	hostShelf := shelf[hostModule]
 	meshWide := map[string]bool{}
@@ -280,6 +311,9 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot
 		if n.HostVersion != "" {
 			engines[n.HostVersion] = true
 		}
+		if values := nodeConfig[n.Name]; len(values) > 0 {
+			m.Config = scrub.Values(values)
+		}
 		m.System = systemOf(hostShelf, n.HostVersion)
 		m.Libc = libcOf(m.System)
 		reported, err := inv.DescribedOf(ctx, n.Name)
@@ -345,6 +379,10 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot
 				return snapshot.Facts{}, err
 			}
 			for _, layer := range layers {
+				if layer.Config {
+					// Mesh and node config are said once, beside the machine and the mesh (novox/hq issue 630).
+					continue
+				}
 				if layer.From == catalogue.MeshWideLayer {
 					if !meshWide[module] {
 						meshWide[module] = true

cmd/mesh-controller/merge_gate.go

@@ -296,6 +296,24 @@ func judgeChange(ctx context.Context, in mergeCheckInput) (mergeVerdict, error)
 		}
 		v.Failures = append(v.Failures, failures...)
 	}
+	// The list of predefined keys a head's catalogue tree carries, which the change composes with
+	// (novox/hq issue 630); the mesh's own list otherwise.
+	var changedList catalogue.Predefined
+	for _, h := range heads {
+		if h.Tree == "" {
+			continue
+		}
+		raw, err := os.ReadFile(filepath.Join(h.Tree, catalogue.PredefinedFile))
+		if err != nil {
+			continue
+		}
+		list, err := catalogue.ParsePredefined(raw)
+		if err != nil {
+			v.Failures = append(v.Failures, err.Error())
+			continue
+		}
+		changedList = list
+	}
 	if len(in.group) > 0 {
 		say("a delivery group: %d head(s) composed together", len(heads))
 	}
@@ -319,7 +337,7 @@ func judgeChange(ctx context.Context, in mergeCheckInput) (mergeVerdict, error)
 
 	// Both compositions, each in a store of its own.
 	say("composing every machine as the mesh is")
-	before, _, notes, err := composeEveryMachine(ctx, in, base, sources, nil)
+	before, _, notes, err := composeEveryMachine(ctx, in, base, sources, nil, nil)
 	if err != nil {
 		return v, fmt.Errorf("the mesh as the snapshot says it is could not be raised: %w", err)
 	}
@@ -332,7 +350,7 @@ func judgeChange(ctx context.Context, in mergeCheckInput) (mergeVerdict, error)
 		}
 	}
 	sort.Strings(added)
-	after, tried, notes, err := composeEveryMachine(ctx, in, change, sources, added)
+	after, tried, notes, err := composeEveryMachine(ctx, in, change, sources, added, changedList)
 	if err != nil {
 		return v, fmt.Errorf("the mesh with the change could not be raised: %w", err)
 	}
@@ -748,13 +766,13 @@ func standInBuild(m catalogue.Manifest) (catalogue.Manifest, error) {
 // grant to compose — and answers each machine's composition. Notes are what of the snapshot could not
 // be raised: a refusal to register a module, keep a setting or a pin.
 func composeEveryMachine(ctx context.Context, in mergeCheckInput, shelf map[string]catalogue.Manifest,
-	sources map[string]inventory.Source, trials []string) (map[string]mergeComposed, map[string]mergeTrial, []string, error) {
+	sources map[string]inventory.Source, trials []string, list catalogue.Predefined) (map[string]mergeComposed, map[string]mergeTrial, []string, error) {
 	open, drop, err := throwawayStores(ctx, in.admin)
 	if err != nil {
 		return nil, nil, nil, err
 	}
 	defer drop()
-	notes, err := raiseFromFacts(ctx, open, in.facts, shelf, sources)
+	notes, err := raiseFromFacts(ctx, open, in.facts, shelf, sources, list)
 	if err != nil {
 		return nil, nil, notes, err
 	}
@@ -950,9 +968,22 @@ func throwawayStores(ctx context.Context, admin string) (*stores, func(), error)
 // for each standing in for the one it holds, its assignments, the seats it holds, its pins and its
 // settings, and a stand-in for every secret a person gave.
 func raiseFromFacts(ctx context.Context, open *stores, f snapshot.Facts, shelf map[string]catalogue.Manifest,
-	sources map[string]inventory.Source) ([]string, error) {
+	sources map[string]inventory.Source, list catalogue.Predefined) ([]string, error) {
 	inv := open.inventory
 	var notes []string
+	// The list of predefined keys before any module, which registration judges its preferences against
+	// (novox/hq issue 630): the change's own when its catalogue tree carries one, the mesh's otherwise.
+	if list == nil && len(f.Predefined) > 0 {
+		list = catalogue.Predefined{}
+		for key, k := range f.Predefined {
+			list[key] = catalogue.PredefinedKey{Type: k.Type, Why: k.Why}
+		}
+	}
+	if len(list) > 0 {
+		if err := inv.SetPredefined(ctx, list, "", ""); err != nil {
+			notes = append(notes, fmt.Sprintf("the list of predefined keys is not kept: %s", oneLine(err.Error())))
+		}
+	}
 	names := make([]string, 0, len(shelf))
 	for name := range shelf {
 		names = append(names, name)
@@ -1045,6 +1076,20 @@ func raiseFromFacts(ctx context.Context, open *stores, f snapshot.Facts, shelf m
 			}
 		}
 	}
+	// Mesh and node config, as the mesh holds them (novox/hq issue 630).
+	for _, key := range sortedAnyKeys(f.Config) {
+		if err := inv.SetConfig(ctx, key, "", f.Config[key], "the facts snapshot", "raised from the facts snapshot"); err != nil {
+			notes = append(notes, fmt.Sprintf("the mesh config of %s is not kept: %s", key, oneLine(err.Error())))
+		}
+	}
+	for _, m := range f.Machines {
+		for _, key := range sortedAnyKeys(m.Config) {
+			if err := inv.SetConfig(ctx, key, m.Name, m.Config[key], "the facts snapshot", "raised from the facts snapshot"); err != nil {
+				notes = append(notes, fmt.Sprintf("%s's node config of %s is not kept: %s", m.Described(), key,
+					oneLine(err.Error())))
+			}
+		}
+	}
 	for _, s := range f.Settings {
 		if err := inv.KeepSettings(ctx, "", s.Module, standInPaths(s.Values)); err != nil {
 			notes = append(notes, fmt.Sprintf("the mesh's settings of %s are not kept: %s", s.Module, oneLine(err.Error())))
@@ -1312,3 +1357,13 @@ func firstOr(items, otherwise []string) string {
 	}
 	return "it says nothing more"
 }
+
+// sortedAnyKeys is a map's keys, sorted.
+func sortedAnyKeys(in map[string]any) []string {
+	keys := make([]string, 0, len(in))
+	for k := range in {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	return keys
+}