diff --git a/cmd/mesh-controller/build.go b/cmd/mesh-controller/build.go index 23cdcfd3..d4a64a38 100644 --- a/cmd/mesh-controller/build.go +++ b/cmd/mesh-controller/build.go @@ -696,6 +696,20 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu } } } + // **The list of predefined keys arrives with a build of the catalogue** (novox/hq issue 630): the + // builder reads predefined.json at the catalogue repository's root at the commit built, and it is kept + // before the module is registered, so a preference falling back to a key added in the same commit is + // judged against the list that adds it. Only from the catalogue: another repository names no list. + if len(result.Predefined) > 0 && isCatalogueRepository(result.Repository) { + list, err := catalogue.ParsePredefined(result.Predefined) + if err != nil { + return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On, + manifest.Module, short(result.Commit), err) + } + if err := inv.SetPredefined(ctx, list, result.Repository, result.Commit); err != nil { + return manifest, kept, err + } + } if err := inv.RegisterModule(ctx, manifest, recorded); err != nil { if errors.Is(err, inventory.ErrSuperseded) { return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", @@ -1025,3 +1039,16 @@ func buildLog(ctx context.Context, id string) error { } return nil } + +// catalogueRepository is the repository the catalogue is: the one whose predefined.json is the list of +// predefined keys (novox/hq issue 630). +const catalogueRepository = "mesh-catalog" + +// isCatalogueRepository is whether a repository, as a path on the forge or a URL, is the catalogue. +func isCatalogueRepository(repository string) bool { + trimmed := strings.TrimSuffix(strings.TrimSuffix(strings.TrimSpace(repository), "/"), ".git") + if i := strings.LastIndex(trimmed, "/"); i >= 0 { + trimmed = trimmed[i+1:] + } + return trimmed == catalogueRepository +} diff --git a/cmd/mesh-controller/check.go b/cmd/mesh-controller/check.go index 16e16ba6..c3f772ee 100644 --- a/cmd/mesh-controller/check.go +++ b/cmd/mesh-controller/check.go @@ -47,6 +47,13 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error { shelf := catalogue.Shelf{} faulted := map[string]bool{} failed := 0 + // The list of predefined keys a preference may fall back to (novox/hq issue 630): the predefined.json + // at the root of the repository the manifests are in, found above the first of them. + list, listAt, listErr := predefinedAbove(paths[0]) + if listErr != nil { + fmt.Fprintf(out, "%s: %v\n", listAt, listErr) + failed++ + } for _, path := range paths { raw, err := os.ReadFile(path) if err != nil { @@ -92,6 +99,20 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error { failed += len(named) faulted[m.Module] = true } + // A preference falls back only to a key on the list, with its type (novox/hq issue 630). + if named := catalogue.PredefinedNamed(m); len(named) > 0 { + if listAt == "" { + fmt.Fprintf(out, "note: %s: its preferences fall back to %s, and no %s was found above it; "+ + "registration judges them against the list the mesh holds\n", path, strings.Join(named, ", "), + catalogue.PredefinedFile) + } else if wrong := catalogue.PredefinedProblems(m, list); len(wrong) > 0 { + for _, p := range wrong { + fmt.Fprintf(out, "%s: %s\n", path, p) + } + failed += len(wrong) + faulted[m.Module] = true + } + } shelf[m.Module] = m } @@ -285,3 +306,27 @@ func manifestsUnder(dir string) ([]string, error) { sort.Strings(found) return found, err } + +// predefinedAbove reads the predefined.json found in the directory of a manifest or one above it, up to +// the repository's root (the directory holding .git). The path it was found at is "" when there is none. +func predefinedAbove(manifest string) (catalogue.Predefined, string, error) { + dir, err := filepath.Abs(filepath.Dir(manifest)) + if err != nil { + return nil, "", nil + } + for { + at := filepath.Join(dir, catalogue.PredefinedFile) + if raw, err := os.ReadFile(at); err == nil { + list, err := catalogue.ParsePredefined(raw) + return list, at, err + } + if _, err := os.Stat(filepath.Join(dir, ".git")); err == nil { + return nil, "", nil + } + up := filepath.Dir(dir) + if up == dir { + return nil, "", nil + } + dir = up + } +} diff --git a/cmd/mesh-controller/facts.go b/cmd/mesh-controller/facts.go index 50b891cf..e0b92c53 100644 --- a/cmd/mesh-controller/facts.go +++ b/cmd/mesh-controller/facts.go @@ -263,6 +263,37 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot f.Controller.Commit = c.Commit } + // The list of predefined keys, and mesh and node config (novox/hq issue 630). + predefined, err := inv.Predefined(ctx) + if err != nil { + return snapshot.Facts{}, err + } + for key, k := range predefined { + if f.Predefined == nil { + f.Predefined = map[string]snapshot.PredefinedKey{} + } + f.Predefined[key] = snapshot.PredefinedKey{Type: k.Type, Why: k.Why} + } + config, err := inv.Config(ctx) + if err != nil { + return snapshot.Facts{}, err + } + nodeConfig := map[string]map[string]any{} + meshConfig := map[string]any{} + for _, c := range config { + if c.Node == "" { + meshConfig[c.Key] = c.Value + continue + } + if nodeConfig[c.Node] == nil { + nodeConfig[c.Node] = map[string]any{} + } + nodeConfig[c.Node][c.Key] = c.Value + } + if len(meshConfig) > 0 { + f.Config = scrub.Values(meshConfig) + } + gens, gensErr := generators(ctx, open) hostShelf := shelf[hostModule] meshWide := map[string]bool{} @@ -280,6 +311,9 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot if n.HostVersion != "" { engines[n.HostVersion] = true } + if values := nodeConfig[n.Name]; len(values) > 0 { + m.Config = scrub.Values(values) + } m.System = systemOf(hostShelf, n.HostVersion) m.Libc = libcOf(m.System) reported, err := inv.DescribedOf(ctx, n.Name) @@ -345,6 +379,10 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot return snapshot.Facts{}, err } for _, layer := range layers { + if layer.Config { + // Mesh and node config are said once, beside the machine and the mesh (novox/hq issue 630). + continue + } if layer.From == catalogue.MeshWideLayer { if !meshWide[module] { meshWide[module] = true diff --git a/cmd/mesh-controller/merge_gate.go b/cmd/mesh-controller/merge_gate.go index ddcfd609..0e614fd2 100644 --- a/cmd/mesh-controller/merge_gate.go +++ b/cmd/mesh-controller/merge_gate.go @@ -296,6 +296,24 @@ func judgeChange(ctx context.Context, in mergeCheckInput) (mergeVerdict, error) } v.Failures = append(v.Failures, failures...) } + // The list of predefined keys a head's catalogue tree carries, which the change composes with + // (novox/hq issue 630); the mesh's own list otherwise. + var changedList catalogue.Predefined + for _, h := range heads { + if h.Tree == "" { + continue + } + raw, err := os.ReadFile(filepath.Join(h.Tree, catalogue.PredefinedFile)) + if err != nil { + continue + } + list, err := catalogue.ParsePredefined(raw) + if err != nil { + v.Failures = append(v.Failures, err.Error()) + continue + } + changedList = list + } if len(in.group) > 0 { say("a delivery group: %d head(s) composed together", len(heads)) } @@ -319,7 +337,7 @@ func judgeChange(ctx context.Context, in mergeCheckInput) (mergeVerdict, error) // Both compositions, each in a store of its own. say("composing every machine as the mesh is") - before, _, notes, err := composeEveryMachine(ctx, in, base, sources, nil) + before, _, notes, err := composeEveryMachine(ctx, in, base, sources, nil, nil) if err != nil { return v, fmt.Errorf("the mesh as the snapshot says it is could not be raised: %w", err) } @@ -332,7 +350,7 @@ func judgeChange(ctx context.Context, in mergeCheckInput) (mergeVerdict, error) } } sort.Strings(added) - after, tried, notes, err := composeEveryMachine(ctx, in, change, sources, added) + after, tried, notes, err := composeEveryMachine(ctx, in, change, sources, added, changedList) if err != nil { return v, fmt.Errorf("the mesh with the change could not be raised: %w", err) } @@ -748,13 +766,13 @@ func standInBuild(m catalogue.Manifest) (catalogue.Manifest, error) { // grant to compose — and answers each machine's composition. Notes are what of the snapshot could not // be raised: a refusal to register a module, keep a setting or a pin. func composeEveryMachine(ctx context.Context, in mergeCheckInput, shelf map[string]catalogue.Manifest, - sources map[string]inventory.Source, trials []string) (map[string]mergeComposed, map[string]mergeTrial, []string, error) { + sources map[string]inventory.Source, trials []string, list catalogue.Predefined) (map[string]mergeComposed, map[string]mergeTrial, []string, error) { open, drop, err := throwawayStores(ctx, in.admin) if err != nil { return nil, nil, nil, err } defer drop() - notes, err := raiseFromFacts(ctx, open, in.facts, shelf, sources) + notes, err := raiseFromFacts(ctx, open, in.facts, shelf, sources, list) if err != nil { return nil, nil, notes, err } @@ -950,9 +968,22 @@ func throwawayStores(ctx context.Context, admin string) (*stores, func(), error) // for each standing in for the one it holds, its assignments, the seats it holds, its pins and its // settings, and a stand-in for every secret a person gave. func raiseFromFacts(ctx context.Context, open *stores, f snapshot.Facts, shelf map[string]catalogue.Manifest, - sources map[string]inventory.Source) ([]string, error) { + sources map[string]inventory.Source, list catalogue.Predefined) ([]string, error) { inv := open.inventory var notes []string + // The list of predefined keys before any module, which registration judges its preferences against + // (novox/hq issue 630): the change's own when its catalogue tree carries one, the mesh's otherwise. + if list == nil && len(f.Predefined) > 0 { + list = catalogue.Predefined{} + for key, k := range f.Predefined { + list[key] = catalogue.PredefinedKey{Type: k.Type, Why: k.Why} + } + } + if len(list) > 0 { + if err := inv.SetPredefined(ctx, list, "", ""); err != nil { + notes = append(notes, fmt.Sprintf("the list of predefined keys is not kept: %s", oneLine(err.Error()))) + } + } names := make([]string, 0, len(shelf)) for name := range shelf { names = append(names, name) @@ -1045,6 +1076,20 @@ func raiseFromFacts(ctx context.Context, open *stores, f snapshot.Facts, shelf m } } } + // Mesh and node config, as the mesh holds them (novox/hq issue 630). + for _, key := range sortedAnyKeys(f.Config) { + if err := inv.SetConfig(ctx, key, "", f.Config[key], "the facts snapshot", "raised from the facts snapshot"); err != nil { + notes = append(notes, fmt.Sprintf("the mesh config of %s is not kept: %s", key, oneLine(err.Error()))) + } + } + for _, m := range f.Machines { + for _, key := range sortedAnyKeys(m.Config) { + if err := inv.SetConfig(ctx, key, m.Name, m.Config[key], "the facts snapshot", "raised from the facts snapshot"); err != nil { + notes = append(notes, fmt.Sprintf("%s's node config of %s is not kept: %s", m.Described(), key, + oneLine(err.Error()))) + } + } + } for _, s := range f.Settings { if err := inv.KeepSettings(ctx, "", s.Module, standInPaths(s.Values)); err != nil { notes = append(notes, fmt.Sprintf("the mesh's settings of %s are not kept: %s", s.Module, oneLine(err.Error()))) @@ -1312,3 +1357,13 @@ func firstOr(items, otherwise []string) string { } return "it says nothing more" } + +// sortedAnyKeys is a map's keys, sorted. +func sortedAnyKeys(in map[string]any) []string { + keys := make([]string, 0, len(in)) + for k := range in { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +}