diff --git a/cmd/mesh-builder/main.go b/cmd/mesh-builder/main.go index 7bf565be..df5735ae 100644 --- a/cmd/mesh-builder/main.go +++ b/cmd/mesh-builder/main.go @@ -418,6 +418,11 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri for _, s := range built.Sources { result.Sources = append(result.Sources, link.BuildSource{Repository: s.Repository, Ref: s.Ref, Paths: s.Paths}) } + // The list of predefined keys the manifest was judged against, for the controller to publish + // (novox/hq ADR 0348 §7). + if p := built.Predefined; p != nil { + result.Predefined = &link.PredefinedList{Keys: p.Keys, Generation: p.Generation} + } say("built", built.Manifest.Module+" from "+short(built.Commit)) } } diff --git a/cmd/mesh-controller/build.go b/cmd/mesh-controller/build.go index d7eb9c2d..ae337692 100644 --- a/cmd/mesh-controller/build.go +++ b/cmd/mesh-controller/build.go @@ -183,8 +183,10 @@ func buildFrom(result link.BuildResult) inventory.Build { kept.Manifest = recordedManifest(result.Manifest, announced) // The module name comes from the manifest, which only exists when the build got that far. if len(kept.Manifest) > 0 { - if m, err := catalogue.ParseManifest(kept.Manifest); err == nil { - kept.Module = m.Module + if list, err := judgedWith(result); err == nil { + if m, err := catalogue.ParseManifestWith(kept.Manifest, list); err == nil { + kept.Module = m.Module + } } } return kept @@ -603,6 +605,11 @@ func saysWhenThePolicyActs(ctx context.Context, inv *inventory.Inventory, module // written with the same values. func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResult) ( catalogue.Manifest, inventory.Build, error) { + // The list of predefined keys as the store keeps it now: another controller process may have kept a newer one. + if err := loadPredefined(ctx, inv); err != nil { + return catalogue.Manifest{}, inventory.Build{}, fmt.Errorf("the list of predefined keys kept cannot be "+ + "read: %w", err) + } kept := buildFrom(result) if err := inv.RecordBuild(ctx, kept); err != nil { return catalogue.Manifest{}, kept, err @@ -613,7 +620,15 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu return catalogue.Manifest{}, kept, fmt.Errorf("%s could not build %s:\n%s", result.On, result.Repository, result.Failed) } - manifest, err := catalogue.ParseManifest(kept.Manifest) + // Judged first against the list the build was judged against: the catalogue's at the commit built (novox/hq + // ADR 0348 §7), so a module naming a key its own commit adds is read; and again below against the list the + // mesh publishes, before it is registered. + list, err := judgedWith(result) + if err != nil { + return catalogue.Manifest{}, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On, + result.Repository, short(result.Commit), err) + } + manifest, err := catalogue.ParseManifestWith(kept.Manifest, list) if err != nil { return catalogue.Manifest{}, kept, fmt.Errorf("%s built %s and what came back is not a manifest: %w", result.On, result.Repository, err) @@ -687,6 +702,23 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu "back: it is recorded and not registered again — a newer build is", result.On, manifest.Module, short(result.Commit)) } + // **The newest trunk build's list of predefined keys is published** (novox/hq ADR 0348 §7): a build of the + // catalogue that may be registered keeps its list when its commit is newer than the one kept. Then the module + // is judged against the list the mesh publishes, as every manifest read again is: one naming a key that list + // does not hold, or of another type, is refused here and never enters the catalogue. + if carried, err := carriedList(result); err != nil { + return manifest, kept, err + } else if carried != nil { + if err := publishPredefined(ctx, inv, result, carried); err != nil { + fmt.Printf("%s: the list of predefined keys it carried is not published: %v\n", result.ID, err) + } + } + if manifest.NamesPredefined() { + if _, err := catalogue.ParseManifest(kept.Manifest); err != nil { + return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: judged against the "+ + "list of predefined keys the mesh publishes, %w", result.On, manifest.Module, short(result.Commit), err) + } + } // **A build whose source is unchanged is never a move** (novox/hq issue 280): a rebuild made from // what the build the mesh stands on was made from registers that build's artifacts at the new // commit, so no machine is sent a new digest for a source nobody changed — an image is not diff --git a/cmd/mesh-controller/check.go b/cmd/mesh-controller/check.go index 64b04429..cd7d57bc 100644 --- a/cmd/mesh-controller/check.go +++ b/cmd/mesh-controller/check.go @@ -47,6 +47,24 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error { shelf := catalogue.Shelf{} faulted := map[string]bool{} failed := 0 + // The list of predefined keys at the root of the repository above each manifest (novox/hq ADR 0348 §7), read + // once per repository: a manifest naming a key is judged against the list its own tree holds. + lists := map[string]catalogue.PredefinedList{} + listOf := func(path string) (catalogue.PredefinedList, error) { + root := catalogue.RepositoryRoot(filepath.Dir(path)) + if root == "" { + return nil, nil + } + if list, read := lists[root]; read { + return list, nil + } + list, err := catalogue.ReadPredefined(root) + if err != nil { + return nil, err + } + lists[root] = list + return list, nil + } for _, path := range paths { raw, err := os.ReadFile(path) if err != nil { @@ -54,7 +72,13 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error { failed++ continue } - m, err := catalogue.ParseManifest(raw) + list, err := listOf(path) + if err != nil { + fmt.Fprintf(out, "%s: %v\n", path, err) + failed++ + continue + } + m, err := catalogue.ParseManifestWith(raw, list) if err == nil { // Each template's text read from the tree beside the manifest, and judged as inline content is // (novox/hq ADR 0349 §2): a missing template is refused by name. diff --git a/cmd/mesh-controller/merge_gate.go b/cmd/mesh-controller/merge_gate.go index 78b5332a..b555e4e5 100644 --- a/cmd/mesh-controller/merge_gate.go +++ b/cmd/mesh-controller/merge_gate.go @@ -550,8 +550,17 @@ func shelfWithChange(base map[string]catalogue.Manifest, sources map[string]inve out[k] = m } var failures []string + // The list of predefined keys at the tree's root (novox/hq ADR 0348 §7): what the build of this tree judges each + // manifest against and registration publishes, so what the gate judges each against too. A list that is + // wrong is refused by name, and every manifest is then judged against none. + predefined, err := catalogue.ReadPredefined(tree) + if err != nil { + failures = append(failures, fmt.Sprintf("%s: the controller judging this (%s) refuses it — %s", + catalogue.PredefinedFile, version, oneLine(err.Error()))) + predefined = nil + } found := map[string]string{} // module → its directory in the tree - err := filepath.WalkDir(tree, func(path string, d fs.DirEntry, err error) error { + err = filepath.WalkDir(tree, func(path string, d fs.DirEntry, err error) error { if err != nil { return err } @@ -575,7 +584,7 @@ func shelfWithChange(base map[string]catalogue.Manifest, sources map[string]inve if err != nil { return err } - m, err := catalogue.ParseManifest(raw) + m, err := catalogue.ParseManifestWith(raw, predefined) if err == nil { // Each template's text from the tree beside the manifest, as the build reads it back from the archive it // packs (novox/hq ADR 0349 §2): so a change to a template alone is a change to its module. @@ -610,7 +619,7 @@ func shelfWithChange(base map[string]catalogue.Manifest, sources map[string]inve // As registration reads a built manifest: strictly, again (takeIn). var raw []byte if raw, err = json.Marshal(resolved); err == nil { - resolved, err = catalogue.ParseManifest(raw) + resolved, err = catalogue.ParseManifestWith(raw, predefined) } } if err == nil { diff --git a/cmd/mesh-controller/predefined.go b/cmd/mesh-controller/predefined.go new file mode 100644 index 00000000..7b93b748 --- /dev/null +++ b/cmd/mesh-controller/predefined.go @@ -0,0 +1,93 @@ +package main + +import ( + "context" + "fmt" + "path" + "strings" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// The list of predefined keys the mesh publishes (novox/hq ADR 0348 §7, design 57 package K2, issue 661). +// +// **The newest trunk build of the catalogue wins.** A build of the catalogue reads predefined.json at the commit +// it builds and carries it with the commit's generation (link.PredefinedList). Registration keeps it in the +// controller's store when that generation is larger than the one kept (inventory.KeepPredefined), so an older +// build's list heard later never replaces a newer one, and every manifest the controller reads again is judged +// against the list kept (catalogue.UsePredefined). mesh-config publishes it on the bus from there (design 57 +// package K3); the controller keeps no config value, only the keys. + +// fromTheCatalogue is whether a build is of the catalogue's repository, whose list alone the mesh publishes: by +// its name, in the URL form or the seat form. +func fromTheCatalogue(result link.BuildResult) bool { + for _, repository := range []string{result.Repository, sourceRepository(result)} { + name := strings.TrimSuffix(path.Base(strings.TrimRight(repository, "/")), ".git") + if name == catalogue.CatalogueRepository { + return true + } + } + return false +} + +func sourceRepository(result link.BuildResult) string { + if result.Source == nil { + return "" + } + return result.Source.Repository +} + +// carriedList is the list a build of the catalogue carried, read strictly; nil for a build of another +// repository, or one that carried none. +func carriedList(result link.BuildResult) (catalogue.PredefinedList, error) { + if result.Predefined == nil || !fromTheCatalogue(result) { + return nil, nil + } + list, err := catalogue.ParsePredefined(result.Predefined.Keys) + if err != nil { + return nil, fmt.Errorf("%s carried a list of predefined keys that is refused: %w", result.ID, err) + } + return list, nil +} + +// judgedWith is the list a build's manifest is first judged against at registration: the catalogue's own at the +// commit built, which its build judged it against, where it carried one; the list kept otherwise. +func judgedWith(result link.BuildResult) (catalogue.PredefinedList, error) { + list, err := carriedList(result) + if err != nil || list != nil { + return list, err + } + return catalogue.PublishedPredefined(), nil +} + +// loadPredefined makes the list kept in the store the one this controller judges manifests against. A store +// keeping none leaves the list it has. +func loadPredefined(ctx context.Context, inv *inventory.Inventory) error { + kept, has, err := inv.PredefinedKept(ctx) + if err != nil || !has { + return err + } + catalogue.UsePredefined(kept.Keys) + return nil +} + +// publishPredefined keeps a trunk build's list when it is newer than the one kept, says so, and loads what the +// store keeps then — this build's, or a newer one another controller process kept. +func publishPredefined(ctx context.Context, inv *inventory.Inventory, result link.BuildResult, + list catalogue.PredefinedList) error { + kept, err := inv.KeepPredefined(ctx, inventory.PredefinedRecord{Keys: list, Repository: result.Repository, + Commit: result.Commit, Generation: result.Predefined.Generation, Build: result.ID}) + if err != nil { + return err + } + if kept { + fmt.Printf("%s: the list of predefined keys at %s (generation %d) is published: %s\n", result.ID, + short(result.Commit), result.Predefined.Generation, strings.Join(list.Keys(), ", ")) + } else { + fmt.Printf("%s: the list of predefined keys at %s (generation %d) is not newer than the one kept, which "+ + "stands\n", result.ID, short(result.Commit), result.Predefined.Generation) + } + return loadPredefined(ctx, inv) +} diff --git a/cmd/mesh-controller/predefined_test.go b/cmd/mesh-controller/predefined_test.go new file mode 100644 index 00000000..25f09c50 --- /dev/null +++ b/cmd/mesh-controller/predefined_test.go @@ -0,0 +1,197 @@ +package main + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/link" +) + +// The list of predefined keys (novox/hq ADR 0348 §7, design 57 package K2, issue 661): `module check` and the +// merge gate judge each manifest against the list at the root of its tree; registration publishes the newest +// trunk build's list of the catalogue and judges every module against the list the mesh publishes. + +const theFirstKeys = `{ + "font-size": {"type": "number", "description": "the size text is shown in, in points", "minimum": 6, "maximum": 72}, + "agent-managed-settings": {"type": "object", "description": "the managed settings of every agent session, a JSON object", "trusted": true} +}` + +// notifierFallingBackTo is the notifier in the config form, its font size falling back to the key named. +func notifierFallingBackTo(key, kind string) string { + def := "10" + if kind == "string" { + def = `"10"` + } + return `{"module":"notifier","version":"1", + "config":{"font-size":{"type":"` + kind + `","description":"the notification's font size, in points", + "default":` + def + `,"predefined":"` + key + `"}}, + "resources":[{"id":"conf","type":"file","path":"/etc/notifier/notifier.conf","mode":"0644", + "content":"font = ${config:font-size}\n"}]}` +} + +// withoutAPublishedList judges the test against no list the mesh published, and puts the process's back after. +func withoutAPublishedList(t *testing.T) { + t.Helper() + was := catalogue.ReplacePublished(nil) + t.Cleanup(func() { catalogue.ReplacePublished(was) }) +} + +// aRepositoryTree is a repository's checkout holding one module at modules/notifier, and the list when given. +func aRepositoryTree(t *testing.T, manifest string, list *string) string { + t.Helper() + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, ".git"), 0o755); err != nil { + t.Fatal(err) + } + if list != nil { + if err := os.WriteFile(filepath.Join(root, catalogue.PredefinedFile), []byte(*list), 0o644); err != nil { + t.Fatal(err) + } + } + return aTemplatedModule(t, filepath.Join(root, "modules", "notifier"), manifest, nil) +} + +func TestModuleCheckJudgesAManifestAgainstTheListAtItsRepositoryRoot(t *testing.T) { + withoutAPublishedList(t) + list := theFirstKeys + check := func(path string) (string, error) { + var out bytes.Buffer + err := moduleCheck([]string{path}, &out) + return out.String(), err + } + if out, err := check(aRepositoryTree(t, notifierFallingBackTo("font-size", "number"), &list)); err != nil || + !strings.Contains(out, "notifier: ok") { + t.Fatalf("a module naming font-size was refused: %v\n%s", err, out) + } + for _, c := range []struct { + manifest string + list *string + says string + }{ + {notifierFallingBackTo("font-scale", "number"), &list, `"font-scale", which is not on the list`}, + {notifierFallingBackTo("font-size", "string"), &list, "which is a number, and this value is a string"}, + {notifierFallingBackTo("font-size", "number"), nil, "is not present"}, + {notifierFallingBackTo("font-size", "number"), ptr(`{"font-size": {"type": "number"}}`), "no description"}, + } { + if out, err := check(aRepositoryTree(t, c.manifest, c.list)); err == nil || !strings.Contains(out, c.says) { + t.Errorf("wanted the check refused saying %q, got %v:\n%s", c.says, err, out) + } + } +} + +func ptr(s string) *string { return &s } + +// writeTheList puts the list at the root of a tree the gate judges. +func writeTheList(t *testing.T, tree, list string) string { + t.Helper() + if err := os.WriteFile(filepath.Join(tree, catalogue.PredefinedFile), []byte(list), 0o644); err != nil { + t.Fatal(err) + } + return tree +} + +func TestTheMergeGateJudgesEachManifestAgainstTheListOfTheTree(t *testing.T) { + withoutAPublishedList(t) + f, manifests := templatedMesh(t) + + v := gateJudged(t, f, writeTheList(t, aTree(t, withEdit(manifests, "notifier", + notifierFallingBackTo("font-size", "number"))), theFirstKeys), "modules/notifier/module.json") + if v.Verdict == "fail" || v.Modules["notifier"] != "changed" { + t.Fatalf("a module naming font-size on the tree's list failed the gate (%v):\n%s", v.Modules, v.Report()) + } + for _, c := range []struct{ manifest, list, says string }{ + {notifierFallingBackTo("font-scale", "number"), theFirstKeys, `"font-scale", which is not on the list`}, + {notifierFallingBackTo("font-size", "string"), theFirstKeys, "which is a number"}, + {inlineNotifier, `{"font-size": {"type": "number", "description": "points", "unit": "pt"}}`, `unknown field "unit"`}, + } { + v := gateJudged(t, f, writeTheList(t, aTree(t, withEdit(manifests, "notifier", c.manifest)), c.list), + "modules/notifier/module.json", catalogue.PredefinedFile) + if v.Verdict != "fail" || !strings.Contains(v.Report(), c.says) { + t.Errorf("wanted the gate to fail saying %q:\n%s", c.says, v.Report()) + } + } +} + +// catalogueBuild is a trunk build of the catalogue carrying the list at a generation. +func catalogueBuild(id, manifest, list string, generation int) link.BuildResult { + r := link.BuildResult{ID: id, Repository: "http://forge.internal:20000/novox/mesh-catalog.git", + Path: "modules/notifier", Ref: "main", On: "anchor", Commit: strings.Repeat(id[len(id)-1:], 12), + Manifest: json.RawMessage(manifest), Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}} + if list != "" { + r.Predefined = &link.PredefinedList{Keys: json.RawMessage(list), Generation: generation} + } + return r +} + +func TestRegistrationPublishesTheNewestTrunkListAndJudgesEveryModuleAgainstIt(t *testing.T) { + withoutAPublishedList(t) + open := aMesh(t) + ctx := t.Context() + + // A manifest naming font-size registers, and its build's list is published. + if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, + catalogueBuild("b-1", notifierFallingBackTo("font-size", "number"), theFirstKeys, 10))); err != nil { + t.Fatalf("a module naming font-size was not registered: %v", err) + } + kept, has, err := open.inventory.PredefinedKept(ctx) + if err != nil || !has || kept.Generation != 10 || kept.Build != "b-1" { + t.Fatalf("the list was not published: %+v %v %v", kept, has, err) + } + if got := catalogue.PublishedPredefined(); len(got) != 2 { + t.Fatalf("the controller judges against %v, not the list published", got) + } + + // One naming a key not on the list, or of another type, is refused and never enters the catalogue. + for i, c := range []struct{ manifest, says string }{ + {notifierFallingBackTo("font-scale", "number"), "not on the list"}, + {notifierFallingBackTo("font-size", "string"), "which is a number"}, + } { + id := []string{"b-2", "b-3"}[i] + if _, _, err := takeIn(ctx, open.inventory, catalogueBuild(id, c.manifest, theFirstKeys, 10)); err == nil || + !strings.Contains(err.Error(), c.says) { + t.Errorf("wanted the registration refused saying %q, got %v", c.says, err) + } + } + src, err := open.inventory.SourceOf(ctx, "notifier") + if err != nil || src.BuiltFrom != strings.Repeat("1", 12) { + t.Fatalf("a refused build replaced the registered one: %+v %v", src, err) + } + + // A newer build's list, which drops agent-managed-settings, replaces the published one. + smaller := `{"font-size": {"type": "number", "description": "points", "minimum": 6, "maximum": 72}}` + if _, _, err := takeIn(ctx, open.inventory, catalogueBuild("b-4", + notifierFallingBackTo("font-size", "number"), smaller, 12)); err != nil { + t.Fatal(err) + } + // The older build's outcome heard later registers nothing new about the list: the newer stands. + if _, _, err := takeIn(ctx, open.inventory, catalogueBuild("b-5", + notifierFallingBackTo("font-size", "number"), theFirstKeys, 11)); err != nil { + t.Fatal(err) + } + kept, _, _ = open.inventory.PredefinedKept(ctx) + if kept.Build != "b-4" || len(kept.Keys) != 1 || len(catalogue.PublishedPredefined()) != 1 { + t.Fatalf("an older build's list heard later replaced the newer one: %+v", kept) + } + + // A build of another repository publishes nothing, and its manifest is judged against the published list. + other := catalogueBuild("b-6", notifierFallingBackTo("font-size", "number"), theFirstKeys, 99) + other.Repository, other.Source.Repository = "http://forge.internal:20000/novox/notifier.git", "novox/notifier" + other.Manifest = json.RawMessage(strings.Replace(notifierFallingBackTo("font-size", "number"), `"notifier"`, + `"chimes"`, 1)) + if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, other)); err != nil { + t.Fatalf("a module of another repository naming a published key: %v", err) + } + if kept, _, _ = open.inventory.PredefinedKept(ctx); kept.Build != "b-4" { + t.Fatalf("a build of another repository published its list: %+v", kept) + } + other.ID, other.Manifest = "b-7", json.RawMessage(strings.Replace(notifierFallingBackTo("agent-managed-settings", + "number"), `"notifier"`, `"chimes"`, 1)) + if _, _, err := takeIn(ctx, open.inventory, other); err == nil || !strings.Contains(err.Error(), "not on the list") { + t.Fatalf("a key only another repository's list holds was registered: %v", err) + } +} diff --git a/cmd/mesh-controller/stores.go b/cmd/mesh-controller/stores.go index aeea12c1..f2db129a 100644 --- a/cmd/mesh-controller/stores.go +++ b/cmd/mesh-controller/stores.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "os" "time" "github.com/novox/mesh-controller/internal/catalogue" @@ -124,6 +125,11 @@ func openInventory(ctx context.Context) (*inventory.Inventory, error) { if aliases, err := inv.Aliases(ctx); err == nil { catalogue.UseAliases(aliases) } + // And the list of predefined keys the mesh publishes (novox/hq ADR 0348 §7), which every manifest read again + // is judged against. A store a migration has not reached yet keeps none, and nothing names a key then. + if err := loadPredefined(ctx, inv); err != nil { + fmt.Fprintf(os.Stderr, "the list of predefined keys kept cannot be read: %v\n", err) + } return inv, nil } diff --git a/internal/builder/builder.go b/internal/builder/builder.go index c2c1cbf9..0729a351 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -108,6 +108,17 @@ type Result struct { // recipe and the package it compiles; for any other, nothing is said and the whole of what the build // sees stays its source, as before. Sources []BuildSource + + // Predefined is the list of predefined keys at the root of the repository at the commit built (novox/hq ADR + // 0348 §7), which the manifest was judged against; nil for a repository that holds none. + Predefined *Predefined +} + +// Predefined is the list of predefined keys a build read: its text, canonical (catalogue.PredefinedList), and +// the generation of the commit built — how many commits it stands on, which orders two builds of one trunk. +type Predefined struct { + Keys []byte + Generation int } // BuildSource is the build source a build read in one repository (novox/hq ADR 0267). @@ -261,11 +272,28 @@ func build(ctx context.Context, run Runner, publish Publisher, "%s has no %s at %s, so there is nothing saying what it is: %w", repository, ManifestName, describe(path), err) } - manifest, err := catalogue.ParseManifest(raw) + // **The list of predefined keys at the commit built** (novox/hq ADR 0348 §7): predefined.json at the + // repository's root, read strictly; a list that is wrong fails the build, and the manifest is judged against + // the list of its own tree, never against one the mesh published from another commit. + predefined, err := catalogue.ReadPredefined(tree) + if err != nil { + say("predefined", "INVALID: %v", err) + return Result{}, err + } + manifest, err := catalogue.ParseManifestWith(raw, predefined) if err != nil { say("manifest", "INVALID: %v", err) return Result{}, err } + var listed *Predefined + if predefined != nil { + text, err := predefined.Canonical() + if err != nil { + return Result{}, err + } + listed = &Predefined{Keys: text, Generation: generationOf(ctx, run, tree, commit)} + say("predefined", "%d key(s) at %s, generation %d", len(predefined), short(commit), listed.Generation) + } // What is judged only where a definition is written (novox/hq ADR 0326 §3, issue 574): a build is // where a changed definition is first read, so it is refused here before anything is made of it. // Judged with each template's text in its file, as written in the tree (novox/hq ADR 0349 §2): a template the @@ -291,6 +319,11 @@ func build(ctx context.Context, run Runner, publish Publisher, if src.tree, err = gitTree(ctx, run, tree, path); err != nil { src.notPinned("its tree could not be named: " + err.Error()) } + // A module falling back to a predefined key was judged against the list, so a change to the list is a change + // to it: the planner rebuilds it, and its build publishes the list (ADR 0348 §7). + if predefined != nil && manifest.NamesPredefined() { + src.rootHas(catalogue.PredefinedFile) + } // A build-time credential, written into the build context as .npmrc, but ONLY for a module that // asks for it: a `package` artifact (which publishes), or an image whose Dockerfile COPYs .npmrc. @@ -451,7 +484,23 @@ func build(ctx context.Context, run Runner, publish Publisher, return Result{Manifest: resolved, Commit: commit, Built: built, Against: against(within, manifest, stoodOn), Read: readBy(manifest), Source: fingerprint, Trunk: trunk, OnTrunk: onTrunk, Branches: branches, Sources: sources, - ReuseKey: key.Key, ReuseParts: key.Parts, NoKey: key.Why, Reused: reused, Why: builtWhy}, nil + ReuseKey: key.Key, ReuseParts: key.Parts, NoKey: key.Why, Reused: reused, Why: builtWhy, + Predefined: listed}, nil +} + +// generationOf is how many commits the commit built stands on, itself included: of two commits on one trunk the +// later has more, so the controller keeps the list of the newer one whatever order it hears them in (ADR 0348 +// §7). 0 when git could not say. +func generationOf(ctx context.Context, run Runner, tree, commit string) int { + out, err := run(ctx, tree, "git", "rev-list", "--count", commit) + if err != nil { + return 0 + } + n, err := strconv.Atoi(strings.TrimSpace(out)) + if err != nil || n < 0 { + return 0 + } + return n } // takeACheckBuild is the walk's look for a check build of its key (novox/hq ADR 0350 §2, §3): the key taken by diff --git a/internal/builder/builder_test.go b/internal/builder/builder_test.go index afe57572..4bfe447f 100644 --- a/internal/builder/builder_test.go +++ b/internal/builder/builder_test.go @@ -83,6 +83,8 @@ func (r *recorded) run(_ context.Context, dir, name string, args ...string) (str return r.tree + "\n", nil case name == "git" && len(args) > 0 && args[0] == "rev-parse": return "c0ffeec0ffeec0ffeec0ffeec0ffeec0ffeec0ff\n", nil + case name == "git" && len(args) > 1 && args[0] == "rev-list" && args[1] == "--count": + return "42\n", nil } return "", nil } diff --git a/internal/builder/predefined_test.go b/internal/builder/predefined_test.go new file mode 100644 index 00000000..71ac4fec --- /dev/null +++ b/internal/builder/predefined_test.go @@ -0,0 +1,106 @@ +package builder + +import ( + "context" + "slices" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// A build reads the list of predefined keys at the root of the repository at the commit it builds (novox/hq ADR +// 0348 §7, design 57 package K2, issue 661): it refuses a list that is wrong, judges the manifest against it, and +// carries it with the commit's generation for the controller to publish. + +const theFirstKeys = `{ + "font-size": {"type": "number", "description": "the size text is shown in, in points", "minimum": 6, "maximum": 72}, + "agent-managed-settings": {"type": "object", "description": "the managed settings of every agent session, a JSON object", "trusted": true} +}` + +// fallingBack is a module of the catalogue whose font size falls back to the predefined key named. +func fallingBack(key, kind string) string { + def := "10" + if kind == "string" { + def = `"10"` + } + return `{"module":"dunst","version":"1", + "build":{"artifacts":[{"name":"look","kind":"archive","from":"files"}]}, + "config":{"font-size":{"type":"` + kind + `","description":"the notification's font size, in points", + "default":` + def + `,"predefined":"` + key + `"}}, + "resources":[ + {"id":"theme","type":"archive","path":"/opt/dunst","artifact":"look"}, + {"id":"rc","type":"file","path":"/etc/xdg/dunst/dunstrc","content":"font = Inter ${config:font-size}\n"}]}` +} + +func buildInTheCatalogue(t *testing.T, manifest string, list *string) (Result, error) { + t.Helper() + r := &recorded{contents: map[string]string{"modules/dunst/" + ManifestName: manifest, + "modules/dunst/files/theme.conf": "dark", "README.md": "the catalogue"}} + if list != nil { + r.contents[catalogue.PredefinedFile] = *list + } + return Build(context.Background(), onTrunk{recorded: r}.run, r, "https://forge.invalid/mesh-catalog.git", + "modules/dunst", "", t.TempDir(), nil, Npmrc{}, GitCredential{}, nil) +} + +func TestABuildNamingAPredefinedKeyIsJudgedAgainstItsTreesListAndCarriesIt(t *testing.T) { + list := theFirstKeys + got, err := buildInTheCatalogue(t, fallingBack("font-size", "number"), &list) + if err != nil { + t.Fatalf("a module falling back to font-size did not build: %v", err) + } + if got.Predefined == nil || got.Predefined.Generation != 42 { + t.Fatalf("the list was not carried with its generation: %+v", got.Predefined) + } + carried, err := catalogue.ParsePredefined(got.Predefined.Keys) + if err != nil || len(carried) != 2 || carried["font-size"].Type != catalogue.ConfigNumber { + t.Fatalf("the list carried is %s (%v)", got.Predefined.Keys, err) + } + // A change to the list is a change to the module that was judged against it. + if len(got.Sources) == 0 || !slices.Contains(got.Sources[0].Paths, catalogue.PredefinedFile) { + t.Fatalf("its build source leaves the list out: %+v", got.Sources) + } + + for _, c := range []struct{ manifest, says string }{ + {fallingBack("font-scale", "number"), `"font-scale", which is not on the list`}, + {fallingBack("font-size", "string"), "which is a number, and this value is a string"}, + } { + if _, err := buildInTheCatalogue(t, c.manifest, &list); err == nil || !strings.Contains(err.Error(), c.says) { + t.Errorf("wanted the build refused saying %q, got %v", c.says, err) + } + } +} + +func TestABuildRefusesAWrongList(t *testing.T) { + wrong := `{"font-size": {"type": "number", "description": "points", "unit": "pt"}}` + if _, err := buildInTheCatalogue(t, fallingBack("font-size", "number"), &wrong); err == nil || + !strings.Contains(err.Error(), `unknown field "unit"`) { + t.Fatalf("a wrong list was built with: %v", err) + } +} + +// A repository without the list builds as before, carrying none; a value naming a key there is refused. +func TestABuildWithoutAListCarriesNone(t *testing.T) { + plain := strings.Replace(fallingBack("font-size", "number"), `,"predefined":"font-size"`, "", 1) + got, err := buildInTheCatalogue(t, plain, nil) + if err != nil { + t.Fatal(err) + } + if got.Predefined != nil { + t.Fatalf("a list was carried from a tree holding none: %+v", got.Predefined) + } + if slices.Contains(got.Sources[0].Paths, catalogue.PredefinedFile) { + t.Fatalf("the build source names a list: %+v", got.Sources) + } + if _, err := buildInTheCatalogue(t, fallingBack("font-size", "number"), nil); err == nil || + !strings.Contains(err.Error(), "is not present") { + t.Fatalf("a key named where no list is: %v", err) + } + // A list present and a module naming no key: carried, and not the module's build source. + list := theFirstKeys + got, err = buildInTheCatalogue(t, plain, &list) + if err != nil || got.Predefined == nil || slices.Contains(got.Sources[0].Paths, catalogue.PredefinedFile) { + t.Fatalf("a module naming no key: %+v %+v %v", got.Predefined, got.Sources, err) + } +} diff --git a/internal/builder/source.go b/internal/builder/source.go index 7d9b9c0e..4d033a33 100644 --- a/internal/builder/source.go +++ b/internal/builder/source.go @@ -58,6 +58,9 @@ type sourceInputs struct { // whole when an artifact's is not known (novox/hq ADR 0267). own map[string]bool ownIsAll bool + // root is the module's build source in its own repository outside its directory, relative to the + // repository's root: the list of predefined keys a manifest naming one was judged against (ADR 0348 §7). + root map[string]bool // read is, per context (repository and ref), the build source read there; nil for one read whole. read map[string]map[string]bool readAs map[string]catalogue.ArtifactContext @@ -86,6 +89,9 @@ func (s *sourceInputs) copy() *sourceInputs { for k, v := range s.own { c.own[k] = v } + for k, v := range s.root { + c.root[k] = v + } for k, v := range s.read { if v == nil { c.read[k] = nil @@ -112,7 +118,7 @@ func (s *sourceInputs) contextOffHead() { func newSourceInputs(module string) *sourceInputs { return &sourceInputs{module: module, contexts: map[string]string{}, toolchains: map[string]string{}, - own: map[string]bool{}, read: map[string]map[string]bool{}, readAs: map[string]catalogue.ArtifactContext{}, + own: map[string]bool{}, root: map[string]bool{}, read: map[string]map[string]bool{}, readAs: map[string]catalogue.ArtifactContext{}, contextsAtHead: true} } @@ -126,6 +132,16 @@ func (s *sourceInputs) ownHas(entries ...string) { } } +// rootHas adds entries, relative to the repository's root, to the module's build source in its own repository. +func (s *sourceInputs) rootHas(entries ...string) { + if s == nil { + return + } + for _, e := range entries { + s.root[e] = true + } +} + // ownWhole says an artifact's build source in the module's own repository is not known: the module's // whole directory is its source, as it was before. func (s *sourceInputs) ownWhole() { @@ -178,6 +194,9 @@ func (s *sourceInputs) buildSources() []BuildSource { for e := range s.own { own = append(own, withPrefix(s.prefix, e)) } + for e := range s.root { + own = append(own, e) + } sort.Strings(own) out = append(out, BuildSource{Paths: compactSorted(own)}) } diff --git a/internal/catalogue/config.go b/internal/catalogue/config.go index 90e9ea71..6feaee70 100644 --- a/internal/catalogue/config.go +++ b/internal/catalogue/config.go @@ -61,18 +61,6 @@ const ( // accountFacts are what the mesh places for a module's account, never config values (ADR 0348 §11). var accountFacts = map[string]bool{"puid": true, "pgid": true} -// PredefinedKey is one key of the catalogue's list of predefined keys, as far as a module's value falling back -// to it is judged here: its type, and whether it is trusted (ADR 0348 §7, §8). -type PredefinedKey struct { - Type string - Trusted bool -} - -// predefinedKeys is the list of predefined keys, nil until it is present. The list (`predefined.json` at the -// catalogue's root) arrives with design 57's package K2 (issue 661); until then a value naming one is refused -// saying the list is not yet present. -var predefinedKeys map[string]PredefinedKey - // configForms is which of the two forms a manifest was written in, by the words of each it uses. type configForms struct { config, settings []string @@ -259,12 +247,13 @@ func ConfigProblems(m Manifest) []string { "(novox/hq ADR 0348 §4, ADR 0112)", word) } if d.Predefined != "" { - p, listed := predefinedKeys[d.Predefined] + p, listed := m.predefined[d.Predefined] switch { - case predefinedKeys == nil: - say("falls back to the predefined key %q, and the list of predefined keys (the catalogue's "+ - "predefined.json, design 57 package K2) is not yet present: no value can fall back to one "+ - "until it is; leave \"predefined\" out for now (novox/hq ADR 0348 §7)", d.Predefined) + case m.predefined == nil: + say("falls back to the predefined key %q, and the list of predefined keys (%s at the root of "+ + "the catalogue's repository) is not present where it is judged: a value falls back only to a "+ + "key on that list; judge it in the catalogue, or leave \"predefined\" out (novox/hq ADR 0348 §7)", + d.Predefined, PredefinedFile) case !listed: say("falls back to %q, which is not on the list of predefined keys: name a key on it, or leave "+ "\"predefined\" out (novox/hq ADR 0348 §7)", d.Predefined) diff --git a/internal/catalogue/config_test.go b/internal/catalogue/config_test.go index 1a2d8499..2cbaeea1 100644 --- a/internal/catalogue/config_test.go +++ b/internal/catalogue/config_test.go @@ -157,20 +157,20 @@ func TestConfigRefusesAnObjectOrListNotTrusted(t *testing.T) { } } -// withPredefined is the list of predefined keys as package K2 will give it, for the length of a test. -func withPredefined(t *testing.T, keys map[string]PredefinedKey) { +// withPredefined is the list of predefined keys the mesh published, for the length of a test. +func withPredefined(t *testing.T, keys PredefinedList) { t.Helper() - was := predefinedKeys - predefinedKeys = keys - t.Cleanup(func() { predefinedKeys = was }) + was := ReplacePublished(keys) + t.Cleanup(func() { ReplacePublished(was) }) } -// Rule 5: a predefined key not on the list, or of another type. Until the list is present (package K2), a value -// naming one is refused saying so. +// Rule 5: a predefined key not on the list, or of another type. Where no list is present, a value naming one is +// refused saying so. func TestConfigRefusesAPredefinedKeyNotOnTheListOrOfAnotherType(t *testing.T) { + withPredefined(t, nil) m := inConfigForm() declaration(m, "font-size")["predefined"] = "font-size" - refused(t, m, `"font-size"`, "the list of predefined keys", "not yet present") + refused(t, m, `"font-size"`, "the list of predefined keys", "is not present") withPredefined(t, map[string]PredefinedKey{"font-size": {Type: "number"}, "accent": {Type: "string"}}) if _, err := parseRaw(t, m); err != nil { @@ -186,9 +186,10 @@ func TestConfigRefusesAPredefinedKeyNotOnTheListOrOfAnotherType(t *testing.T) { // Rule 6: an untrusted value falling back to a trusted predefined key. func TestConfigRefusesAnUntrustedValueFallingBackToATrustedKey(t *testing.T) { + withPredefined(t, nil) m := inConfigForm() declaration(m, "theme")["predefined"] = "theme" - refused(t, m, `"theme"`, "not yet present") + refused(t, m, `"theme"`, "is not present") withPredefined(t, map[string]PredefinedKey{"theme": {Type: "string", Trusted: true}}) refused(t, m, `"theme"`, "trusted", "is not") diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index 49866bdd..480ce761 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -541,6 +541,10 @@ type Manifest struct { // machine's declaration by name until the controller is updated (LeftOut). unknown string + // predefined is the list of predefined keys it was judged against (predefined.go, novox/hq ADR 0348 §7): + // read again — its templates filled from the tree — it is judged against the same list. + predefined PredefinedList + // bestEffort marks the view of a left-out module that only its backup lines are made from // (backupView, novox/hq ADR 0262): a line of it that cannot be placed is said in the plan's list of // what could not be placed, never an error that would cost the whole machine its declaration. @@ -1600,6 +1604,17 @@ func SecretLocal(to, local string) string { func (m Manifest) UnknownField() string { return m.unknown } func ParseManifest(raw []byte) (Manifest, error) { + return parseManifest(raw, PublishedPredefined()) +} + +// ParseManifestWith is ParseManifest judging every value that falls back to a predefined key against the list +// given — the one read from the tree the manifest is in (predefined.go, novox/hq ADR 0348 §7) — where +// ParseManifest judges against the list the mesh published. Nil is no list: a value naming a key is refused. +func ParseManifestWith(raw []byte, predefined PredefinedList) (Manifest, error) { + return parseManifest(raw, predefined) +} + +func parseManifest(raw []byte, predefined PredefinedList) (Manifest, error) { var m Manifest // Strictly. **An unknown key is refused**, which is the discipline the host's declaration // parser has and manifests lacked (novox/hq 04-ISSUES/003): a `scope:` key survived in five @@ -1627,6 +1642,7 @@ func ParseManifest(raw []byte) (Manifest, error) { } return Manifest{}, fmt.Errorf("this is not a module manifest: %w", err) } + m.predefined = predefined var problems []string if !name.MatchString(m.Module) { diff --git a/internal/catalogue/predefined.go b/internal/catalogue/predefined.go new file mode 100644 index 00000000..73d2cd9b --- /dev/null +++ b/internal/catalogue/predefined.go @@ -0,0 +1,236 @@ +package catalogue + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + "sync" +) + +// The list of predefined keys (novox/hq ADR 0348 §7, design 57 package K2, issue 661). +// +// **A fixed, reviewable list, owned by the build.** Mesh config and node config hold values only for the keys +// in the catalogue's `predefined.json`, at the root of its repository: each key with its type, a description +// including its unit, its validation (enum, minimum and maximum, pattern) and whether it is trusted. A module's +// value may fall back to one of them, of the same type (ADR 0348 §2, §4). +// +// **Read where a manifest is judged, from the tree it is judged in.** A build reads the file at the root of the +// repository at the commit it builds, `module check` at the root of the repository above each manifest, the +// merge gate at the root of the tree under check; each refuses a file that is not exactly a list of keys, and +// judges every manifest it reads against it (ParseManifestWith). Everywhere else — registration, a stored build +// read again — a manifest is judged against the list the mesh published (PublishedPredefined): the newest trunk +// build's of the catalogue, which the controller keeps and loads with its store. + +// PredefinedFile is the list's file at the root of the catalogue's repository. +const PredefinedFile = "predefined.json" + +// CatalogueRepository is the name of the repository whose list the mesh publishes: the catalogue's, by the name +// a check finds it beside under (internal/builder's check, ADR 0121). +const CatalogueRepository = "mesh-catalog" + +// PredefinedKey is one key of the list: its type, its description, its validation and whether it is trusted +// (ADR 0348 §7). It has no default: a module's value falling back to it has its own. +type PredefinedKey struct { + Type string `json:"type"` + Description string `json:"description"` + Enum []any `json:"enum,omitempty"` + Minimum *float64 `json:"minimum,omitempty"` + Maximum *float64 `json:"maximum,omitempty"` + Pattern string `json:"pattern,omitempty"` + Trusted bool `json:"trusted,omitempty"` +} + +// PredefinedList is the list by key. Nil is no list at all, which is not the same as an empty one: a value +// naming a key is refused then saying the list is not present. +type PredefinedList map[string]PredefinedKey + +// ParsePredefined reads predefined.json strictly: one JSON object of keys, each with exactly the fields of a +// PredefinedKey, and every key judged as a module's config value is (PredefinedProblems). Anything else is +// refused whole, naming each problem — a list half read would let a module fall back to a key nobody reviewed. +func ParsePredefined(raw []byte) (PredefinedList, error) { + var keys map[string]json.RawMessage + if err := json.Unmarshal(raw, &keys); err != nil { + return nil, fmt.Errorf("%s is not a JSON object of keys: %w", PredefinedFile, err) + } + if keys == nil { + return nil, fmt.Errorf("%s is not a JSON object of keys: it says null", PredefinedFile) + } + list := PredefinedList{} + var problems []string + for _, key := range sortedKeys(keys) { + decoder := json.NewDecoder(bytes.NewReader(keys[key])) + decoder.DisallowUnknownFields() + var k PredefinedKey + if err := decoder.Decode(&k); err != nil { + problems = append(problems, fmt.Sprintf("%q is not a predefined key's declaration (type, description, "+ + "enum, minimum, maximum, pattern, trusted): %v", key, err)) + continue + } + list[key] = k + } + problems = append(problems, PredefinedProblems(list)...) + if len(problems) > 0 { + return nil, fmt.Errorf("%s is refused: %s", PredefinedFile, strings.Join(problems, "; ")) + } + return list, nil +} + +// PredefinedProblems is every way a key of the list is declared wrongly, by the rules a module's config value +// is held to (ADR 0348 §4, §7, §11): a usable name that is not placement, a type, a description, validation +// that fits the type, and an object or a list only when trusted. +func PredefinedProblems(list PredefinedList) []string { + var problems []string + for _, key := range sortedKeys(list) { + k := list[key] + say := func(format string, args ...any) { + problems = append(problems, fmt.Sprintf("the predefined key %q: ", key)+fmt.Sprintf(format, args...)) + } + if !settingRef.MatchString("${setting:" + key + "}") { + say("not a usable key: lower-case letters, digits, dots, dashes and underscores") + continue + } + if meshWords[key] || accountFacts[key] { + say("%q is placement, which the mesh decides, never a config value (novox/hq ADR 0348 §11)", key) + continue + } + switch k.Type { + case ConfigNumber, ConfigString, ConfigBoolean: + case ConfigObject, ConfigList: + if !k.Trusted { + say("is %s and not trusted: an object or a list is trusted, \"trusted\": true (novox/hq ADR 0348 §4)", k.Type) + } + case "": + say("no type: say \"type\": number, string or boolean, or object or list when trusted (novox/hq ADR 0348 §4)") + default: + say("type %q is none of number, string, boolean, object and list (novox/hq ADR 0348 §4)", k.Type) + } + if strings.TrimSpace(k.Description) == "" { + say("no description: say in a sentence what the value is, including its unit (novox/hq ADR 0348 §7)") + } + for _, wrong := range validationProblems(SettingDeclaration{Type: k.Type, Enum: k.Enum, Minimum: k.Minimum, + Maximum: k.Maximum, Pattern: k.Pattern}) { + say("%s", wrong) + } + } + return problems +} + +// ReadPredefined is the list in predefined.json at a repository's root, and nil with no error when the root +// holds none: a repository other than the catalogue, or the catalogue before the file. +func ReadPredefined(root string) (PredefinedList, error) { + path := filepath.Join(root, PredefinedFile) + info, err := os.Lstat(path) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("%s is %s, not a plain file", path, kindOf(info.Mode())) + } + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + return ParsePredefined(raw) +} + +// RepositoryRoot is the nearest directory at or above dir holding `.git`: the root of the repository a manifest +// is in, where `module check` reads the list. Empty when none is: a manifest outside a repository is judged +// against no list. +func RepositoryRoot(dir string) string { + at, err := filepath.Abs(dir) + if err != nil { + return "" + } + for { + if _, err := os.Lstat(filepath.Join(at, ".git")); err == nil { + return at + } + up := filepath.Dir(at) + if up == at { + return "" + } + at = up + } +} + +// PredefinedAbove is the list at the root of the repository above dir (RepositoryRoot), nil when there is no +// repository or it holds no list. +func PredefinedAbove(dir string) (PredefinedList, error) { + root := RepositoryRoot(dir) + if root == "" { + return nil, nil + } + return ReadPredefined(root) +} + +// Canonical is the list as JSON with its keys sorted: what a build carries and the controller keeps, so one list +// is one text. +func (l PredefinedList) Canonical() ([]byte, error) { + if l == nil { + return nil, nil + } + return json.Marshal(map[string]PredefinedKey(l)) +} + +// Keys is the list's keys, sorted. +func (l PredefinedList) Keys() []string { + out := make([]string, 0, len(l)) + for k := range l { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// NamesPredefined is whether any value of the manifest falls back to a predefined key. +func (m Manifest) NamesPredefined() bool { + for _, d := range m.Settings { + if d.Predefined != "" { + return true + } + } + return false +} + +// publishedPredefined is the list the mesh published, which a manifest is judged against where no tree is read. +var publishedPredefined struct { + sync.RWMutex + list PredefinedList +} + +// UsePredefined makes list the one a manifest is judged against where no tree is read: what the controller +// loads from its store, and what it keeps when a newer trunk build of the catalogue publishes one. Nil leaves +// the list it has: a store that holds none yet never empties one already loaded. +func UsePredefined(list PredefinedList) { + if list == nil { + return + } + publishedPredefined.Lock() + defer publishedPredefined.Unlock() + publishedPredefined.list = list +} + +// PublishedPredefined is the list a manifest is judged against where no tree is read; nil when none is loaded. +func PublishedPredefined() PredefinedList { + publishedPredefined.RLock() + defer publishedPredefined.RUnlock() + return publishedPredefined.list +} + +// ReplacePublished sets the published list, nil included, and answers the one it replaced: for a test that +// judges against a list of its own and puts the mesh's back. +func ReplacePublished(list PredefinedList) (was PredefinedList) { + publishedPredefined.Lock() + defer publishedPredefined.Unlock() + was, publishedPredefined.list = publishedPredefined.list, list + return was +} diff --git a/internal/catalogue/predefined_test.go b/internal/catalogue/predefined_test.go new file mode 100644 index 00000000..c683f763 --- /dev/null +++ b/internal/catalogue/predefined_test.go @@ -0,0 +1,175 @@ +package catalogue + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +// The list of predefined keys (novox/hq ADR 0348 §7, design 57 package K2, issue 661): read strictly, and every +// manifest naming a key judged against the list of the tree it is read in. + +// theFirstKeys is the catalogue's first list: font-size and agent-managed-settings. +const theFirstKeys = `{ + "font-size": {"type": "number", "description": "the size text is shown in, in points", "minimum": 6, "maximum": 72}, + "agent-managed-settings": {"type": "object", "description": "the managed settings of every agent session, a JSON object", "trusted": true} +}` + +func TestTheFirstListIsRead(t *testing.T) { + list, err := ParsePredefined([]byte(theFirstKeys)) + if err != nil { + t.Fatal(err) + } + if got := strings.Join(list.Keys(), ","); got != "agent-managed-settings,font-size" { + t.Fatalf("keys %s", got) + } + if k := list["font-size"]; k.Type != ConfigNumber || *k.Minimum != 6 || *k.Maximum != 72 || k.Trusted { + t.Fatalf("font-size read as %+v", k) + } + if k := list["agent-managed-settings"]; k.Type != ConfigObject || !k.Trusted { + t.Fatalf("agent-managed-settings read as %+v", k) + } + // One list is one text, however it was written. + a, _ := list.Canonical() + again, _ := ParsePredefined(a) + b, _ := again.Canonical() + if string(a) != string(b) { + t.Fatalf("canonical text moved:\n%s\n%s", a, b) + } +} + +func TestAListIsRefusedWholeForAnyWrongKey(t *testing.T) { + for _, c := range []struct{ file, says string }{ + {`[]`, "not a JSON object of keys"}, + {`null`, "not a JSON object of keys"}, + {`{"font-size": {"type": "number", "description": "points", "default": 10}}`, `unknown field "default"`}, + {`{"font-size": {"description": "points"}}`, "no type"}, + {`{"font-size": {"type": "decimal", "description": "points"}}`, `type "decimal"`}, + {`{"font-size": {"type": "number"}}`, "no description"}, + {`{"accent": {"type": "object", "description": "colours"}}`, "not trusted"}, + {`{"font-size": {"type": "number", "description": "points", "minimum": 9, "maximum": 6}}`, "above its maximum"}, + {`{"accent": {"type": "string", "description": "a name", "pattern": "("}}`, "not a regular expression"}, + {`{"accent": {"type": "boolean", "description": "on or off", "minimum": 1}}`, "minimum or maximum is for a number"}, + {`{"Font Size": {"type": "number", "description": "points"}}`, "not a usable key"}, + {`{"places": {"type": "string", "description": "where"}}`, "placement"}, + {`{"puid": {"type": "number", "description": "an id"}}`, "placement"}, + } { + if _, err := ParsePredefined([]byte(c.file)); err == nil || !strings.Contains(err.Error(), c.says) { + t.Errorf("%s: wanted a refusal saying %q, got %v", c.file, c.says, err) + } + } +} + +// fallingBackTo is inConfigForm with its font size falling back to a predefined key. +func fallingBackTo(t *testing.T, key string) []byte { + t.Helper() + m := inConfigForm() + declaration(m, "font-size")["predefined"] = key + raw, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + return raw +} + +// A manifest naming font-size registers against the list; one naming a key not on it, or of another type, is +// refused — judged against the list given, whatever the mesh published. +func TestAManifestIsJudgedAgainstTheListOfItsTree(t *testing.T) { + withPredefined(t, nil) + list, err := ParsePredefined([]byte(theFirstKeys)) + if err != nil { + t.Fatal(err) + } + if _, err := ParseManifestWith(fallingBackTo(t, "font-size"), list); err != nil { + t.Fatalf("a value falling back to font-size is refused: %v", err) + } + if _, err := ParseManifestWith(fallingBackTo(t, "font-scale"), list); err == nil || + !strings.Contains(err.Error(), `"font-scale", which is not on the list`) { + t.Fatalf("a key not on the list: %v", err) + } + m := inConfigForm() + declaration(m, "theme")["predefined"] = "font-size" + raw, _ := json.Marshal(m) + if _, err := ParseManifestWith(raw, list); err == nil || !strings.Contains(err.Error(), "which is a number") { + t.Fatalf("a key of another type: %v", err) + } + // Where no tree is read, the published list judges: none published refuses. + if _, err := ParseManifest(fallingBackTo(t, "font-size")); err == nil || !strings.Contains(err.Error(), "is not present") { + t.Fatalf("judged against no list: %v", err) + } + withPredefined(t, list) + if _, err := ParseManifest(fallingBackTo(t, "font-size")); err != nil { + t.Fatalf("judged against the published list: %v", err) + } +} + +// A template's text filled from the tree keeps the list the manifest was judged against: read again, it is not +// judged against the published one. +func TestTemplatesFilledKeepTheListTheManifestWasJudgedAgainst(t *testing.T) { + withPredefined(t, nil) + list, _ := ParsePredefined([]byte(theFirstKeys)) + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, TemplatesDir), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, TemplatesDir, "rc"), []byte("font = ${config:font-size}\n"), 0o644); err != nil { + t.Fatal(err) + } + m := inConfigForm() + declaration(m, "font-size")["predefined"] = "font-size" + r := resourceOf(m, "configuration") + delete(r, "content") + r["template"] = TemplatesDir + "/rc" + m["build"] = map[string]any{"artifacts": []any{map[string]any{"name": "files", "kind": "archive", "from": "files"}}} + declaration(m, "width")["runtime"] = true + declaration(m, "sender")["runtime"] = true + raw, _ := json.Marshal(m) + parsed, err := ParseManifestWith(raw, list) + if err != nil { + t.Fatal(err) + } + if _, err := WithTemplatesFrom(parsed, dir); err != nil { + t.Fatalf("filled from the tree, it was judged against another list: %v", err) + } +} + +// module check reads the list at the root of the repository above the manifest; no repository, no list. +func TestTheListIsFoundAtTheRepositoryRootAboveAManifest(t *testing.T) { + root := t.TempDir() + module := filepath.Join(root, "modules", "dunst") + if err := os.MkdirAll(module, 0o755); err != nil { + t.Fatal(err) + } + if list, err := PredefinedAbove(module); err != nil || list != nil { + t.Fatalf("outside a repository: %v %v", list, err) + } + if err := os.Mkdir(filepath.Join(root, ".git"), 0o755); err != nil { + t.Fatal(err) + } + if list, err := PredefinedAbove(module); err != nil || list != nil { + t.Fatalf("a repository without the file: %v %v", list, err) + } + if err := os.WriteFile(filepath.Join(root, PredefinedFile), []byte(theFirstKeys), 0o644); err != nil { + t.Fatal(err) + } + list, err := PredefinedAbove(module) + if err != nil || len(list) != 2 { + t.Fatalf("the list above: %v %v", list, err) + } + if err := os.WriteFile(filepath.Join(root, PredefinedFile), []byte(`{"font-size": {}}`), 0o644); err != nil { + t.Fatal(err) + } + if _, err := PredefinedAbove(module); err == nil { + t.Fatal("a wrong list above was read") + } + if err := os.Remove(filepath.Join(root, PredefinedFile)); err != nil { + t.Fatal(err) + } + if err := os.Symlink("/etc/hostname", filepath.Join(root, PredefinedFile)); err == nil { + if _, err := PredefinedAbove(module); err == nil || !strings.Contains(err.Error(), "not a plain file") { + t.Fatalf("a link read as the list: %v", err) + } + } +} diff --git a/internal/catalogue/template.go b/internal/catalogue/template.go index 2a363bf2..d4aa104a 100644 --- a/internal/catalogue/template.go +++ b/internal/catalogue/template.go @@ -312,7 +312,7 @@ func WithTemplatesFrom(m Manifest, dir string) (Manifest, error) { if err != nil { return Manifest{}, err } - return ParseManifest(raw) + return parseManifest(raw, m.predefined) } // HeldTemplateProblem is why a manifest cannot be held by the mesh because it still names a template, or "": the diff --git a/internal/inventory/migrations/0098-the-list-of-predefined-keys.sql b/internal/inventory/migrations/0098-the-list-of-predefined-keys.sql new file mode 100644 index 00000000..fa4d42b4 --- /dev/null +++ b/internal/inventory/migrations/0098-the-list-of-predefined-keys.sql @@ -0,0 +1,20 @@ +-- The list of predefined keys (novox/hq ADR 0348 clause 7, design 57 package K2, issue 661). +-- +-- Mesh config and node config hold values only for the keys in the catalogue's `predefined.json`. A build of +-- the catalogue reads the file at the commit it builds and carries it; the controller keeps the newest trunk +-- build's, which every manifest it reads again is judged against, and which mesh-config publishes from here +-- (design 57 package K3). One list for the mesh: one row. +-- +-- **The newest trunk build wins**, by the generation of the commit built — how many commits it stands on, which +-- of two commits on one trunk is larger for the later — so a list heard late from an older build never replaces +-- a newer one. Not a config value: a key, its type, description, validation and whether it is trusted, all of it +-- public in the catalogue's repository. +create table predefined_list ( + one boolean primary key default true check (one), + keys jsonb not null, + repository text not null, + commit text not null, + generation bigint not null check (generation > 0), + build text not null, + kept_at timestamptz not null default now() +); diff --git a/internal/inventory/predefined.go b/internal/inventory/predefined.go new file mode 100644 index 00000000..03fc588e --- /dev/null +++ b/internal/inventory/predefined.go @@ -0,0 +1,76 @@ +package inventory + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// The list of predefined keys the mesh publishes (novox/hq ADR 0348 §7, design 57 package K2): the newest trunk +// build's of the catalogue, by the generation of the commit built. + +// PredefinedRecord is the list kept, and the build it came from. +type PredefinedRecord struct { + Keys catalogue.PredefinedList + Repository string + Commit string + Generation int + Build string + KeptAt time.Time +} + +// KeepPredefined keeps a trunk build's list when its commit is newer than the one kept — a larger generation — +// and answers whether it did. An older build's list, or one of the same generation, heard later is not kept: the +// newer one stands, whatever order the outcomes arrive in. One statement, so two builds heard at once cannot both +// win. +func (i *Inventory) KeepPredefined(ctx context.Context, r PredefinedRecord) (bool, error) { + if r.Keys == nil { + return false, errors.New("no list to keep") + } + if r.Generation <= 0 { + return false, fmt.Errorf("the build %s did not say its commit's generation, so whether its list is newer "+ + "than the one kept cannot be said", r.Build) + } + if problems := catalogue.PredefinedProblems(r.Keys); len(problems) > 0 { + return false, fmt.Errorf("the list of %s is refused: %v", r.Build, problems) + } + text, err := r.Keys.Canonical() + if err != nil { + return false, err + } + tag, err := i.store.Pool().Exec(ctx, ` + insert into predefined_list (keys, repository, commit, generation, build) + values ($1, $2, $3, $4, $5) + on conflict (one) do update set keys = excluded.keys, repository = excluded.repository, + commit = excluded.commit, generation = excluded.generation, build = excluded.build, kept_at = now() + where predefined_list.generation < excluded.generation`, + text, r.Repository, r.Commit, r.Generation, r.Build) + if err != nil { + return false, err + } + return tag.RowsAffected() == 1, nil +} + +// PredefinedKept is the list kept, and false when none is yet. +func (i *Inventory) PredefinedKept(ctx context.Context) (PredefinedRecord, bool, error) { + var r PredefinedRecord + var text []byte + err := i.store.Pool().QueryRow(ctx, + `select keys, repository, commit, generation, build, kept_at from predefined_list`). + Scan(&text, &r.Repository, &r.Commit, &r.Generation, &r.Build, &r.KeptAt) + if errors.Is(err, pgx.ErrNoRows) { + return PredefinedRecord{}, false, nil + } + if err != nil { + return PredefinedRecord{}, false, err + } + if r.Keys, err = catalogue.ParsePredefined(text); err != nil { + return PredefinedRecord{}, false, fmt.Errorf("the list kept from %s cannot be read: %w", r.Build, err) + } + return r, true, nil +} diff --git a/internal/inventory/predefined_test.go b/internal/inventory/predefined_test.go new file mode 100644 index 00000000..d2ab52d2 --- /dev/null +++ b/internal/inventory/predefined_test.go @@ -0,0 +1,69 @@ +package inventory + +import ( + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// The list of predefined keys the controller keeps (novox/hq ADR 0348 §7 and its check "a newer trunk build's +// list replaces the published one, and an older build's list heard later does not"; design 57 package K2). + +func listOf(t *testing.T, text string) catalogue.PredefinedList { + t.Helper() + l, err := catalogue.ParsePredefined([]byte(text)) + if err != nil { + t.Fatal(err) + } + return l +} + +func TestANewerBuildsListReplacesTheKeptOneAndAnOlderOneHeardLaterDoesNot(t *testing.T) { + inv := ForTest(t) + ctx := t.Context() + if _, has, err := inv.PredefinedKept(ctx); err != nil || has { + t.Fatalf("a fresh store keeps a list: %v %v", has, err) + } + older := PredefinedRecord{Keys: listOf(t, `{"font-size": {"type": "number", "description": "points"}}`), + Repository: "novox/mesh-catalog", Commit: "aaaa", Generation: 100, Build: "b-older"} + newer := PredefinedRecord{Keys: listOf(t, `{"font-size": {"type": "number", "description": "points"}, + "agent-managed-settings": {"type": "object", "description": "a JSON object", "trusted": true}}`), + Repository: "novox/mesh-catalog", Commit: "bbbb", Generation: 101, Build: "b-newer"} + + if kept, err := inv.KeepPredefined(ctx, older); err != nil || !kept { + t.Fatalf("the first list was not kept: %v %v", kept, err) + } + if kept, err := inv.KeepPredefined(ctx, newer); err != nil || !kept { + t.Fatalf("a newer build's list did not replace the kept one: %v %v", kept, err) + } + // The older build's outcome heard after the newer one's. + if kept, err := inv.KeepPredefined(ctx, older); err != nil || kept { + t.Fatalf("an older build's list heard later replaced the newer one: %v %v", kept, err) + } + // Nor one of the same generation: the one kept stands. + same := newer + same.Keys, same.Build = older.Keys, "b-again" + if kept, err := inv.KeepPredefined(ctx, same); err != nil || kept { + t.Fatalf("a list of the same generation replaced the kept one: %v %v", kept, err) + } + got, has, err := inv.PredefinedKept(ctx) + if err != nil || !has { + t.Fatal(has, err) + } + if got.Build != "b-newer" || got.Commit != "bbbb" || got.Generation != 101 || len(got.Keys) != 2 || + !got.Keys["agent-managed-settings"].Trusted { + t.Fatalf("kept %+v", got) + } + + // A build that could not say its generation, or carried a wrong list, is never kept. + unordered := newer + unordered.Generation = 0 + if kept, err := inv.KeepPredefined(ctx, unordered); err == nil || kept { + t.Fatalf("a list without a generation was kept: %v", err) + } + wrong := newer + wrong.Generation, wrong.Keys = 200, catalogue.PredefinedList{"accent": {Type: "object", Description: "x"}} + if kept, err := inv.KeepPredefined(ctx, wrong); err == nil || kept { + t.Fatalf("a wrong list was kept: %v", err) + } +} diff --git a/internal/link/build.go b/internal/link/build.go index 6eb16588..a8d56c18 100644 --- a/internal/link/build.go +++ b/internal/link/build.go @@ -334,6 +334,12 @@ type BuildResult struct { ReusedFrom *ReusedFrom `json:"reused-from,omitempty"` BuiltWhy string `json:"built-why,omitempty"` + // Predefined is the list of predefined keys at the root of the repository at the commit built (novox/hq ADR + // 0348 §7), which the manifest was judged against: the controller publishes the catalogue's from a trunk + // build, the newest by Generation winning. Absent for a repository holding none, and from a builder that + // predates the list. + Predefined *PredefinedList `json:"predefined,omitempty"` + // Source is the request's, echoed: the seat form of the repository, for whoever registers. Source *SourceOnSeat `json:"source,omitempty"` @@ -346,6 +352,14 @@ type BuildResult struct { Checked *CheckRequest `json:"check,omitempty"` } +// PredefinedList is the list of predefined keys a build read (novox/hq ADR 0348 §7): Keys its canonical text, a +// JSON object of keys, and Generation how many commits the commit built stands on — of two commits on one trunk +// the later has more, which is what orders two builds' lists; 0 when the builder could not say. +type PredefinedList struct { + Keys json.RawMessage `json:"keys"` + Generation int `json:"generation,omitempty"` +} + // ReadRepository is a repository a build read source from besides the module's own, at the branch, // tag or commit it read. Spelled here as well as in the catalogue and the inventory, for the reason // MadeArtifact is: one direction of dependency. diff --git a/testdata/store-tests b/testdata/store-tests index d3af9659..725df90d 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -61,6 +61,20 @@ ./cmd/mesh-controller TestAProviderElsewhereServesItsOwnComposedName ./cmd/mesh-controller TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne ./internal/inventory TestAReportOfABusyContainerIsKeptAndReadBack +./internal/inventory TestANewerBuildsListReplacesTheKeptOneAndAnOlderOneHeardLaterDoesNot +./cmd/mesh-controller TestTheMergeGateJudgesEachManifestAgainstTheListOfTheTree +./cmd/mesh-controller TestRegistrationPublishesTheNewestTrunkListAndJudgesEveryModuleAgainstIt ./internal/inventory TestPlacementMovesOutOfTheSettingsLayers ./internal/inventory TestASettingRoutesPlacementBesideTheLayer ./cmd/mesh-controller TestADeclarationIsTheSameAfterPlacementMoves +./cmd/mesh-controller TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses +./cmd/mesh-controller TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn +./cmd/mesh-controller TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere +./cmd/mesh-controller TestAFirstSendMadeAndNotKeptIsNotSentAgain +./cmd/mesh-controller TestAReleaseSendKeptAndMadeIsNotMadeAgain +./cmd/mesh-controller TestAFirstMachineThatFailsStillStopsTheWalk +./cmd/mesh-controller TestTheControlNodeIsSentLastInAStep +./cmd/mesh-controller TestARestMachineSilentPastItsBoundIsLeftAndSaid +./cmd/mesh-controller TestANoMoveTierWaitsForNoNewReport +./cmd/mesh-controller TestOneProviderDownAndThreeConsumersFailingAreOneCondition +./cmd/mesh-controller TestAConsumerFailingBesideAHealthyProviderIsItsOwn