diff --git a/internal/catalogue/predefined_test.go b/internal/catalogue/predefined_test.go new file mode 100644 index 00000000..c683f763 --- /dev/null +++ b/internal/catalogue/predefined_test.go @@ -0,0 +1,175 @@ +package catalogue + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +// The list of predefined keys (novox/hq ADR 0348 §7, design 57 package K2, issue 661): read strictly, and every +// manifest naming a key judged against the list of the tree it is read in. + +// theFirstKeys is the catalogue's first list: font-size and agent-managed-settings. +const theFirstKeys = `{ + "font-size": {"type": "number", "description": "the size text is shown in, in points", "minimum": 6, "maximum": 72}, + "agent-managed-settings": {"type": "object", "description": "the managed settings of every agent session, a JSON object", "trusted": true} +}` + +func TestTheFirstListIsRead(t *testing.T) { + list, err := ParsePredefined([]byte(theFirstKeys)) + if err != nil { + t.Fatal(err) + } + if got := strings.Join(list.Keys(), ","); got != "agent-managed-settings,font-size" { + t.Fatalf("keys %s", got) + } + if k := list["font-size"]; k.Type != ConfigNumber || *k.Minimum != 6 || *k.Maximum != 72 || k.Trusted { + t.Fatalf("font-size read as %+v", k) + } + if k := list["agent-managed-settings"]; k.Type != ConfigObject || !k.Trusted { + t.Fatalf("agent-managed-settings read as %+v", k) + } + // One list is one text, however it was written. + a, _ := list.Canonical() + again, _ := ParsePredefined(a) + b, _ := again.Canonical() + if string(a) != string(b) { + t.Fatalf("canonical text moved:\n%s\n%s", a, b) + } +} + +func TestAListIsRefusedWholeForAnyWrongKey(t *testing.T) { + for _, c := range []struct{ file, says string }{ + {`[]`, "not a JSON object of keys"}, + {`null`, "not a JSON object of keys"}, + {`{"font-size": {"type": "number", "description": "points", "default": 10}}`, `unknown field "default"`}, + {`{"font-size": {"description": "points"}}`, "no type"}, + {`{"font-size": {"type": "decimal", "description": "points"}}`, `type "decimal"`}, + {`{"font-size": {"type": "number"}}`, "no description"}, + {`{"accent": {"type": "object", "description": "colours"}}`, "not trusted"}, + {`{"font-size": {"type": "number", "description": "points", "minimum": 9, "maximum": 6}}`, "above its maximum"}, + {`{"accent": {"type": "string", "description": "a name", "pattern": "("}}`, "not a regular expression"}, + {`{"accent": {"type": "boolean", "description": "on or off", "minimum": 1}}`, "minimum or maximum is for a number"}, + {`{"Font Size": {"type": "number", "description": "points"}}`, "not a usable key"}, + {`{"places": {"type": "string", "description": "where"}}`, "placement"}, + {`{"puid": {"type": "number", "description": "an id"}}`, "placement"}, + } { + if _, err := ParsePredefined([]byte(c.file)); err == nil || !strings.Contains(err.Error(), c.says) { + t.Errorf("%s: wanted a refusal saying %q, got %v", c.file, c.says, err) + } + } +} + +// fallingBackTo is inConfigForm with its font size falling back to a predefined key. +func fallingBackTo(t *testing.T, key string) []byte { + t.Helper() + m := inConfigForm() + declaration(m, "font-size")["predefined"] = key + raw, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + return raw +} + +// A manifest naming font-size registers against the list; one naming a key not on it, or of another type, is +// refused — judged against the list given, whatever the mesh published. +func TestAManifestIsJudgedAgainstTheListOfItsTree(t *testing.T) { + withPredefined(t, nil) + list, err := ParsePredefined([]byte(theFirstKeys)) + if err != nil { + t.Fatal(err) + } + if _, err := ParseManifestWith(fallingBackTo(t, "font-size"), list); err != nil { + t.Fatalf("a value falling back to font-size is refused: %v", err) + } + if _, err := ParseManifestWith(fallingBackTo(t, "font-scale"), list); err == nil || + !strings.Contains(err.Error(), `"font-scale", which is not on the list`) { + t.Fatalf("a key not on the list: %v", err) + } + m := inConfigForm() + declaration(m, "theme")["predefined"] = "font-size" + raw, _ := json.Marshal(m) + if _, err := ParseManifestWith(raw, list); err == nil || !strings.Contains(err.Error(), "which is a number") { + t.Fatalf("a key of another type: %v", err) + } + // Where no tree is read, the published list judges: none published refuses. + if _, err := ParseManifest(fallingBackTo(t, "font-size")); err == nil || !strings.Contains(err.Error(), "is not present") { + t.Fatalf("judged against no list: %v", err) + } + withPredefined(t, list) + if _, err := ParseManifest(fallingBackTo(t, "font-size")); err != nil { + t.Fatalf("judged against the published list: %v", err) + } +} + +// A template's text filled from the tree keeps the list the manifest was judged against: read again, it is not +// judged against the published one. +func TestTemplatesFilledKeepTheListTheManifestWasJudgedAgainst(t *testing.T) { + withPredefined(t, nil) + list, _ := ParsePredefined([]byte(theFirstKeys)) + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, TemplatesDir), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, TemplatesDir, "rc"), []byte("font = ${config:font-size}\n"), 0o644); err != nil { + t.Fatal(err) + } + m := inConfigForm() + declaration(m, "font-size")["predefined"] = "font-size" + r := resourceOf(m, "configuration") + delete(r, "content") + r["template"] = TemplatesDir + "/rc" + m["build"] = map[string]any{"artifacts": []any{map[string]any{"name": "files", "kind": "archive", "from": "files"}}} + declaration(m, "width")["runtime"] = true + declaration(m, "sender")["runtime"] = true + raw, _ := json.Marshal(m) + parsed, err := ParseManifestWith(raw, list) + if err != nil { + t.Fatal(err) + } + if _, err := WithTemplatesFrom(parsed, dir); err != nil { + t.Fatalf("filled from the tree, it was judged against another list: %v", err) + } +} + +// module check reads the list at the root of the repository above the manifest; no repository, no list. +func TestTheListIsFoundAtTheRepositoryRootAboveAManifest(t *testing.T) { + root := t.TempDir() + module := filepath.Join(root, "modules", "dunst") + if err := os.MkdirAll(module, 0o755); err != nil { + t.Fatal(err) + } + if list, err := PredefinedAbove(module); err != nil || list != nil { + t.Fatalf("outside a repository: %v %v", list, err) + } + if err := os.Mkdir(filepath.Join(root, ".git"), 0o755); err != nil { + t.Fatal(err) + } + if list, err := PredefinedAbove(module); err != nil || list != nil { + t.Fatalf("a repository without the file: %v %v", list, err) + } + if err := os.WriteFile(filepath.Join(root, PredefinedFile), []byte(theFirstKeys), 0o644); err != nil { + t.Fatal(err) + } + list, err := PredefinedAbove(module) + if err != nil || len(list) != 2 { + t.Fatalf("the list above: %v %v", list, err) + } + if err := os.WriteFile(filepath.Join(root, PredefinedFile), []byte(`{"font-size": {}}`), 0o644); err != nil { + t.Fatal(err) + } + if _, err := PredefinedAbove(module); err == nil { + t.Fatal("a wrong list above was read") + } + if err := os.Remove(filepath.Join(root, PredefinedFile)); err != nil { + t.Fatal(err) + } + if err := os.Symlink("/etc/hostname", filepath.Join(root, PredefinedFile)); err == nil { + if _, err := PredefinedAbove(module); err == nil || !strings.Contains(err.Error(), "not a plain file") { + t.Fatalf("a link read as the list: %v", err) + } + } +} diff --git a/internal/catalogue/template.go b/internal/catalogue/template.go index 2a363bf2..d4aa104a 100644 --- a/internal/catalogue/template.go +++ b/internal/catalogue/template.go @@ -312,7 +312,7 @@ func WithTemplatesFrom(m Manifest, dir string) (Manifest, error) { if err != nil { return Manifest{}, err } - return ParseManifest(raw) + return parseManifest(raw, m.predefined) } // HeldTemplateProblem is why a manifest cannot be held by the mesh because it still names a template, or "": the diff --git a/internal/inventory/migrations/0098-the-list-of-predefined-keys.sql b/internal/inventory/migrations/0098-the-list-of-predefined-keys.sql new file mode 100644 index 00000000..fa4d42b4 --- /dev/null +++ b/internal/inventory/migrations/0098-the-list-of-predefined-keys.sql @@ -0,0 +1,20 @@ +-- The list of predefined keys (novox/hq ADR 0348 clause 7, design 57 package K2, issue 661). +-- +-- Mesh config and node config hold values only for the keys in the catalogue's `predefined.json`. A build of +-- the catalogue reads the file at the commit it builds and carries it; the controller keeps the newest trunk +-- build's, which every manifest it reads again is judged against, and which mesh-config publishes from here +-- (design 57 package K3). One list for the mesh: one row. +-- +-- **The newest trunk build wins**, by the generation of the commit built — how many commits it stands on, which +-- of two commits on one trunk is larger for the later — so a list heard late from an older build never replaces +-- a newer one. Not a config value: a key, its type, description, validation and whether it is trusted, all of it +-- public in the catalogue's repository. +create table predefined_list ( + one boolean primary key default true check (one), + keys jsonb not null, + repository text not null, + commit text not null, + generation bigint not null check (generation > 0), + build text not null, + kept_at timestamptz not null default now() +); diff --git a/internal/inventory/predefined.go b/internal/inventory/predefined.go new file mode 100644 index 00000000..03fc588e --- /dev/null +++ b/internal/inventory/predefined.go @@ -0,0 +1,76 @@ +package inventory + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// The list of predefined keys the mesh publishes (novox/hq ADR 0348 §7, design 57 package K2): the newest trunk +// build's of the catalogue, by the generation of the commit built. + +// PredefinedRecord is the list kept, and the build it came from. +type PredefinedRecord struct { + Keys catalogue.PredefinedList + Repository string + Commit string + Generation int + Build string + KeptAt time.Time +} + +// KeepPredefined keeps a trunk build's list when its commit is newer than the one kept — a larger generation — +// and answers whether it did. An older build's list, or one of the same generation, heard later is not kept: the +// newer one stands, whatever order the outcomes arrive in. One statement, so two builds heard at once cannot both +// win. +func (i *Inventory) KeepPredefined(ctx context.Context, r PredefinedRecord) (bool, error) { + if r.Keys == nil { + return false, errors.New("no list to keep") + } + if r.Generation <= 0 { + return false, fmt.Errorf("the build %s did not say its commit's generation, so whether its list is newer "+ + "than the one kept cannot be said", r.Build) + } + if problems := catalogue.PredefinedProblems(r.Keys); len(problems) > 0 { + return false, fmt.Errorf("the list of %s is refused: %v", r.Build, problems) + } + text, err := r.Keys.Canonical() + if err != nil { + return false, err + } + tag, err := i.store.Pool().Exec(ctx, ` + insert into predefined_list (keys, repository, commit, generation, build) + values ($1, $2, $3, $4, $5) + on conflict (one) do update set keys = excluded.keys, repository = excluded.repository, + commit = excluded.commit, generation = excluded.generation, build = excluded.build, kept_at = now() + where predefined_list.generation < excluded.generation`, + text, r.Repository, r.Commit, r.Generation, r.Build) + if err != nil { + return false, err + } + return tag.RowsAffected() == 1, nil +} + +// PredefinedKept is the list kept, and false when none is yet. +func (i *Inventory) PredefinedKept(ctx context.Context) (PredefinedRecord, bool, error) { + var r PredefinedRecord + var text []byte + err := i.store.Pool().QueryRow(ctx, + `select keys, repository, commit, generation, build, kept_at from predefined_list`). + Scan(&text, &r.Repository, &r.Commit, &r.Generation, &r.Build, &r.KeptAt) + if errors.Is(err, pgx.ErrNoRows) { + return PredefinedRecord{}, false, nil + } + if err != nil { + return PredefinedRecord{}, false, err + } + if r.Keys, err = catalogue.ParsePredefined(text); err != nil { + return PredefinedRecord{}, false, fmt.Errorf("the list kept from %s cannot be read: %w", r.Build, err) + } + return r, true, nil +} diff --git a/internal/inventory/predefined_test.go b/internal/inventory/predefined_test.go new file mode 100644 index 00000000..d2ab52d2 --- /dev/null +++ b/internal/inventory/predefined_test.go @@ -0,0 +1,69 @@ +package inventory + +import ( + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// The list of predefined keys the controller keeps (novox/hq ADR 0348 §7 and its check "a newer trunk build's +// list replaces the published one, and an older build's list heard later does not"; design 57 package K2). + +func listOf(t *testing.T, text string) catalogue.PredefinedList { + t.Helper() + l, err := catalogue.ParsePredefined([]byte(text)) + if err != nil { + t.Fatal(err) + } + return l +} + +func TestANewerBuildsListReplacesTheKeptOneAndAnOlderOneHeardLaterDoesNot(t *testing.T) { + inv := ForTest(t) + ctx := t.Context() + if _, has, err := inv.PredefinedKept(ctx); err != nil || has { + t.Fatalf("a fresh store keeps a list: %v %v", has, err) + } + older := PredefinedRecord{Keys: listOf(t, `{"font-size": {"type": "number", "description": "points"}}`), + Repository: "novox/mesh-catalog", Commit: "aaaa", Generation: 100, Build: "b-older"} + newer := PredefinedRecord{Keys: listOf(t, `{"font-size": {"type": "number", "description": "points"}, + "agent-managed-settings": {"type": "object", "description": "a JSON object", "trusted": true}}`), + Repository: "novox/mesh-catalog", Commit: "bbbb", Generation: 101, Build: "b-newer"} + + if kept, err := inv.KeepPredefined(ctx, older); err != nil || !kept { + t.Fatalf("the first list was not kept: %v %v", kept, err) + } + if kept, err := inv.KeepPredefined(ctx, newer); err != nil || !kept { + t.Fatalf("a newer build's list did not replace the kept one: %v %v", kept, err) + } + // The older build's outcome heard after the newer one's. + if kept, err := inv.KeepPredefined(ctx, older); err != nil || kept { + t.Fatalf("an older build's list heard later replaced the newer one: %v %v", kept, err) + } + // Nor one of the same generation: the one kept stands. + same := newer + same.Keys, same.Build = older.Keys, "b-again" + if kept, err := inv.KeepPredefined(ctx, same); err != nil || kept { + t.Fatalf("a list of the same generation replaced the kept one: %v %v", kept, err) + } + got, has, err := inv.PredefinedKept(ctx) + if err != nil || !has { + t.Fatal(has, err) + } + if got.Build != "b-newer" || got.Commit != "bbbb" || got.Generation != 101 || len(got.Keys) != 2 || + !got.Keys["agent-managed-settings"].Trusted { + t.Fatalf("kept %+v", got) + } + + // A build that could not say its generation, or carried a wrong list, is never kept. + unordered := newer + unordered.Generation = 0 + if kept, err := inv.KeepPredefined(ctx, unordered); err == nil || kept { + t.Fatalf("a list without a generation was kept: %v", err) + } + wrong := newer + wrong.Generation, wrong.Keys = 200, catalogue.PredefinedList{"accent": {Type: "object", Description: "x"}} + if kept, err := inv.KeepPredefined(ctx, wrong); err == nil || kept { + t.Fatalf("a wrong list was kept: %v", err) + } +} diff --git a/internal/link/build.go b/internal/link/build.go index 73384e9f..93172b5d 100644 --- a/internal/link/build.go +++ b/internal/link/build.go @@ -358,6 +358,12 @@ type BuildResult struct { ReusedFrom *ReusedFrom `json:"reused-from,omitempty"` BuiltWhy string `json:"built-why,omitempty"` + // Predefined is the list of predefined keys at the root of the repository at the commit built (novox/hq ADR + // 0348 §7), which the manifest was judged against: the controller publishes the catalogue's from a trunk + // build, the newest by Generation winning. Absent for a repository holding none, and from a builder that + // predates the list. + Predefined *PredefinedList `json:"predefined,omitempty"` + // Source is the request's, echoed: the seat form of the repository, for whoever registers. Source *SourceOnSeat `json:"source,omitempty"` @@ -370,6 +376,14 @@ type BuildResult struct { Checked *CheckRequest `json:"check,omitempty"` } +// PredefinedList is the list of predefined keys a build read (novox/hq ADR 0348 §7): Keys its canonical text, a +// JSON object of keys, and Generation how many commits the commit built stands on — of two commits on one trunk +// the later has more, which is what orders two builds' lists; 0 when the builder could not say. +type PredefinedList struct { + Keys json.RawMessage `json:"keys"` + Generation int `json:"generation,omitempty"` +} + // ReadRepository is a repository a build read source from besides the module's own, at the branch, // tag or commit it read. Spelled here as well as in the catalogue and the inventory, for the reason // MadeArtifact is: one direction of dependency. diff --git a/testdata/store-tests b/testdata/store-tests index 6b0cc587..6d3153f3 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -50,6 +50,7 @@ ./internal/inventory TestOneAcceptForAProviderIsSealedToEveryConsumer ./internal/inventory TestASharedOwnSecretIssuedOutsideIsNeverMade ./internal/recordmirror TestNatsAnAssignmentAndARegistrationEachChangeOneKeyAndRaiseOneEvent +./internal/inventory TestAModuleInTheConfigFormIsMirroredWithItsWholeDeclaration ./cmd/mesh-controller TestOneAcceptForAProviderIsPlacedForEveryConsumer ./cmd/mesh-controller TestReplay164 ./cmd/mesh-controller TestAnApprovedLayerOpensTheWalkThatSendsIt @@ -62,6 +63,9 @@ ./cmd/mesh-controller TestAProviderElsewhereServesItsOwnComposedName ./cmd/mesh-controller TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne ./internal/inventory TestAReportOfABusyContainerIsKeptAndReadBack +./internal/inventory TestANewerBuildsListReplacesTheKeptOneAndAnOlderOneHeardLaterDoesNot +./cmd/mesh-controller TestTheMergeGateJudgesEachManifestAgainstTheListOfTheTree +./cmd/mesh-controller TestRegistrationPublishesTheNewestTrunkListAndJudgesEveryModuleAgainstIt ./internal/inventory TestPlacementMovesOutOfTheSettingsLayers ./internal/inventory TestASettingRoutesPlacementBesideTheLayer ./cmd/mesh-controller TestADeclarationIsTheSameAfterPlacementMoves @@ -76,4 +80,3 @@ ./cmd/mesh-controller TestANoMoveTierWaitsForNoNewReport ./cmd/mesh-controller TestOneProviderDownAndThreeConsumersFailingAreOneCondition ./cmd/mesh-controller TestAConsumerFailingBesideAHealthyProviderIsItsOwn -./internal/inventory TestAModuleInTheConfigFormIsMirroredWithItsWholeDeclaration